<commit_before>
from __future__ import absolute_import, division

import logging
import os
import re

from matplotlib import pyplot as plt
import numpy as np

# Misc extras
from adjustText.adjustText import adjust_text

import pyproteome as pyp

LOGGER = logging.getLogger("pyproteome.volcano")

def _make_folder(data=None, folder_name=None):
    if folder_name is None:
        folder_name = os.path.join(
            data.name
            if data is not None else
            os.getcwd(),
            "Volcano",
        )

    return pyp.utils.makedirs(folder_name)

def _remove_lesser_dups(labels, compress_sym=False):
    new_labels = []

    for index, (p, change, label, color, highlight) in enumerate(labels):
        for o_index, (o_p, o_change, o_label, _, _) in enumerate(labels):
            if index == o_index:
                continue
            if label != o_label:
                continue
            if not compress_sym and (change < 0) != (o_change < 0):
                continue

            mul = 1 if change >= 0 else -1
            o_mul = 1 if o_change >= 0 else -1

            if (
                p + mul * change < o_p + o_mul * o_change
            ) or (
                (
                    p + mul * change <= o_p + o_mul * o_change
                ) and
                index < o_index
            ):
                break
        else:
            new_labels.append((p, change, label, color, highlight))

    return new_labels

def _get_color(txt, x, y):
    return "#BFEE90" if x > 0 else "#FFC1C1"

def volcano_plot(
    data,
    group_a=None,
    group_b=None,
    p=0.05,
    fold=1.25,
    fold_and_p=True,
    xminmax=None, yminmax=None,
    options=None,
    title=None,
    filename=None,
    folder_name=None,
    figsize=(12, 10),
    adjust=True,
    compress_dups=True,
    compress_sym=False,
    show_xlabel=True,
    show_ylabel=True,
    show_title=True,
    full_site_labels=False,
    sequence_labels=False,
    alpha=1,
    bonferoni=False,
):
    """
    Display a volcano plot of data.

    This plot inclues the fold-changes and p-values associated with said
    changes.

    Parameters
    ----------
    data : :class:`DataSet<pyproteome.data_sets.DataSet>`
    group_a : str or list of str, optional
    group_b : str or list of str, optional
    pval_cutoff : float, optional
    fold : float, optional
    options: dict of (str, list), optional
    folder_name : str, optional
    title : str, optional
    figsize : tuple of float, float
    adjust : bool, optional
    compress_dups : bool, optional
    full_site_labels : bool, optional
    """
    (channels_a, channels_b), (label_a, label_b), _ = data.get_groups(
        group_a=group_a,
        group_b=group_b,
    )

    if group_a and group_b:
        data.update_group_changes(group_a=group_a, group_b=group_b)

    folder_name = _make_folder(data, folder_name=folder_name)

    options = options or {}

    highlight = options.get('highlight', {})
    hide = options.get('hide', {})
    show = options.get('show', {})
    edgecolors = options.get('edgecolors', {})
    rename = options.get('rename', {})

    log_p = -np.log10(p)

    if not title:
        title = data.name

    if not filename:
        filename = re.sub("[ ></?]", "_", title) + ".png"

    upper_fold = np.log2(fold)
    lower_fold = -upper_fold

    rows = [
        row
        for _, row in data.psms.dropna(
            subset=["p-value", "Fold Change"],
        ).iterrows()
        if not (
            np.isnan(-np.log10(row["p-value"])) or
            np.isnan(np.log2(row["Fold Change"])) or
            np.isinf(-np.log10(row["p-value"])) or
            np.isinf(np.log2(row["Fold Change"]))
        )
    ]

    if bonferoni:
        p = p / len(rows)
        log_p = -np.log10(p)

    # Calculate the Fold-Change / p-values
    pvals = []
    changes = []
    colors = []
    labels = []

    for row in rows:
        color = "grey"
        row_pval = -np.log10(row["p-value"])
        row_change = np.log2(row["Fold Change"])

        if xminmax and (row_change < xminmax[0] or row_change > xminmax[1]):
            continue

        if sequence_labels:
            row_label = row["Sequence"].__str__()
        else:
            row_label = " / ".join(sorted(row["Proteins"].genes))
        old_row_label, old_re_row_label = None, None

        if (
            full_site_labels and
            len(list(row["Modifications"].skip_labels_iter())) > 0
        ):
            old_row_label = row_label
            old_re_row_label = rename.get(old_row_label, old_row_label)
            row_label = " / ".join(
                sorted(
                    "{} {}".format(
                        gene,
                        row["Modifications"].__str__(prot_index=index),
                    )
                    for index, gene in enumerate(row["Proteins"].genes)
                )
            )

        re_row_label = rename.get(row_label, row_label)
        edgecolor = edgecolors.get(
            re_row_label, edgecolors.get(
                row_label,
                edgecolors.get(old_row_label, None),
            )
        )

        if edgecolor is None:
            gene_colors = [
                edgecolors.get(gene, None)
                for gene in row["Proteins"].genes
            ]
            if len(set(gene_colors)) == 1:
                edgecolor = gene_colors[0]

        pvals.append(row_pval)
        changes.append(row_change)

        names = [
            row_label, re_row_label, old_row_label, old_re_row_label,
        ]

        if (
            any(
                i in show
                for i in names
            ) or ((
                row_pval > log_p and
                (row_change > upper_fold or row_change < lower_fold)
            ) if fold_and_p else (
                row_pval > log_p or
                (row_change > upper_fold or row_change < lower_fold)
            ))
        ):
            color = "blue"
            highlight_label = any(i in highlight for i in names)

            if (
                not any(i.strip() in hide for i in row_label.split("/")) and
                re_row_label not in hide
            ):
                labels.append((
                    row_pval, row_change,
                    re_row_label, edgecolor, highlight_label,
                ))

        colors.append(color)

    if compress_dups:
        labels = _remove_lesser_dups(labels, compress_sym=compress_sym)

    # Draw the figure
    fig, ax = plt.subplots(figsize=figsize)
    ax.scatter(changes, pvals, c=colors, alpha=alpha)
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)

    if xminmax:
        ax.set_xlim(xmin=xminmax[0], xmax=xminmax[1])
    else:
        ax.set_xlim(
            xmin=np.floor(min(changes) * 2) / 2,
            xmax=np.ceil(max(changes) * 2) / 2,
        )

    if yminmax:
        ax.set_ylim(bottom=yminmax[0], top=yminmax[1])
    else:
        ax.set_ylim(bottom=-0.1)

    ax.set_xticks(
        list(
            sorted(
                tick
                for tick in ax.get_xticks()
                if tick < lower_fold or tick > upper_fold
            ) + [lower_fold, upper_fold]
        )
    )
    ax.set_yticks(
        list(
            sorted(
                [
                    tick
                    for tick in ax.get_yticks()
                    if "{}".format(np.power(1/10, tick)).strip("0.")[:1] in
                    ["1", "5"] and
                    tick > log_p
                ] + [log_p]
            )
        )
    )
    ax.set_xticklabels(
        [
            "{:.3}".format(i)
            for i in np.exp2(ax.get_xticks())
        ],
        fontsize=20,
    )
    ax.set_yticklabels(
        [
            ("{:.3}" if i > 0.005 else "{:.0e}").format(i)
            for i in np.power(1/10, ax.get_yticks())
        ],
        fontsize=20,
    )

    if show_xlabel:
        ax.set_xlabel(
            "Fold Change {} (n={}) / {} (n={})".format(
                label_a,
                len(channels_a),
                label_b,
                len(channels_b),
            ),
            fontsize=20,
        )

    if show_ylabel:
        ax.set_ylabel(
            "p-value",
            fontsize=20,
        )

    if not np.isnan(log_p):
        ax.axhline(
            log_p,
            color="r", linestyle="dashed", linewidth=0.5,
        )

    if abs(fold - 1) > 0.01:
        ax.axvline(upper_fold, color="r", linestyle="dashed", linewidth=0.5)
        ax.axvline(lower_fold, color="r", linestyle="dashed", linewidth=0.5)

    # Position the labels
    texts = []
    txt_lim = 100 if full_site_labels else 20

    for y, x, txt, edgecolor, highlight_label in labels:
        text = ax.text(
            x, y,
            txt[:txt_lim] + ("..." if len(txt) > txt_lim else ""),
        )

        if highlight_label:
            text.set_fontsize(20)

        text.set_bbox(
            dict(
                facecolor=_get_color(txt, x, y),
                alpha=1,
                linewidth=0.5 if not edgecolor else 3,
                edgecolor=edgecolor or "black",
                boxstyle="round",
            )
        )

        texts.append(text)

    if adjust:
        adjust_text(
            x=[i[0] for i in labels],
            y=[i[1] for i in labels],
            texts=texts,
            ax=ax,
            lim=500,
            force_text=0.5,
            force_points=0.01,
            arrowprops=dict(arrowstyle="->", relpos=(0, 0), lw=1),
            only_move={
                "points": "y",
                "text": "xy",
            }
        )

    if title:
        if show_title:
            ax.set_title(
                title,
                fontsize=20,
            )

    if filename:
        fig.savefig(
            os.path.join(folder_name, filename),
            bbox_inches="tight",
            dpi=pyp.DEFAULT_DPI,
            transparent=True,
        )

    return fig, ax, folder_name, filename

def plot_volcano_filtered(data, f, **kwargs):
    data = data.copy()
    data.update_group_changes(
        group_a=kwargs.get("group_a", None),
        group_b=kwargs.get("group_b", None),
    )

    d = data.filter(f)

    changes = []
    pvals = []

    for _, row in data.psms.iterrows():
        row_pval = -np.log10(row["p-value"])
        row_change = np.log2(row["Fold Change"])

        if (
            np.isnan(row_pval) or
            np.isnan(row_change) or
            np.isinf(row_pval) or
            np.isinf(row_change)
        ):
            continue

        pvals.append(row_pval)
        changes.append(row_change)

    xminmax = kwargs.pop(
        "xminmax",
        (
            np.floor(min(changes) * 2) / 2,
            np.ceil(max(changes) * 2) / 2,
        ),
    )
    yminmax = kwargs.pop(
        "yminmax",
        (-0.1, np.ceil(max(pvals))),
    )

    f, ax, folder_name, filename = volcano_plot(
        d,
        xminmax=xminmax,
        yminmax=yminmax,
        **kwargs
    )

    changes, pvals = zip(*[
        (x, y)
        for x, y in zip(changes, pvals)
        if x > xminmax[0] and x < xminmax[1] and
        y > yminmax[0] and y < yminmax[1]
    ])

    ax.scatter(changes, pvals, c="lightblue", zorder=0, alpha=0.3)

    if filename:
        f.savefig(
            os.path.join(folder_name, filename),
            bbox_inches="tight",
            dpi=pyp.DEFAULT_DPI,
            transparent=True,
        )

    return f, ax
<commit_msg>Don't error on empty data<commit_after>
from __future__ import absolute_import, division

import logging
import os
import re

from matplotlib import pyplot as plt
import numpy as np

# Misc extras
from adjustText.adjustText import adjust_text

import pyproteome as pyp

LOGGER = logging.getLogger("pyproteome.volcano")

def _make_folder(data=None, folder_name=None):
    if folder_name is None:
        folder_name = os.path.join(
            data.name
            if data is not None else
            os.getcwd(),
            "Volcano",
        )

    return pyp.utils.makedirs(folder_name)

def _remove_lesser_dups(labels, compress_sym=False):
    new_labels = []

    for index, (p, change, label, color, highlight) in enumerate(labels):
        for o_index, (o_p, o_change, o_label, _, _) in enumerate(labels):
            if index == o_index:
                continue
            if label != o_label:
                continue
            if not compress_sym and (change < 0) != (o_change < 0):
                continue

            mul = 1 if change >= 0 else -1
            o_mul = 1 if o_change >= 0 else -1

            if (
                p + mul * change < o_p + o_mul * o_change
            ) or (
                (
                    p + mul * change <= o_p + o_mul * o_change
                ) and
                index < o_index
            ):
                break
        else:
            new_labels.append((p, change, label, color, highlight))

    return new_labels

def _get_color(txt, x, y):
    return "#BFEE90" if x > 0 else "#FFC1C1"

def volcano_plot(
    data,
    group_a=None,
    group_b=None,
    p=0.05,
    fold=1.25,
    fold_and_p=True,
    xminmax=None, yminmax=None,
    options=None,
    title=None,
    filename=None,
    folder_name=None,
    figsize=(12, 10),
    adjust=True,
    compress_dups=True,
    compress_sym=False,
    show_xlabel=True,
    show_ylabel=True,
    show_title=True,
    full_site_labels=False,
    sequence_labels=False,
    alpha=1,
    bonferoni=False,
):
    """
    Display a volcano plot of data.

    This plot inclues the fold-changes and p-values associated with said
    changes.

    Parameters
    ----------
    data : :class:`DataSet<pyproteome.data_sets.DataSet>`
    group_a : str or list of str, optional
    group_b : str or list of str, optional
    pval_cutoff : float, optional
    fold : float, optional
    options: dict of (str, list), optional
    folder_name : str, optional
    title : str, optional
    figsize : tuple of float, float
    adjust : bool, optional
    compress_dups : bool, optional
    full_site_labels : bool, optional
    """
    (channels_a, channels_b), (label_a, label_b), _ = data.get_groups(
        group_a=group_a,
        group_b=group_b,
    )

    if group_a and group_b:
        data.update_group_changes(group_a=group_a, group_b=group_b)

    folder_name = _make_folder(data, folder_name=folder_name)

    options = options or {}

    highlight = options.get('highlight', {})
    hide = options.get('hide', {})
    show = options.get('show', {})
    edgecolors = options.get('edgecolors', {})
    rename = options.get('rename', {})

    log_p = -np.log10(p)

    if not title:
        title = data.name

    if not filename:
        filename = re.sub("[ ></?]", "_", title) + ".png"

    upper_fold = np.log2(fold)
    lower_fold = -upper_fold

    rows = [
        row
        for _, row in data.psms.dropna(
            subset=["p-value", "Fold Change"],
        ).iterrows()
        if not (
            np.isnan(-np.log10(row["p-value"])) or
            np.isnan(np.log2(row["Fold Change"])) or
            np.isinf(-np.log10(row["p-value"])) or
            np.isinf(np.log2(row["Fold Change"]))
        )
    ]

    if bonferoni:
        p = p / len(rows)
        log_p = -np.log10(p)

    # Calculate the Fold-Change / p-values
    pvals = []
    changes = []
    colors = []
    labels = []

    for row in rows:
        color = "grey"
        row_pval = -np.log10(row["p-value"])
        row_change = np.log2(row["Fold Change"])

        if xminmax and (row_change < xminmax[0] or row_change > xminmax[1]):
            continue

        if sequence_labels:
            row_label = row["Sequence"].__str__()
        else:
            row_label = " / ".join(sorted(row["Proteins"].genes))
        old_row_label, old_re_row_label = None, None

        if (
            full_site_labels and
            len(list(row["Modifications"].skip_labels_iter())) > 0
        ):
            old_row_label = row_label
            old_re_row_label = rename.get(old_row_label, old_row_label)
            row_label = " / ".join(
                sorted(
                    "{} {}".format(
                        gene,
                        row["Modifications"].__str__(prot_index=index),
                    )
                    for index, gene in enumerate(row["Proteins"].genes)
                )
            )

        re_row_label = rename.get(row_label, row_label)
        edgecolor = edgecolors.get(
            re_row_label, edgecolors.get(
                row_label,
                edgecolors.get(old_row_label, None),
            )
        )

        if edgecolor is None:
            gene_colors = [
                edgecolors.get(gene, None)
                for gene in row["Proteins"].genes
            ]
            if len(set(gene_colors)) == 1:
                edgecolor = gene_colors[0]

        pvals.append(row_pval)
        changes.append(row_change)

        names = [
            row_label, re_row_label, old_row_label, old_re_row_label,
        ]

        if (
            any(
                i in show
                for i in names
            ) or ((
                row_pval > log_p and
                (row_change > upper_fold or row_change < lower_fold)
            ) if fold_and_p else (
                row_pval > log_p or
                (row_change > upper_fold or row_change < lower_fold)
            ))
        ):
            color = "blue"
            highlight_label = any(i in highlight for i in names)

            if (
                not any(i.strip() in hide for i in row_label.split("/")) and
                re_row_label not in hide
            ):
                labels.append((
                    row_pval, row_change,
                    re_row_label, edgecolor, highlight_label,
                ))

        colors.append(color)

    if compress_dups:
        labels = _remove_lesser_dups(labels, compress_sym=compress_sym)

    # Draw the figure
    fig, ax = plt.subplots(figsize=figsize)
    ax.scatter(changes, pvals, c=colors, alpha=alpha)
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)

    if xminmax:
        ax.set_xlim(xmin=xminmax[0], xmax=xminmax[1])
    else:
        ax.set_xlim(
            xmin=np.floor(min(changes + [0]) * 2) / 2,
            xmax=np.ceil(max(changes + [0]) * 2) / 2,
        )

    if yminmax:
        ax.set_ylim(bottom=yminmax[0], top=yminmax[1])
    else:
        ax.set_ylim(bottom=-0.1)

    ax.set_xticks(
        list(
            sorted(
                tick
                for tick in ax.get_xticks()
                if tick < lower_fold or tick > upper_fold
            ) + [lower_fold, upper_fold]
        )
    )
    ax.set_yticks(
        list(
            sorted(
                [
                    tick
                    for tick in ax.get_yticks()
                    if "{}".format(np.power(1/10, tick)).strip("0.")[:1] in
                    ["1", "5"] and
                    tick > log_p
                ] + [log_p]
            )
        )
    )
    ax.set_xticklabels(
        [
            "{:.3}".format(i)
            for i in np.exp2(ax.get_xticks())
        ],
        fontsize=20,
    )
    ax.set_yticklabels(
        [
            ("{:.3}" if i > 0.005 else "{:.0e}").format(i)
            for i in np.power(1/10, ax.get_yticks())
        ],
        fontsize=20,
    )

    if show_xlabel:
        ax.set_xlabel(
            "Fold Change {} (n={}) / {} (n={})".format(
                label_a,
                len(channels_a),
                label_b,
                len(channels_b),
            ),
            fontsize=20,
        )

    if show_ylabel:
        ax.set_ylabel(
            "p-value",
            fontsize=20,
        )

    if not np.isnan(log_p):
        ax.axhline(
            log_p,
            color="r", linestyle="dashed", linewidth=0.5,
        )

    if abs(fold - 1) > 0.01:
        ax.axvline(upper_fold, color="r", linestyle="dashed", linewidth=0.5)
        ax.axvline(lower_fold, color="r", linestyle="dashed", linewidth=0.5)

    # Position the labels
    texts = []
    txt_lim = 100 if full_site_labels else 20

    for y, x, txt, edgecolor, highlight_label in labels:
        text = ax.text(
            x, y,
            txt[:txt_lim] + ("..." if len(txt) > txt_lim else ""),
        )

        if highlight_label:
            text.set_fontsize(20)

        text.set_bbox(
            dict(
                facecolor=_get_color(txt, x, y),
                alpha=1,
                linewidth=0.5 if not edgecolor else 3,
                edgecolor=edgecolor or "black",
                boxstyle="round",
            )
        )

        texts.append(text)

    if adjust:
        adjust_text(
            x=[i[0] for i in labels],
            y=[i[1] for i in labels],
            texts=texts,
            ax=ax,
            lim=500,
            force_text=0.5,
            force_points=0.01,
            arrowprops=dict(arrowstyle="->", relpos=(0, 0), lw=1),
            only_move={
                "points": "y",
                "text": "xy",
            }
        )

    if title:
        if show_title:
            ax.set_title(
                title,
                fontsize=20,
            )

    if filename:
        fig.savefig(
            os.path.join(folder_name, filename),
            bbox_inches="tight",
            dpi=pyp.DEFAULT_DPI,
            transparent=True,
        )

    return fig, ax, folder_name, filename

def plot_volcano_filtered(data, f, **kwargs):
    data = data.copy()
    data.update_group_changes(
        group_a=kwargs.get("group_a", None),
        group_b=kwargs.get("group_b", None),
    )

    d = data.filter(f)

    changes = []
    pvals = []

    for _, row in data.psms.iterrows():
        row_pval = -np.log10(row["p-value"])
        row_change = np.log2(row["Fold Change"])

        if (
            np.isnan(row_pval) or
            np.isnan(row_change) or
            np.isinf(row_pval) or
            np.isinf(row_change)
        ):
            continue

        pvals.append(row_pval)
        changes.append(row_change)

    xminmax = kwargs.pop(
        "xminmax",
        (
            np.floor(min(changes + [0]) * 2) / 2,
            np.ceil(max(changes + [0]) * 2) / 2,
        ),
    )
    yminmax = kwargs.pop(
        "yminmax",
        (-0.1, np.ceil(max(pvals))),
    )

    f, ax, folder_name, filename = volcano_plot(
        d,
        xminmax=xminmax,
        yminmax=yminmax,
        **kwargs
    )

    changes, pvals = zip(*[
        (x, y)
        for x, y in zip(changes, pvals)
        if x > xminmax[0] and x < xminmax[1] and
        y > yminmax[0] and y < yminmax[1]
    ])

    ax.scatter(changes, pvals, c="lightblue", zorder=0, alpha=0.3)

    if filename:
        f.savefig(
            os.path.join(folder_name, filename),
            bbox_inches="tight",
            dpi=pyp.DEFAULT_DPI,
            transparent=True,
        )

    return f, ax
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

# copy to ~/.gimp-2.8/plug-ins/
# dependencies: GIMP 2.8, python-imaging-tk

from gimpfu import register, main, pdb, PF_BOOL, PF_STRING, PF_RADIO
from tempfile import mkstemp
from PIL import Image, ImageTk
from Tkinter import Tk, Canvas, Button, Checkbutton, IntVar, Frame, LEFT, NW
from pysstv import __main__ as pysstv_main
from pysstv.examples.pyaudio_sstv import PyAudioSSTV
from pysstv.sstv import SSTV
from itertools import repeat
from threading import Thread
from Queue import Queue, Empty
import os

MODULE_MAP = pysstv_main.build_module_map()

class AudioThread(Thread):
    def __init__(self, sstv, parent):
        Thread.__init__(self)
        self.pas = PyAudioSSTV(sstv)
        self.parent = parent

    def run(self):
        self.pas.execute()
        self.parent.audio_thread_ended()

    def stop(self):
        self.pas.sampler = []
        self.pas = None

class Sine1750(SSTV):
    encode_line = None

    def gen_freq_bits(self):
        return repeat((1750, 1000))

class Transmitter(object):
    def __init__(self, sstv, root, progress):
        def encode_line_hooked(line):
            progress.update_image(line)
            return self.original_encode_line(line)
        self.progress = progress
        self.sstv = sstv
        self.original_encode_line = sstv.encode_line
        sstv.encode_line = encode_line_hooked
        self.root = root
        self.tx_enabled = IntVar()
        self.audio_thread = None
        self.stopping = False

    def start_stop_tx(self):
        if self.tx_enabled.get():
            self.stopping = False
            self.audio_thread = AudioThread(self.sstv, self)
            self.audio_thread.start()
        else:
            self.stop()
            if self.progress is not None:
                self.progress.update_image()

    def stop(self):
        if self.audio_thread is not None:
            self.stopping = True
            self.audio_thread.stop()

    def audio_thread_ended(self):
        if not self.stopping:
            self.tx_enabled.set(0)

    def close(self):
        self.root.destroy()

class CanvasUpdater(Thread):
    def __init__(self, progress):
        Thread.__init__(self)
        self.progress = progress
        self.queue = Queue()
        self.should_run = True

    def stop(self):
        self.should_run = False

    def update_image(self, line=None):
        self.queue.put(line)

    def run(self):
        while self.should_run:
            try:
                self.progress.update_image(self.queue.get(timeout=0.5))
            except Empty:
                pass

class ProgressCanvas(Canvas):
    def __init__(self, master, image):
        self.height_ratio = 1
        width, height = image.size
        pixels = image.load()
        RED, GREEN, BLUE = range(3)
        self.colors = ['#{0:02x}{1:02x}{2:02x}'.format(
            255 - sum(pixels[x, y][RED]   for x in xrange(width)) / width,
            255 - sum(pixels[x, y][GREEN] for x in xrange(width)) / width,
            255 - sum(pixels[x, y][BLUE]  for x in xrange(width)) / width)
            for y in xrange(height)]
        if height / float(width) > 1.5:
            width *= 2
        elif width < 200:
            width *= 2
            height *= 2
            self.height_ratio = 2
        if (width, height) != image.size:
            image = image.resize((width, height))
        Canvas.__init__(self, master, width=width, height=height)
        self.tk_img = ImageTk.PhotoImage(image)
        self.update_image()

    def update_image(self, line=None):
        image = self.tk_img
        self.create_image(0, 0, anchor=NW, image=image)
        if line is not None:
            fill = self.colors[line]
            line *= self.height_ratio
            self.create_line(0, line, image.width(), line, fill=fill)

def transmit_current_image(image, drawable, mode, vox, fskid):
    sstv = MODULE_MAP[mode]
    handle, png_fn = mkstemp(suffix='.png', prefix='pysstv-gimp-')
    os.fdopen(handle).close()
    try:
        pdb.gimp_file_save(image, drawable, png_fn, png_fn)
        pil_img = Image.open(png_fn)
        sstv_size = sstv.WIDTH, sstv.HEIGHT
        if pil_img.size != sstv_size:
            pil_img = pil_img.resize(sstv_size, Image.ANTIALIAS)
        if 'grayscale' in sstv.__module__:
            pil_img = pil_img.convert('LA').convert('RGB')
        root = Tk()
        s = sstv(pil_img, 44100, 16)
        s.vox_enabled = vox
        if fskid:
            s.add_fskid_text(fskid)
        pc = ProgressCanvas(root, pil_img)
        pc.pack()
        cu = CanvasUpdater(pc)
        cu.start()
        tm = Transmitter(s, root, cu)
        tm1750 = Transmitter(Sine1750(None, 44100, 16), None, None)
        buttons = Frame(root)
        for text, tram in (('TX', tm), ('1750 Hz', tm1750)):
            Checkbutton(buttons, text=text, indicatoron=False, padx=5, pady=5,
                    variable=tram.tx_enabled, command=tram.start_stop_tx).pack(side=LEFT)
        Button(buttons, text="Close", command=tm.close).pack(side=LEFT)
        buttons.pack()
        root.mainloop()
        tm.stop()
        tm1750.stop()
        cu.stop()
    finally:
        os.remove(png_fn)

register(
        "pysstv_for_gimp",
        "PySSTV for GIMP",
        "Transmits the current image using PySSTV",
        "Andras Veres-Szentkiralyi",
        "Andras Veres-Szentkiralyi",
        "November 2013",
        "<Image>/PySSTV/Transmit...",
        "*",
        [
            (PF_RADIO, "mode", "SSTV mode", "MartinM1",
                tuple((n, n) for n in sorted(MODULE_MAP.iterkeys()))),
            (PF_BOOL, "vox", "Include VOX tones", True),
            (PF_STRING, "fskid", "FSK ID", ""),
        ],
        [],
        transmit_current_image
        )

main()
<commit_msg>avoid None dereference upon double AudioThread.stop()<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

# copy to ~/.gimp-2.8/plug-ins/
# dependencies: GIMP 2.8, python-imaging-tk

from gimpfu import register, main, pdb, PF_BOOL, PF_STRING, PF_RADIO
from tempfile import mkstemp
from PIL import Image, ImageTk
from Tkinter import Tk, Canvas, Button, Checkbutton, IntVar, Frame, LEFT, NW
from pysstv import __main__ as pysstv_main
from pysstv.examples.pyaudio_sstv import PyAudioSSTV
from pysstv.sstv import SSTV
from itertools import repeat
from threading import Thread
from Queue import Queue, Empty
import os

MODULE_MAP = pysstv_main.build_module_map()

class AudioThread(Thread):
    def __init__(self, sstv, parent):
        Thread.__init__(self)
        self.pas = PyAudioSSTV(sstv)
        self.parent = parent

    def run(self):
        self.pas.execute()
        self.parent.audio_thread_ended()

    def stop(self):
        if self.pas is not None:
            self.pas.sampler = []
            self.pas = None

class Sine1750(SSTV):
    encode_line = None

    def gen_freq_bits(self):
        return repeat((1750, 1000))

class Transmitter(object):
    def __init__(self, sstv, root, progress):
        def encode_line_hooked(line):
            progress.update_image(line)
            return self.original_encode_line(line)
        self.progress = progress
        self.sstv = sstv
        self.original_encode_line = sstv.encode_line
        sstv.encode_line = encode_line_hooked
        self.root = root
        self.tx_enabled = IntVar()
        self.audio_thread = None
        self.stopping = False

    def start_stop_tx(self):
        if self.tx_enabled.get():
            self.stopping = False
            self.audio_thread = AudioThread(self.sstv, self)
            self.audio_thread.start()
        else:
            self.stop()
            if self.progress is not None:
                self.progress.update_image()

    def stop(self):
        if self.audio_thread is not None:
            self.stopping = True
            self.audio_thread.stop()

    def audio_thread_ended(self):
        if not self.stopping:
            self.tx_enabled.set(0)

    def close(self):
        self.root.destroy()

class CanvasUpdater(Thread):
    def __init__(self, progress):
        Thread.__init__(self)
        self.progress = progress
        self.queue = Queue()
        self.should_run = True

    def stop(self):
        self.should_run = False

    def update_image(self, line=None):
        self.queue.put(line)

    def run(self):
        while self.should_run:
            try:
                self.progress.update_image(self.queue.get(timeout=0.5))
            except Empty:
                pass

class ProgressCanvas(Canvas):
    def __init__(self, master, image):
        self.height_ratio = 1
        width, height = image.size
        pixels = image.load()
        RED, GREEN, BLUE = range(3)
        self.colors = ['#{0:02x}{1:02x}{2:02x}'.format(
            255 - sum(pixels[x, y][RED]   for x in xrange(width)) / width,
            255 - sum(pixels[x, y][GREEN] for x in xrange(width)) / width,
            255 - sum(pixels[x, y][BLUE]  for x in xrange(width)) / width)
            for y in xrange(height)]
        if height / float(width) > 1.5:
            width *= 2
        elif width < 200:
            width *= 2
            height *= 2
            self.height_ratio = 2
        if (width, height) != image.size:
            image = image.resize((width, height))
        Canvas.__init__(self, master, width=width, height=height)
        self.tk_img = ImageTk.PhotoImage(image)
        self.update_image()

    def update_image(self, line=None):
        image = self.tk_img
        self.create_image(0, 0, anchor=NW, image=image)
        if line is not None:
            fill = self.colors[line]
            line *= self.height_ratio
            self.create_line(0, line, image.width(), line, fill=fill)

def transmit_current_image(image, drawable, mode, vox, fskid):
    sstv = MODULE_MAP[mode]
    handle, png_fn = mkstemp(suffix='.png', prefix='pysstv-gimp-')
    os.fdopen(handle).close()
    try:
        pdb.gimp_file_save(image, drawable, png_fn, png_fn)
        pil_img = Image.open(png_fn)
        sstv_size = sstv.WIDTH, sstv.HEIGHT
        if pil_img.size != sstv_size:
            pil_img = pil_img.resize(sstv_size, Image.ANTIALIAS)
        if 'grayscale' in sstv.__module__:
            pil_img = pil_img.convert('LA').convert('RGB')
        root = Tk()
        s = sstv(pil_img, 44100, 16)
        s.vox_enabled = vox
        if fskid:
            s.add_fskid_text(fskid)
        pc = ProgressCanvas(root, pil_img)
        pc.pack()
        cu = CanvasUpdater(pc)
        cu.start()
        tm = Transmitter(s, root, cu)
        tm1750 = Transmitter(Sine1750(None, 44100, 16), None, None)
        buttons = Frame(root)
        for text, tram in (('TX', tm), ('1750 Hz', tm1750)):
            Checkbutton(buttons, text=text, indicatoron=False, padx=5, pady=5,
                    variable=tram.tx_enabled, command=tram.start_stop_tx).pack(side=LEFT)
        Button(buttons, text="Close", command=tm.close).pack(side=LEFT)
        buttons.pack()
        root.mainloop()
        tm.stop()
        tm1750.stop()
        cu.stop()
    finally:
        os.remove(png_fn)

register(
        "pysstv_for_gimp",
        "PySSTV for GIMP",
        "Transmits the current image using PySSTV",
        "Andras Veres-Szentkiralyi",
        "Andras Veres-Szentkiralyi",
        "November 2013",
        "<Image>/PySSTV/Transmit...",
        "*",
        [
            (PF_RADIO, "mode", "SSTV mode", "MartinM1",
                tuple((n, n) for n in sorted(MODULE_MAP.iterkeys()))),
            (PF_BOOL, "vox", "Include VOX tones", True),
            (PF_STRING, "fskid", "FSK ID", ""),
        ],
        [],
        transmit_current_image
        )

main()
<|endoftext|>


<commit_before>"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

import enum
from typing import Any, List

from .writer import (
    BoldUnicodeTableWriter,
    BorderlessTableWriter,
    CssTableWriter,
    CsvTableWriter,
    ElasticsearchWriter,
    ExcelXlsTableWriter,
    ExcelXlsxTableWriter,
    HtmlTableWriter,
    JavaScriptTableWriter,
    JsonLinesTableWriter,
    JsonTableWriter,
    LatexMatrixWriter,
    LatexTableWriter,
    LtsvTableWriter,
    MarkdownTableWriter,
    MediaWikiTableWriter,
    NullTableWriter,
    NumpyTableWriter,
    PandasDataFrameWriter,
    PythonCodeTableWriter,
    RstCsvTableWriter,
    RstGridTableWriter,
    RstSimpleTableWriter,
    SpaceAlignedTableWriter,
    SqliteTableWriter,
    TomlTableWriter,
    TsvTableWriter,
    UnicodeTableWriter,
    YamlTableWriter,
)

class FormatAttr:
    """
    Bitmaps to represent table attributes.
    """

    NONE = 1 << 1

    #: Can create a file with the format.
    FILE = 1 << 2

    #: Table format that can represent as a text.
    TEXT = 1 << 3

    #: Table format that can represent as a binary file.
    BIN = 1 << 4

    #: Can create a source code (variables definition)
    #: one of the programming language.
    SOURCECODE = 1 << 5

    #: Can call API for external service.
    API = 1 << 6

    SECONDARY_EXT = 1 << 10
    SECONDARY_NAME = 1 << 11

@enum.unique
class TableFormat(enum.Enum):
    """
    Enum to represent table format attributes.
    """

    CSV = ([CsvTableWriter.FORMAT_NAME], CsvTableWriter, FormatAttr.FILE | FormatAttr.TEXT, ["csv"])
    CSS = (
        [CssTableWriter.FORMAT_NAME],
        CssTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["css"],
    )
    ELASTICSEARCH = (
        [ElasticsearchWriter.FORMAT_NAME],  # type: ignore
        ElasticsearchWriter,
        FormatAttr.API,
        [],
    )
    EXCEL_XLSX = (
        [ExcelXlsxTableWriter.FORMAT_NAME],
        ExcelXlsxTableWriter,
        FormatAttr.FILE | FormatAttr.BIN,
        ["xlsx"],
    )
    EXCEL_XLS = (
        [ExcelXlsTableWriter.FORMAT_NAME],
        ExcelXlsTableWriter,
        FormatAttr.FILE | FormatAttr.BIN | FormatAttr.SECONDARY_NAME,
        ["xls"],
    )
    HTML = (
        [HtmlTableWriter.FORMAT_NAME, "htm"],
        HtmlTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["html", "htm"],
    )
    JAVASCRIPT = (
        [JavaScriptTableWriter.FORMAT_NAME, "js"],
        JavaScriptTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SOURCECODE,
        ["js"],
    )
    JSON = (
        [JsonTableWriter.FORMAT_NAME],
        JsonTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["json"],
    )
    JSON_LINES = (
        [JsonLinesTableWriter.FORMAT_NAME, "jsonl", "ldjson", "ndjson"],
        JsonLinesTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["jsonl", "ldjson", "ndjson"],
    )
    LATEX_MATRIX = (
        [LatexMatrixWriter.FORMAT_NAME],
        LatexMatrixWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["tex"],
    )
    LATEX_TABLE = (
        [LatexTableWriter.FORMAT_NAME],
        LatexTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SECONDARY_EXT,
        ["tex"],
    )
    LTSV = (
        [LtsvTableWriter.FORMAT_NAME],
        LtsvTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["ltsv"],
    )
    MARKDOWN = (
        [MarkdownTableWriter.FORMAT_NAME, "md"],
        MarkdownTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["md"],
    )
    MEDIAWIKI = (
        [MediaWikiTableWriter.FORMAT_NAME],  # type: ignore
        MediaWikiTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        [],
    )
    NULL = (
        [NullTableWriter.FORMAT_NAME],  # type: ignore
        NullTableWriter,
        FormatAttr.NONE,
        [],
    )
    NUMPY = (
        [NumpyTableWriter.FORMAT_NAME],
        NumpyTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SOURCECODE | FormatAttr.SECONDARY_EXT,
        ["py"],
    )
    PANDAS = (
        [PandasDataFrameWriter.FORMAT_NAME],
        PandasDataFrameWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SOURCECODE | FormatAttr.SECONDARY_EXT,
        ["py"],
    )
    PYTHON = (
        [PythonCodeTableWriter.FORMAT_NAME, "py"],
        PythonCodeTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SOURCECODE,
        ["py"],
    )
    RST_CSV_TABLE = (
        [RstCsvTableWriter.FORMAT_NAME, "rst_csv"],
        RstCsvTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SECONDARY_EXT,
        ["rst"],
    )
    RST_GRID_TABLE = (
        [RstGridTableWriter.FORMAT_NAME, "rst_grid", "rst"],
        RstGridTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["rst"],
    )
    RST_SIMPLE_TABLE = (
        [RstSimpleTableWriter.FORMAT_NAME, "rst_simple"],
        RstSimpleTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SECONDARY_EXT,
        ["rst"],
    )
    SPACE_ALIGNED = (
        [SpaceAlignedTableWriter.FORMAT_NAME],  # type: ignore
        SpaceAlignedTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        [],
    )
    SQLITE = (
        [SqliteTableWriter.FORMAT_NAME],
        SqliteTableWriter,
        FormatAttr.FILE | FormatAttr.BIN,
        ["sqlite", "sqlite3"],
    )
    TOML = (
        [TomlTableWriter.FORMAT_NAME],
        TomlTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["toml"],
    )
    TSV = ([TsvTableWriter.FORMAT_NAME], TsvTableWriter, FormatAttr.FILE | FormatAttr.TEXT, ["tsv"])
    UNICODE = (
        [UnicodeTableWriter.FORMAT_NAME],  # type: ignore
        UnicodeTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        [],
    )
    YAML = (
        [YamlTableWriter.FORMAT_NAME],
        YamlTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["yml"],
    )
    BOLD_UNICODE = (
        [BoldUnicodeTableWriter.FORMAT_NAME],  # type: ignore
        BoldUnicodeTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        [],
    )
    BORDERLESS = (
        [BorderlessTableWriter.FORMAT_NAME],  # type: ignore
        BorderlessTableWriter,
        FormatAttr.TEXT,
        [],
    )

    @property
    def names(self) -> List[str]:
        """Names associated with the table format.

        Returns:
            List[str]: format names
        """

        return self.__names

    @property
    def writer_class(self) -> Any:
        """Table writer class associated with the table format.

        Returns:
            Type[AbstractTableWriter]:
        """

        return self.__writer_class

    @property
    def format_attribute(self) -> int:
        """Table attributes bitmap.

        Returns:
            :py:class:`pytablewriter.FormatAttr`:
        """

        return self.__format_attribute

    @property
    def file_extensions(self) -> List[str]:
        """File extensions associated with the table format.

        Returns:
            List[str]:
        """

        return self.__file_extensions

    def __init__(self, names, writer_class, format_attribute, file_extensions):
        self.__names = names
        self.__writer_class = writer_class
        self.__format_attribute = format_attribute
        self.__file_extensions = file_extensions

    @classmethod
    def find_all_attr(cls, format_attribute: int) -> List:
        """Searching table formats which have specific attributes.

        Args:
            format_attribute (FormatAttr):
                Table format attributes to look for.

        Returns:
            List[TableFormat]: Table formats that matched the attribute.
        """

        return [
            table_format
            for table_format in TableFormat
            if table_format.format_attribute & format_attribute
        ]

    @classmethod
    def from_name(cls, format_name: str):
        """Get a table format from a name.

        Args:
            format_name (str): Table format specifier.

        Returns:
            Optional[TableFormat]: A table format enum value corresponding to the ``format_name``.
        """

        format_name = format_name.lower().strip()

        for table_format in TableFormat:
            if format_name in table_format.names:
                return table_format

        return None
<commit_msg>Modify a docstring<commit_after>"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

import enum
from typing import Any, List

from .writer import (
    BoldUnicodeTableWriter,
    BorderlessTableWriter,
    CssTableWriter,
    CsvTableWriter,
    ElasticsearchWriter,
    ExcelXlsTableWriter,
    ExcelXlsxTableWriter,
    HtmlTableWriter,
    JavaScriptTableWriter,
    JsonLinesTableWriter,
    JsonTableWriter,
    LatexMatrixWriter,
    LatexTableWriter,
    LtsvTableWriter,
    MarkdownTableWriter,
    MediaWikiTableWriter,
    NullTableWriter,
    NumpyTableWriter,
    PandasDataFrameWriter,
    PythonCodeTableWriter,
    RstCsvTableWriter,
    RstGridTableWriter,
    RstSimpleTableWriter,
    SpaceAlignedTableWriter,
    SqliteTableWriter,
    TomlTableWriter,
    TsvTableWriter,
    UnicodeTableWriter,
    YamlTableWriter,
)

class FormatAttr:
    """
    Bitmaps to represent table attributes.
    """

    NONE = 1 << 1

    #: Can create a file with the format.
    FILE = 1 << 2

    #: Table format that can represent as a text.
    TEXT = 1 << 3

    #: Table format that can represent as a binary file.
    BIN = 1 << 4

    #: Can create a source code (variables definition)
    #: one of the programming language.
    SOURCECODE = 1 << 5

    #: Can call API for external service.
    API = 1 << 6

    SECONDARY_EXT = 1 << 10
    SECONDARY_NAME = 1 << 11

@enum.unique
class TableFormat(enum.Enum):
    """
    Enum to represent table format attributes.
    """

    CSV = ([CsvTableWriter.FORMAT_NAME], CsvTableWriter, FormatAttr.FILE | FormatAttr.TEXT, ["csv"])
    CSS = (
        [CssTableWriter.FORMAT_NAME],
        CssTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["css"],
    )
    ELASTICSEARCH = (
        [ElasticsearchWriter.FORMAT_NAME],  # type: ignore
        ElasticsearchWriter,
        FormatAttr.API,
        [],
    )
    EXCEL_XLSX = (
        [ExcelXlsxTableWriter.FORMAT_NAME],
        ExcelXlsxTableWriter,
        FormatAttr.FILE | FormatAttr.BIN,
        ["xlsx"],
    )
    EXCEL_XLS = (
        [ExcelXlsTableWriter.FORMAT_NAME],
        ExcelXlsTableWriter,
        FormatAttr.FILE | FormatAttr.BIN | FormatAttr.SECONDARY_NAME,
        ["xls"],
    )
    HTML = (
        [HtmlTableWriter.FORMAT_NAME, "htm"],
        HtmlTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["html", "htm"],
    )
    JAVASCRIPT = (
        [JavaScriptTableWriter.FORMAT_NAME, "js"],
        JavaScriptTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SOURCECODE,
        ["js"],
    )
    JSON = (
        [JsonTableWriter.FORMAT_NAME],
        JsonTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["json"],
    )
    JSON_LINES = (
        [JsonLinesTableWriter.FORMAT_NAME, "jsonl", "ldjson", "ndjson"],
        JsonLinesTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["jsonl", "ldjson", "ndjson"],
    )
    LATEX_MATRIX = (
        [LatexMatrixWriter.FORMAT_NAME],
        LatexMatrixWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["tex"],
    )
    LATEX_TABLE = (
        [LatexTableWriter.FORMAT_NAME],
        LatexTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SECONDARY_EXT,
        ["tex"],
    )
    LTSV = (
        [LtsvTableWriter.FORMAT_NAME],
        LtsvTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["ltsv"],
    )
    MARKDOWN = (
        [MarkdownTableWriter.FORMAT_NAME, "md"],
        MarkdownTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["md"],
    )
    MEDIAWIKI = (
        [MediaWikiTableWriter.FORMAT_NAME],  # type: ignore
        MediaWikiTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        [],
    )
    NULL = (
        [NullTableWriter.FORMAT_NAME],  # type: ignore
        NullTableWriter,
        FormatAttr.NONE,
        [],
    )
    NUMPY = (
        [NumpyTableWriter.FORMAT_NAME],
        NumpyTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SOURCECODE | FormatAttr.SECONDARY_EXT,
        ["py"],
    )
    PANDAS = (
        [PandasDataFrameWriter.FORMAT_NAME],
        PandasDataFrameWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SOURCECODE | FormatAttr.SECONDARY_EXT,
        ["py"],
    )
    PYTHON = (
        [PythonCodeTableWriter.FORMAT_NAME, "py"],
        PythonCodeTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SOURCECODE,
        ["py"],
    )
    RST_CSV_TABLE = (
        [RstCsvTableWriter.FORMAT_NAME, "rst_csv"],
        RstCsvTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SECONDARY_EXT,
        ["rst"],
    )
    RST_GRID_TABLE = (
        [RstGridTableWriter.FORMAT_NAME, "rst_grid", "rst"],
        RstGridTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["rst"],
    )
    RST_SIMPLE_TABLE = (
        [RstSimpleTableWriter.FORMAT_NAME, "rst_simple"],
        RstSimpleTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SECONDARY_EXT,
        ["rst"],
    )
    SPACE_ALIGNED = (
        [SpaceAlignedTableWriter.FORMAT_NAME],  # type: ignore
        SpaceAlignedTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        [],
    )
    SQLITE = (
        [SqliteTableWriter.FORMAT_NAME],
        SqliteTableWriter,
        FormatAttr.FILE | FormatAttr.BIN,
        ["sqlite", "sqlite3"],
    )
    TOML = (
        [TomlTableWriter.FORMAT_NAME],
        TomlTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["toml"],
    )
    TSV = ([TsvTableWriter.FORMAT_NAME], TsvTableWriter, FormatAttr.FILE | FormatAttr.TEXT, ["tsv"])
    UNICODE = (
        [UnicodeTableWriter.FORMAT_NAME],  # type: ignore
        UnicodeTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        [],
    )
    YAML = (
        [YamlTableWriter.FORMAT_NAME],
        YamlTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        ["yml"],
    )
    BOLD_UNICODE = (
        [BoldUnicodeTableWriter.FORMAT_NAME],  # type: ignore
        BoldUnicodeTableWriter,
        FormatAttr.FILE | FormatAttr.TEXT,
        [],
    )
    BORDERLESS = (
        [BorderlessTableWriter.FORMAT_NAME],  # type: ignore
        BorderlessTableWriter,
        FormatAttr.TEXT,
        [],
    )

    @property
    def names(self) -> List[str]:
        """Names associated with the table format.

        Returns:
            List[str]: format names
        """

        return self.__names

    @property
    def writer_class(self) -> Any:
        """Table writer class object associated with the table format.

        Returns:
            Type[AbstractTableWriter]:
        """

        return self.__writer_class

    @property
    def format_attribute(self) -> int:
        """Table attributes bitmap.

        Returns:
            :py:class:`pytablewriter.FormatAttr`:
        """

        return self.__format_attribute

    @property
    def file_extensions(self) -> List[str]:
        """File extensions associated with the table format.

        Returns:
            List[str]:
        """

        return self.__file_extensions

    def __init__(self, names, writer_class, format_attribute, file_extensions):
        self.__names = names
        self.__writer_class = writer_class
        self.__format_attribute = format_attribute
        self.__file_extensions = file_extensions

    @classmethod
    def find_all_attr(cls, format_attribute: int) -> List:
        """Searching table formats which have specific attributes.

        Args:
            format_attribute (FormatAttr):
                Table format attributes to look for.

        Returns:
            List[TableFormat]: Table formats that matched the attribute.
        """

        return [
            table_format
            for table_format in TableFormat
            if table_format.format_attribute & format_attribute
        ]

    @classmethod
    def from_name(cls, format_name: str):
        """Get a table format from a name.

        Args:
            format_name (str): Table format specifier.

        Returns:
            Optional[TableFormat]: A table format enum value corresponding to the ``format_name``.
        """

        format_name = format_name.lower().strip()

        for table_format in TableFormat:
            if format_name in table_format.names:
                return table_format

        return None
<|endoftext|>


<commit_before>import asyncio
from contextlib import redirect_stdout
import inspect
import io
import math
import sys
import textwrap
import traceback

import discord
from discord.errors import Forbidden
from discord.ext import commands

from utils import checks

class REPL:
    def __init__(self, bot):
        self.bot = bot
        self._last_result = None
        self.sessions = set()

    def cleanup_code(self, content):
        """Automatically removes code blocks from the code."""
        # remove ```py\n```
        if content.startswith('```') and content.endswith('```'):
            return '\n'.join(content.split('\n')[1:-1])

        # remove `foo`
        return content.strip('` \n').lstrip('<')

    def get_syntax_error(self, e):
        return f'```py\n{e.text}{"^":>{e.offset}}\n{type(e).__name__}: {e}```'

    @commands.command(hidden=True, name='eval')
    @checks.is_owner()
    async def eval_(self, ctx, *, body: str):
        env = {
            'bot': self.bot,
            'ctx': ctx,
            'channel': ctx.message.channel,
            'author': ctx.message.author,
            'guild': ctx.message.guild,
            'message': ctx.message,
            '_': self._last_result
        }

        env.update(globals())

        body = self.cleanup_code(body)
        stdout = io.StringIO()

        to_compile = f'async def func():\n{textwrap.indent(body, "  ")}'

        try:
            exec(to_compile, env)
        except SyntaxError as e:
            return await ctx.send(self.get_syntax_error(e))

        func = env['func']
        try:
            with redirect_stdout(stdout):
                ret = await func()
        except Exception as e:
            value = stdout.getvalue()
            await ctx.send(f'```py\n{value}{traceback.format_exc()}\n```')
        else:
            value = stdout.getvalue()

            if ret is None:
                if value:
                    await ctx.send(f'```py\n{value}\n```')
            else:
                self._last_result = ret
                await ctx.send(f'```py\n{value}{ret}\n```')

    @commands.command(hidden=True)
    @checks.is_owner()
    async def peval(self, ctx, *, body: str):
        body = self.cleanup_code(body)
        await ctx.invoke(self.eval_, body=f'print({body})')

    @commands.command(hidden=True)
    @checks.is_owner()
    async def repl(self, ctx):
        msg = ctx.message

        variables = {
            'ctx': ctx,
            'bot': self.bot,
            'message': msg,
            'guild': msg.guild,
            'channel': msg.channel,
            'author': msg.author,
            '_': None,
        }

        if msg.channel.id in self.sessions:
            await ctx.send('Already running a REPL session in this channel. '
                           'Exit it with `quit`.')
            return

        self.sessions.add(msg.channel.id)
        await ctx.send('Enter code to execute or evaluate. '
                       '`exit()` or `quit` to exit.')
        while True:
            response = await (
                 self.bot.wait_for('message',
                                   check=lambda m: m.content.startswith('<')
                                   and (m.author, m.channel) ==
                                   (msg.author, msg.channel)))

            cleaned = self.cleanup_code(response.content)

            if cleaned in ('quit', 'exit', 'exit()'):
                await ctx.send('Exiting.')
                self.sessions.remove(msg.channel.id)
                return

            executor = exec
            if cleaned.count('\n') == 0:
                # single statement, potentially 'eval'
                try:
                    code = compile(cleaned, '<repl session>', 'eval')
                except SyntaxError:
                    pass
                else:
                    executor = eval

            if executor is exec:
                try:
                    code = compile(cleaned, '<repl session>', 'exec')
                except SyntaxError as e:
                    await ctx.send(self.get_syntax_error(e))
                    continue

            variables['message'] = response

            fmt = None
            stdout = io.StringIO()

            try:
                with redirect_stdout(stdout):
                    result = executor(code, variables)
                    if inspect.isawaitable(result):
                        result = await result
            except Exception as e:
                value = stdout.getvalue()
                fmt = f'```py\n{value}{traceback.format_exc()}\n```'
            else:
                value = stdout.getvalue()
                if result is not None:
                    fmt = f'```py\n{value}{result}\n```'
                    variables['_'] = result
                elif value:
                    fmt = f'```py\n{value}\n```'

            try:
                if fmt is not None:
                    if len(fmt) > 2000:
                        await ctx.send('Content too big to be printed.')
                    else:
                        await ctx.send(fmt)
            except discord.Forbidden:
                pass
            except discord.HTTPException as e:
                await ctx.send(f'Unexpected error: `{e}`')

def setup(bot):
    bot.add_cog(REPL(bot))
<commit_msg>Add objgraph for leak finding purposes<commit_after>import asyncio
from contextlib import redirect_stdout
import inspect
import io
import math
import sys
import textwrap
import traceback

import discord
from discord.errors import Forbidden
from discord.ext import commands
import objgraph

from utils import checks

class REPL:
    def __init__(self, bot):
        self.bot = bot
        self._last_result = None
        self.sessions = set()

    def cleanup_code(self, content):
        """Automatically removes code blocks from the code."""
        # remove ```py\n```
        if content.startswith('```') and content.endswith('```'):
            return '\n'.join(content.split('\n')[1:-1])

        # remove `foo`
        return content.strip('` \n').lstrip('<')

    def get_syntax_error(self, e):
        return f'```py\n{e.text}{"^":>{e.offset}}\n{type(e).__name__}: {e}```'

    @commands.command(hidden=True, name='eval')
    @checks.is_owner()
    async def eval_(self, ctx, *, body: str):
        env = {
            'bot': self.bot,
            'ctx': ctx,
            'channel': ctx.message.channel,
            'author': ctx.message.author,
            'guild': ctx.message.guild,
            'message': ctx.message,
            '_': self._last_result
        }

        env.update(globals())

        body = self.cleanup_code(body)
        stdout = io.StringIO()

        to_compile = f'async def func():\n{textwrap.indent(body, "  ")}'

        try:
            exec(to_compile, env)
        except SyntaxError as e:
            return await ctx.send(self.get_syntax_error(e))

        func = env['func']
        try:
            with redirect_stdout(stdout):
                ret = await func()
        except Exception as e:
            value = stdout.getvalue()
            await ctx.send(f'```py\n{value}{traceback.format_exc()}\n```')
        else:
            value = stdout.getvalue()

            if ret is None:
                if value:
                    await ctx.send(f'```py\n{value}\n```')
            else:
                self._last_result = ret
                await ctx.send(f'```py\n{value}{ret}\n```')

    @commands.command(hidden=True)
    @checks.is_owner()
    async def peval(self, ctx, *, body: str):
        body = self.cleanup_code(body)
        await ctx.invoke(self.eval_, body=f'print({body})')

    @commands.command(hidden=True)
    @checks.is_owner()
    async def repl(self, ctx):
        msg = ctx.message

        variables = {
            'ctx': ctx,
            'bot': self.bot,
            'message': msg,
            'guild': msg.guild,
            'channel': msg.channel,
            'author': msg.author,
            '_': None,
        }

        if msg.channel.id in self.sessions:
            await ctx.send('Already running a REPL session in this channel. '
                           'Exit it with `quit`.')
            return

        self.sessions.add(msg.channel.id)
        await ctx.send('Enter code to execute or evaluate. '
                       '`exit()` or `quit` to exit.')
        while True:
            response = await (
                 self.bot.wait_for('message',
                                   check=lambda m: m.content.startswith('<')
                                   and (m.author, m.channel) ==
                                   (msg.author, msg.channel)))

            cleaned = self.cleanup_code(response.content)

            if cleaned in ('quit', 'exit', 'exit()'):
                await ctx.send('Exiting.')
                self.sessions.remove(msg.channel.id)
                return

            executor = exec
            if cleaned.count('\n') == 0:
                # single statement, potentially 'eval'
                try:
                    code = compile(cleaned, '<repl session>', 'eval')
                except SyntaxError:
                    pass
                else:
                    executor = eval

            if executor is exec:
                try:
                    code = compile(cleaned, '<repl session>', 'exec')
                except SyntaxError as e:
                    await ctx.send(self.get_syntax_error(e))
                    continue

            variables['message'] = response

            fmt = None
            stdout = io.StringIO()

            try:
                with redirect_stdout(stdout):
                    result = executor(code, variables)
                    if inspect.isawaitable(result):
                        result = await result
            except Exception as e:
                value = stdout.getvalue()
                fmt = f'```py\n{value}{traceback.format_exc()}\n```'
            else:
                value = stdout.getvalue()
                if result is not None:
                    fmt = f'```py\n{value}{result}\n```'
                    variables['_'] = result
                elif value:
                    fmt = f'```py\n{value}\n```'

            try:
                if fmt is not None:
                    if len(fmt) > 2000:
                        await ctx.send('Content too big to be printed.')
                    else:
                        await ctx.send(fmt)
            except discord.Forbidden:
                pass
            except discord.HTTPException as e:
                await ctx.send(f'Unexpected error: `{e}`')

def setup(bot):
    bot.add_cog(REPL(bot))
<|endoftext|>


<commit_before># Copyright 2011, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Constants indicating WebSocket protocol version.
VERSION_HYBI07 = 7
VERSION_HYBI00 = 0
VERSION_HIXIE75 = -1

# Port numbers
DEFAULT_WEB_SOCKET_PORT = 80
DEFAULT_WEB_SOCKET_SECURE_PORT = 443

# Schemes
WEB_SOCKET_SCHEME = 'ws'
WEB_SOCKET_SECURE_SCHEME = 'wss'

# Frame opcodes defined in the spec.
OPCODE_CONTINUATION = 0x0
OPCODE_TEXT = 0x1
OPCODE_BINARY = 0x2
OPCODE_CLOSE = 0x8
OPCODE_PING = 0x9
OPCODE_PONG = 0xa

# UUIDs used by HyBi 07 opening handshake and frame masking.
WEBSOCKET_ACCEPT_UUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'

# Opening handshake header names and expected values.
UPGRADE_HEADER = 'Upgrade'
WEBSOCKET_UPGRADE_TYPE = 'websocket'
WEBSOCKET_UPGRADE_TYPE_HIXIE75 = 'WebSocket'
CONNECTION_HEADER = 'Connection'
UPGRADE_CONNECTION_TYPE = 'Upgrade'
HOST_HEADER = 'Host'
SEC_WEBSOCKET_ORIGIN_HEADER = 'Sec-WebSocket-Origin'
SEC_WEBSOCKET_KEY_HEADER = 'Sec-WebSocket-Key'
SEC_WEBSOCKET_ACCEPT_HEADER = 'Sec-WebSocket-Accept'
SEC_WEBSOCKET_VERSION_HEADER = 'Sec-WebSocket-Version'
SEC_WEBSOCKET_PROTOCOL_HEADER = 'Sec-WebSocket-Protocol'
SEC_WEBSOCKET_EXTENSIONS_HEADER = 'Sec-WebSocket-Extensions'

# Status codes
STATUS_NORMAL = 1000
STATUS_GOING_AWAY = 1001
STATUS_PROTOCOL_ERROR = 1002
STATUS_UNSUPPORTED = 1003
STATUS_TOO_LARGE = 1004
<commit_msg>Add a vi mode line to common.py.<commit_after># Copyright 2011, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Constants indicating WebSocket protocol version.
VERSION_HYBI07 = 7
VERSION_HYBI00 = 0
VERSION_HIXIE75 = -1

# Port numbers
DEFAULT_WEB_SOCKET_PORT = 80
DEFAULT_WEB_SOCKET_SECURE_PORT = 443

# Schemes
WEB_SOCKET_SCHEME = 'ws'
WEB_SOCKET_SECURE_SCHEME = 'wss'

# Frame opcodes defined in the spec.
OPCODE_CONTINUATION = 0x0
OPCODE_TEXT = 0x1
OPCODE_BINARY = 0x2
OPCODE_CLOSE = 0x8
OPCODE_PING = 0x9
OPCODE_PONG = 0xa

# UUIDs used by HyBi 07 opening handshake and frame masking.
WEBSOCKET_ACCEPT_UUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'

# Opening handshake header names and expected values.
UPGRADE_HEADER = 'Upgrade'
WEBSOCKET_UPGRADE_TYPE = 'websocket'
WEBSOCKET_UPGRADE_TYPE_HIXIE75 = 'WebSocket'
CONNECTION_HEADER = 'Connection'
UPGRADE_CONNECTION_TYPE = 'Upgrade'
HOST_HEADER = 'Host'
SEC_WEBSOCKET_ORIGIN_HEADER = 'Sec-WebSocket-Origin'
SEC_WEBSOCKET_KEY_HEADER = 'Sec-WebSocket-Key'
SEC_WEBSOCKET_ACCEPT_HEADER = 'Sec-WebSocket-Accept'
SEC_WEBSOCKET_VERSION_HEADER = 'Sec-WebSocket-Version'
SEC_WEBSOCKET_PROTOCOL_HEADER = 'Sec-WebSocket-Protocol'
SEC_WEBSOCKET_EXTENSIONS_HEADER = 'Sec-WebSocket-Extensions'

# Status codes
STATUS_NORMAL = 1000
STATUS_GOING_AWAY = 1001
STATUS_PROTOCOL_ERROR = 1002
STATUS_UNSUPPORTED = 1003
STATUS_TOO_LARGE = 1004

# vi:sts=4 sw=4 et
<|endoftext|>


<commit_before># A Neural Network Architecture Combining Gated Recurrent Unit (GRU) and
# Support Vector Machine (SVM) for Intrusion Detection in Network Traffic Data
# Copyright (C) 2017  Abien Fred Agarap
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
# ==============================================================================

"""Implementation of the GRU+SVM model"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

__version__ = '0.3.6'
__author__ = 'Abien Fred Agarap'

import numpy as np
import os
import sys
import tensorflow as tf
import time

class GruSvm:

    def __init__(self, alpha, batch_size, cell_size, dropout_rate, num_classes, sequence_length, svm_c):
        self.alpha = alpha
        self.batch_size = batch_size
        self.cell_size = cell_size
        self.dropout_rate = dropout_rate
        self.num_classes = num_classes
        self.sequence_length = sequence_length
        self.svm_c = svm_c

        def __graph__():
            """Build the inference graph"""
            with tf.name_scope('input'):
                # [BATCH_SIZE, SEQUENCE_LENGTH]
                x_input = tf.placeholder(dtype=tf.uint8, shape=[None, self.sequence_length], name='x_input')

                # [BATCH_SIZE, SEQUENCE_LENGTH, 10]
                x_onehot = tf.one_hot(indices=x_input, depth=10, on_value=1.0, off_value=0.0, name='x_onehot')

                # [BATCH_SIZE]
                y_input = tf.placeholder(dtype=tf.uint8, shape=[None], name='y_input')

                # [BATCH_SIZE, N_CLASSES]
                y_onehot = tf.one_hot(indices=y_input, depth=self.num_classes, on_value=1.0, off_value=-1.0,
                                      name='y_onehot')

            state = tf.placeholder(dtype=tf.float32, shape=[None, self.cell_size], name='initial_state')

            p_keep = tf.placeholder(dtype=tf.float32, name='p_keep')
            learning_rate = tf.placeholder(dtype=tf.float32, name='learning_rate')

            cell = tf.contrib.rnn.GRUCell(self.cell_size)
            drop_cell = tf.contrib.rnn.DropoutWrapper(cell, input_keep_prob=p_keep)

            # outputs: [BATCH_SIZE, SEQUENCE_LENGTH, CELL_SIZE]
            # states: [BATCH_SIZE, CELL_SIZE]
            outputs, states = tf.nn.dynamic_rnn(drop_cell, x_onehot, initial_state=state, dtype=tf.float32)

            states = tf.identity(states, name='H')

            with tf.name_scope('final_training_ops'):
                with tf.name_scope('weights'):
                    weight = tf.get_variable('weights',
                                             initializer=tf.random_normal([self.cell_size, self.num_classes],
                                                                          stddev=0.01))
                    self.variable_summaries(weight)
                with tf.name_scope('biases'):
                    bias = tf.get_variable('biases', initializer=tf.constant(0.1, shape=[self.num_classes]))
                    self.variable_summaries(bias)
                hf = tf.transpose(outputs, [1, 0, 2])
                last = tf.gather(hf, int(hf.get_shape()[0]) - 1)
                with tf.name_scope('Wx_plus_b'):
                    output = tf.matmul(last, weight) + bias
                    tf.summary.histogram('pre-activations', output)

            # L2-SVM
            with tf.name_scope('svm'):
                regularization_loss = 0.5 * tf.reduce_sum(tf.square(weight))
                hinge_loss = tf.reduce_sum(
                    tf.square(tf.maximum(tf.zeros([self.batch_size, self.num_classes]), 1 - y_onehot * output)))
                with tf.name_scope('loss'):
                    loss = regularization_loss + self.svm_c * hinge_loss
            tf.summary.scalar('loss', loss)

            optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)

            with tf.name_scope('accuracy'):
                predicted_class = tf.sign(output)
                predicted_class = tf.identity(predicted_class, name='prediction')
                with tf.name_scope('correct_prediction'):
                    correct = tf.equal(tf.argmax(predicted_class, 1), tf.argmax(y_onehot, 1))
                with tf.name_scope('accuracy'):
                    accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
            tf.summary.scalar('accuracy', accuracy)

            # merge all the summaries collected from the TF graph
            merged = tf.summary.merge_all()

            # set class properties
            self.x_input = x_input
            self.y_input = y_input
            self.y_onehot = y_onehot
            self.p_keep = p_keep
            self.loss = loss
            self.optimizer = optimizer
            self.state = state
            self.states = states
            self.learning_rate = learning_rate
            self.predicted_class = predicted_class
            self.accuracy = accuracy
            self.merged = merged

        sys.stdout.write('\n<log> Building Graph...')
        __graph__()
        sys.stdout.write('</log>\n')

    def train(self, checkpoint_path, log_path, model_name, epochs, train_data, train_size, validation_data,
              validation_size, result_path):
        """Train the model"""

        if not os.path.exists(path=checkpoint_path):
            os.mkdir(path=checkpoint_path)

        saver = tf.train.Saver(max_to_keep=1000)

        # initialize H (current_state) with values of zeros
        current_state = np.zeros([self.batch_size, self.cell_size])

        # variables initializer
        init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())

        # get the time tuple
        timestamp = str(time.asctime())

        train_writer = tf.summary.FileWriter(logdir=log_path + timestamp + '-training', graph=tf.get_default_graph())
        validation_writer = tf.summary.FileWriter(logdir=log_path + timestamp + '-validation',
                                                  graph=tf.get_default_graph())

        with tf.Session() as sess:
            sess.run(init_op)

            checkpoint = tf.train.get_checkpoint_state(checkpoint_path)

            if checkpoint and checkpoint.model_checkpoint_path:
                saver = tf.train.import_meta_graph(checkpoint.model_checkpoint_path + '.meta')
                saver.restore(sess, tf.train.latest_checkpoint(checkpoint_path))

            try:
                for step in range(epochs * train_size // self.batch_size):

                    # set the value for slicing
                    # e.g. step = 0, batch_size = 256, train_size = 1898240
                    # (0 * 256) % 1898240 = 0
                    # [offset:(offset + batch_size)] = [0:256]
                    offset = (step * self.batch_size) % train_size
                    train_example_batch = train_data[0][offset:(offset + self.batch_size)]
                    train_label_batch = train_data[1][offset:(offset + self.batch_size)]

                    # dictionary for key-value pair input for training
                    feed_dict = {self.x_input: train_example_batch, self.y_input: train_label_batch,
                                 self.state: current_state,
                                 self.learning_rate: self.alpha, self.p_keep: self.dropout_rate}

                    train_summary, _, predictions, next_state = sess.run([self.merged, self.optimizer,
                                                                          self.predicted_class, self.states],
                                                                         feed_dict=feed_dict)

                    # Display training loss and accuracy every 100 steps and at step 0
                    if step % 100 == 0:
                        # get train loss and accuracy
                        train_loss, train_accuracy = sess.run([self.loss, self.accuracy], feed_dict=feed_dict)

                        # display train loss and accuracy
                        print('step [{}] train -- loss : {}, accuracy : {}'.format(step, train_loss, train_accuracy))

                        # write the train summary
                        train_writer.add_summary(train_summary, step)

                        # save the model at current step
                        saver.save(sess, checkpoint_path + model_name, global_step=step)

                    current_state = next_state

                    predictions = sess.run(tf.argmax(predictions, 1))
                    predictions[predictions == 0] = -1

                    self.save_labels(predictions=predictions, actual=train_label_batch, result_path=result_path,
                                     step=step)

                    # # concatenate the predicted labels and actual labels
                    # prediction_and_actual = np.concatenate((predictions, train_label_batch), axis=1)
                    #
                    # # save every prediction_and_actual numpy array to a CSV file for analysis purposes
                    # np.savetxt(os.path.join(result_path, 'gru_svm-{}-training.csv'.format(step)),
                    #            X=prediction_and_actual, fmt='%.3f', delimiter=',', newline='\n')

                for step in range(epochs * validation_size // self.batch_size):

                    offset = (step * self.batch_size) % validation_size
                    test_example_batch = validation_data[0][offset:(offset + self.batch_size)]
                    test_label_batch = validation_data[1][offset:(offset + self.batch_size)]

                    # dictionary for key-value pair input for validation
                    feed_dict = {self.x_input: test_example_batch, self.y_input: test_label_batch,
                                 self.state: np.zeros([self.batch_size, self.cell_size]), self.p_keep: 1.0}

                    validation_summary, predictions, actual, validation_loss, validation_accuracy = \
                        sess.run([self.merged, self.predicted_class, self.y_onehot, self.loss, self.accuracy],
                                 feed_dict=feed_dict)

                    # Display validation loss and accuracy every 100 steps
                    if step % 100 == 0 and step > 0:
                        # get validation loss and accuracy
                        # validation_summary, validation_loss, validation_accuracy = sess.run([self.merged, self.loss,
                        #                                                                      self.accuracy],
                        #                                                                     feed_dict=feed_dict)

                        validation_writer.add_summary(validation_summary, step)

                        # display validation loss and accuracy
                        print('step [{}] validation -- loss : {}, accuracy : {}'.format(step, validation_loss,
                                                                                validation_accuracy))
                    predictions[predictions == 0] = -1

                    self.save_labels(predictions=predictions, actual=actual, result_path=result_path, step=step)

            except KeyboardInterrupt:
                print('Training interrupted at {}'.format(step))
            finally:
                print('EOF -- training done at step {}'.format(step))

            saver.save(sess, checkpoint_path + model_name, global_step=step)

    @staticmethod
    def variable_summaries(var):
        with tf.name_scope('summaries'):
            mean = tf.reduce_mean(var)
            tf.summary.scalar('mean', mean)
            with tf.name_scope('stddev'):
                stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
            tf.summary.scalar('stddev', stddev)
            tf.summary.scalar('max', tf.reduce_max(var))
            tf.summary.scalar('min', tf.reduce_min(var))
            tf.summary.histogram('histogram', var)

    @staticmethod
    def save_labels(predictions, actual, result_path, step):
        """Saves the actual and predicted labels to a CSV file"""

        # concatenate the predicted labels and actual labels
        prediction_and_actual = np.concatenate((predictions, actual), axis=1)

        # save every prediction_and_actual numpy array to a CSV file for analysis purposes
        np.savetxt(os.path.join(result_path, 'gru_svm-{}-training.csv'.format(step)),
                   X=prediction_and_actual, fmt='%.3f', delimiter=',', newline='\n')
<commit_msg>Transforms the actual label to {-1, +1}; Creates a NumPy array to contain the predicted and actual labels instead of concatenating them (since they're both 1-dimensional array already, not suitable for np.concatenate((arr1, arr2), axis=1)<commit_after># A Neural Network Architecture Combining Gated Recurrent Unit (GRU) and
# Support Vector Machine (SVM) for Intrusion Detection in Network Traffic Data
# Copyright (C) 2017  Abien Fred Agarap
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
# ==============================================================================

"""Implementation of the GRU+SVM model"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

__version__ = '0.3.7'
__author__ = 'Abien Fred Agarap'

import numpy as np
import os
import sys
import tensorflow as tf
import time

class GruSvm:

    def __init__(self, alpha, batch_size, cell_size, dropout_rate, num_classes, sequence_length, svm_c):
        self.alpha = alpha
        self.batch_size = batch_size
        self.cell_size = cell_size
        self.dropout_rate = dropout_rate
        self.num_classes = num_classes
        self.sequence_length = sequence_length
        self.svm_c = svm_c

        def __graph__():
            """Build the inference graph"""
            with tf.name_scope('input'):
                # [BATCH_SIZE, SEQUENCE_LENGTH]
                x_input = tf.placeholder(dtype=tf.uint8, shape=[None, self.sequence_length], name='x_input')

                # [BATCH_SIZE, SEQUENCE_LENGTH, 10]
                x_onehot = tf.one_hot(indices=x_input, depth=10, on_value=1.0, off_value=0.0, name='x_onehot')

                # [BATCH_SIZE]
                y_input = tf.placeholder(dtype=tf.uint8, shape=[None], name='y_input')

                # [BATCH_SIZE, N_CLASSES]
                y_onehot = tf.one_hot(indices=y_input, depth=self.num_classes, on_value=1.0, off_value=-1.0,
                                      name='y_onehot')

            state = tf.placeholder(dtype=tf.float32, shape=[None, self.cell_size], name='initial_state')

            p_keep = tf.placeholder(dtype=tf.float32, name='p_keep')
            learning_rate = tf.placeholder(dtype=tf.float32, name='learning_rate')

            cell = tf.contrib.rnn.GRUCell(self.cell_size)
            drop_cell = tf.contrib.rnn.DropoutWrapper(cell, input_keep_prob=p_keep)

            # outputs: [BATCH_SIZE, SEQUENCE_LENGTH, CELL_SIZE]
            # states: [BATCH_SIZE, CELL_SIZE]
            outputs, states = tf.nn.dynamic_rnn(drop_cell, x_onehot, initial_state=state, dtype=tf.float32)

            states = tf.identity(states, name='H')

            with tf.name_scope('final_training_ops'):
                with tf.name_scope('weights'):
                    weight = tf.get_variable('weights',
                                             initializer=tf.random_normal([self.cell_size, self.num_classes],
                                                                          stddev=0.01))
                    self.variable_summaries(weight)
                with tf.name_scope('biases'):
                    bias = tf.get_variable('biases', initializer=tf.constant(0.1, shape=[self.num_classes]))
                    self.variable_summaries(bias)
                hf = tf.transpose(outputs, [1, 0, 2])
                last = tf.gather(hf, int(hf.get_shape()[0]) - 1)
                with tf.name_scope('Wx_plus_b'):
                    output = tf.matmul(last, weight) + bias
                    tf.summary.histogram('pre-activations', output)

            # L2-SVM
            with tf.name_scope('svm'):
                regularization_loss = 0.5 * tf.reduce_sum(tf.square(weight))
                hinge_loss = tf.reduce_sum(
                    tf.square(tf.maximum(tf.zeros([self.batch_size, self.num_classes]), 1 - y_onehot * output)))
                with tf.name_scope('loss'):
                    loss = regularization_loss + self.svm_c * hinge_loss
            tf.summary.scalar('loss', loss)

            optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)

            with tf.name_scope('accuracy'):
                predicted_class = tf.sign(output)
                predicted_class = tf.identity(predicted_class, name='prediction')
                with tf.name_scope('correct_prediction'):
                    correct = tf.equal(tf.argmax(predicted_class, 1), tf.argmax(y_onehot, 1))
                with tf.name_scope('accuracy'):
                    accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
            tf.summary.scalar('accuracy', accuracy)

            # merge all the summaries collected from the TF graph
            merged = tf.summary.merge_all()

            # set class properties
            self.x_input = x_input
            self.y_input = y_input
            self.y_onehot = y_onehot
            self.p_keep = p_keep
            self.loss = loss
            self.optimizer = optimizer
            self.state = state
            self.states = states
            self.learning_rate = learning_rate
            self.predicted_class = predicted_class
            self.accuracy = accuracy
            self.merged = merged

        sys.stdout.write('\n<log> Building Graph...')
        __graph__()
        sys.stdout.write('</log>\n')

    def train(self, checkpoint_path, log_path, model_name, epochs, train_data, train_size, validation_data,
              validation_size, result_path):
        """Train the model"""

        if not os.path.exists(path=checkpoint_path):
            os.mkdir(path=checkpoint_path)

        saver = tf.train.Saver(max_to_keep=1000)

        # initialize H (current_state) with values of zeros
        current_state = np.zeros([self.batch_size, self.cell_size])

        # variables initializer
        init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())

        # get the time tuple
        timestamp = str(time.asctime())

        train_writer = tf.summary.FileWriter(logdir=log_path + timestamp + '-training', graph=tf.get_default_graph())
        validation_writer = tf.summary.FileWriter(logdir=log_path + timestamp + '-validation',
                                                  graph=tf.get_default_graph())

        with tf.Session() as sess:
            sess.run(init_op)

            checkpoint = tf.train.get_checkpoint_state(checkpoint_path)

            if checkpoint and checkpoint.model_checkpoint_path:
                saver = tf.train.import_meta_graph(checkpoint.model_checkpoint_path + '.meta')
                saver.restore(sess, tf.train.latest_checkpoint(checkpoint_path))

            try:
                for step in range(epochs * train_size // self.batch_size):

                    # set the value for slicing
                    # e.g. step = 0, batch_size = 256, train_size = 1898240
                    # (0 * 256) % 1898240 = 0
                    # [offset:(offset + batch_size)] = [0:256]
                    offset = (step * self.batch_size) % train_size
                    train_example_batch = train_data[0][offset:(offset + self.batch_size)]
                    train_label_batch = train_data[1][offset:(offset + self.batch_size)]

                    # dictionary for key-value pair input for training
                    feed_dict = {self.x_input: train_example_batch, self.y_input: train_label_batch,
                                 self.state: current_state,
                                 self.learning_rate: self.alpha, self.p_keep: self.dropout_rate}

                    train_summary, _, predictions, next_state = sess.run([self.merged, self.optimizer,
                                                                          self.predicted_class, self.states],
                                                                         feed_dict=feed_dict)

                    # Display training loss and accuracy every 100 steps and at step 0
                    if step % 100 == 0:
                        # get train loss and accuracy
                        train_loss, train_accuracy = sess.run([self.loss, self.accuracy], feed_dict=feed_dict)

                        # display train loss and accuracy
                        print('step [{}] train -- loss : {}, accuracy : {}'.format(step, train_loss, train_accuracy))

                        # write the train summary
                        train_writer.add_summary(train_summary, step)

                        # save the model at current step
                        saver.save(sess, checkpoint_path + model_name, global_step=step)

                    current_state = next_state

                    predictions = sess.run(tf.argmax(predictions, 1))
                    predictions[predictions == 0] = -1
                    train_label_batch[train_label_batch == 0] = -1

                    self.save_labels(predictions=predictions, actual=train_label_batch, result_path=result_path,
                                     step=step)

                    # # concatenate the predicted labels and actual labels
                    # prediction_and_actual = np.concatenate((predictions, train_label_batch), axis=1)
                    #
                    # # save every prediction_and_actual numpy array to a CSV file for analysis purposes
                    # np.savetxt(os.path.join(result_path, 'gru_svm-{}-training.csv'.format(step)),
                    #            X=prediction_and_actual, fmt='%.3f', delimiter=',', newline='\n')

                for step in range(epochs * validation_size // self.batch_size):

                    offset = (step * self.batch_size) % validation_size
                    test_example_batch = validation_data[0][offset:(offset + self.batch_size)]
                    test_label_batch = validation_data[1][offset:(offset + self.batch_size)]

                    # dictionary for key-value pair input for validation
                    feed_dict = {self.x_input: test_example_batch, self.y_input: test_label_batch,
                                 self.state: np.zeros([self.batch_size, self.cell_size]), self.p_keep: 1.0}

                    validation_summary, predictions, validation_loss, validation_accuracy = \
                        sess.run([self.merged, self.predicted_class, self.loss, self.accuracy], feed_dict=feed_dict)

                    # Display validation loss and accuracy every 100 steps
                    if step % 100 == 0 and step > 0:
                        # get validation loss and accuracy
                        # validation_summary, validation_loss, validation_accuracy = sess.run([self.merged, self.loss,
                        #                                                                      self.accuracy],
                        #                                                                     feed_dict=feed_dict)

                        validation_writer.add_summary(validation_summary, step)

                        # display validation loss and accuracy
                        print('step [{}] validation -- loss : {}, accuracy : {}'.format(step, validation_loss,
                                                                                validation_accuracy))
                    predictions = sess.run(tf.argmax(predictions, 1))
                    predictions[predictions == 0] = -1
                    test_label_batch[test_label_batch == 0] = -1

                    self.save_labels(predictions=predictions, actual=test_label_batch, result_path=result_path,
                                     step=step)

            except KeyboardInterrupt:
                print('Training interrupted at {}'.format(step))
            finally:
                print('EOF -- training done at step {}'.format(step))

            saver.save(sess, checkpoint_path + model_name, global_step=step)

    @staticmethod
    def variable_summaries(var):
        with tf.name_scope('summaries'):
            mean = tf.reduce_mean(var)
            tf.summary.scalar('mean', mean)
            with tf.name_scope('stddev'):
                stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
            tf.summary.scalar('stddev', stddev)
            tf.summary.scalar('max', tf.reduce_max(var))
            tf.summary.scalar('min', tf.reduce_min(var))
            tf.summary.histogram('histogram', var)

    @staticmethod
    def save_labels(predictions, actual, result_path, step):
        """Saves the actual and predicted labels to a CSV file"""

        # concatenate the predicted labels and actual labels
        prediction_and_actual = np.array([predictions, actual], dtype=np.int8)

        # save every prediction_and_actual numpy array to a CSV file for analysis purposes
        np.savetxt(os.path.join(result_path, 'gru_svm-{}-training.csv'.format(step)),
                   X=prediction_and_actual, fmt='%.3f', delimiter=',', newline='\n')
<|endoftext|>