<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
from __future__ import division

import dataproperty
import ipaddress
import six
from subprocrunner import SubprocessRunner
import thutils

from .parser import TcFilterParser
from .parser import TcQdiscParser
from ._common import verify_network_interface
from ._error import TcCommandExecutionError

def _validate_within_min_max(param_name, value, min_value, max_value):
    if value is None:
        return

    if value > max_value:
        raise ValueError(
            "{:s} is too high: expected<={:f}[%], value={:f}[%]".format(
                param_name, max_value, value))

    if value < min_value:
        raise ValueError(
            "{:s} is too low: expected>={:f}[%], value={:f}[%]".format(
                param_name, min_value, value))

class TrafficDirection(object):
    OUTGOING = "outgoing"
    INCOMING = "incoming"
    LIST = [OUTGOING, INCOMING]

class TrafficControl(object):
    __OUT_DEVICE_QDISC_MINOR_ID = 1
    __IN_DEVICE_QDISC_MINOR_ID = 3

    __MIN_PACKET_LOSS_RATE = 0  # [%]
    __MAX_PACKET_LOSS_RATE = 99  # [%]

    __MIN_LATENCY_MS = 0  # [millisecond]
    __MAX_LATENCY_MS = 10000  # [millisecond]

    __MIN_CORRUPTION_RATE = 0  # [%]
    __MAX_CORRUPTION_RATE = 99  # [%]

    __MIN_BUFFER_BYTE = 1600

    __MIN_PORT = 0
    __MAX_PORT = 65535

    @property
    def ifb_device(self):
        return "ifb{:d}".format(self.__get_device_qdisc_major_id())

    def __init__(self, device):
        self.__device = device

        self.direction = None
        self.bandwidth_rate = None  # bandwidth string [G/M/K bps]
        self.latency_ms = None  # [milliseconds]
        self.latency_distro_ms = None  # [milliseconds]
        self.packet_loss_rate = None  # [%]
        self.curruption_rate = None  # [%]
        self.network = None
        self.port = None

    def validate(self):
        verify_network_interface(self.__device)
        self.__validate_bandwidth_rate()
        self.__validate_network_delay()
        self.__validate_packet_loss_rate()
        self.__validate_curruption_rate()
        self.network = self.__validate_network()
        self.__validate_port()

    def __get_device_qdisc_major_id(self):
        import hashlib

        base_device_hash = hashlib.md5(six.b(self.__device)).hexdigest()[:3]
        device_hash_prefix = "1"

        return int(device_hash_prefix + base_device_hash, 16)

    def set_tc(self):
        self.__setup_ifb()

        qdisc_major_id = self.__get_device_qdisc_major_id()
        self.__make_qdisc(qdisc_major_id)
        self.__set_pre_network_filter(qdisc_major_id)
        self.__set_netem(qdisc_major_id)
        self.__set_network_filter(qdisc_major_id)
        self.__set_rate(qdisc_major_id)

    def delete_tc(self):
        return_code_list = []
        command_list = [
            "tc qdisc del dev {:s} root".format(self.__device),
            "tc qdisc del dev {:s} ingress".format(self.__device),
            "tc qdisc del dev {:s} root".format(self.ifb_device),
            "ip link set dev {:s} down".format(self.ifb_device),
            "ip link delete {:s} type ifb".format(self.ifb_device),
        ]

        for command in command_list:
            return_code_list.append(SubprocessRunner(command).run() != 0)

        return -1 if all(return_code_list) else 0

    def get_tc_parameter(self):
        return {
            self.__device: {
                "outgoing": self.__get_filter(self.__device),
                "incoming": self.__get_filter(
                    self.__get_ifb_from_device(self.__device)),
            },
        }

    def __setup_ifb(self):
        if self.direction != TrafficDirection.INCOMING:
            return

        if dataproperty.is_empty_string(self.ifb_device):
            return

        return_code = 0

        command = "modprobe ifb"
        return_code |= SubprocessRunner(command).run()

        command = "ip link add {:s} type ifb".format(self.ifb_device)
        return_code |= SubprocessRunner(command).run()

        command = "ip link set dev {:s} up".format(self.ifb_device)
        return_code |= SubprocessRunner(command).run()

        command = "tc qdisc add dev {:s} ingress".format(self.__device)
        return_code |= SubprocessRunner(command).run()

        command_list = [
            "tc filter add",
            "dev " + self.__device,
            "parent ffff: protocol ip u32 match u32 0 0",
            "flowid {:x}:".format(self.__get_device_qdisc_major_id()),
            "action mirred egress redirect",
            "dev " + self.ifb_device,
        ]
        return_code |= SubprocessRunner(" ".join(command_list)).run()

        return return_code

    def __validate_bandwidth_rate(self):
        if dataproperty.is_empty_string(self.bandwidth_rate):
            return

        rate = thutils.common.humanreadable_to_byte(self.bandwidth_rate)
        if rate <= 0:
            raise ValueError("rate must be greater than zero")

    def __validate_network_delay(self):
        _validate_within_min_max(
            thutils.common.get_var_name(self.latency_ms, locals()),
            self.latency_ms,
            self.__MIN_LATENCY_MS, self.__MAX_LATENCY_MS)

        _validate_within_min_max(
            thutils.common.get_var_name(self.latency_distro_ms, locals()),
            self.latency_distro_ms,
            self.__MIN_LATENCY_MS, self.__MAX_LATENCY_MS)

    def __validate_packet_loss_rate(self):
        _validate_within_min_max(
            thutils.common.get_var_name(self.packet_loss_rate, locals()),
            self.packet_loss_rate,
            self.__MIN_PACKET_LOSS_RATE, self.__MAX_PACKET_LOSS_RATE)

    def __validate_curruption_rate(self):
        _validate_within_min_max(
            thutils.common.get_var_name(self.curruption_rate, locals()),
            self.curruption_rate,
            self.__MIN_CORRUPTION_RATE, self.__MAX_CORRUPTION_RATE)

    def __validate_network(self):
        if dataproperty.is_empty_string(self.network):
            return ""

        try:
            ipaddress.IPv4Address(six.u(self.network))
            return self.network + "/32"
        except ipaddress.AddressValueError:
            pass

        ipaddress.IPv4Network(six.u(self.network))
        return self.network

        raise ValueError("unrecognizable network: " + self.network)

    def __validate_port(self):
        _validate_within_min_max(
            thutils.common.get_var_name(self.port, locals()),
            self.port, self.__MIN_PORT, self.__MAX_PORT)

    def __make_qdisc(self, qdisc_major_id):
        command_list = [
            "tc qdisc add",
            "dev " + self.__get_tc_device(),
            "root",
            "handle {:x}:".format(qdisc_major_id),
            "prio",
        ]

        return SubprocessRunner(" ".join(command_list)).run()

    def __get_tc_device(self):
        if self.direction == TrafficDirection.OUTGOING:
            return self.__device

        if self.direction == TrafficDirection.INCOMING:
            return self.ifb_device

        raise ValueError("unknown direction: " + self.direction)

    def __get_ifb_from_device(self, device):
        filter_parser = TcFilterParser()
        command = "tc filter show dev {:s} root".format(device)
        filter_runner = SubprocessRunner(command)
        filter_runner.run()

        return filter_parser.parse_incoming_device(filter_runner.stdout)

    def __get_network_direction_str(self):
        if self.direction == TrafficDirection.OUTGOING:
            return "dst"

        if self.direction == TrafficDirection.INCOMING:
            return "src"

        raise ValueError("unknown direction: " + self.direction)

    def __get_netem_qdisc_major_id(self, base_qdisc_major_id):
        base_offset = 10

        if self.direction == TrafficDirection.OUTGOING:
            direction_offset = 0
        elif self.direction == TrafficDirection.INCOMING:
            direction_offset = 1

        return base_qdisc_major_id + base_offset + direction_offset

    def __get_qdisc_minor_id(self):
        if self.direction == TrafficDirection.OUTGOING:
            return self.__OUT_DEVICE_QDISC_MINOR_ID

        if self.direction == TrafficDirection.INCOMING:
            return self.__IN_DEVICE_QDISC_MINOR_ID

        raise ValueError("unknown direction: " + self.direction)

    def __get_filter(self, device):
        if dataproperty.is_empty_string(device):
            return {}

        qdisc_parser = TcQdiscParser()
        filter_parser = TcFilterParser()

        # parse qdisc ---
        command = "tc qdisc show dev {:s}".format(device)
        qdisk_show_runner = SubprocessRunner(command)
        if qdisk_show_runner.returncode != 0:
            raise TcCommandExecutionError(qdisk_show_runner.stderr)
        qdisc_param = qdisc_parser.parse(qdisk_show_runner.stdout)

        # parse filter ---
        command = "tc filter show dev {:s}".format(device)
        filter_show_runner = SubprocessRunner(command)
        filter_show_runner.run()

        filter_table = {}

        for filter_param in filter_parser.parse_filter(filter_show_runner.stdout):
            key_item_list = []

            if dataproperty.is_not_empty_string(filter_param.get("network")):
                key_item_list.append("network=" + filter_param.get("network"))

            if dataproperty.is_integer(filter_param.get("port")):
                key_item_list.append(
                    "port={:d}".format(filter_param.get("port")))

            filter_key = ", ".join(key_item_list)
            filter_table[filter_key] = {}
            if filter_param.get("flowid") == qdisc_param.get("parent"):
                work_qdisc_param = dict(qdisc_param)
                del work_qdisc_param["parent"]
                filter_table[filter_key] = work_qdisc_param

        return filter_table

    def __set_pre_network_filter(self, qdisc_major_id):
        if all([
            dataproperty.is_empty_string(self.network),
            not dataproperty.is_integer(self.port),
        ]):
            flowid = "{:x}:{:d}".format(
                qdisc_major_id, self.__get_qdisc_minor_id())
        else:
            flowid = "{:x}:2".format(qdisc_major_id)

        command_list = [
            "tc filter add",
            "dev " + self.__get_tc_device(),
            "protocol ip",
            "parent {:x}:".format(qdisc_major_id),
            "prio 2 u32 match ip {:s} 0.0.0.0/0".format(
                self.__get_network_direction_str()),
            "flowid " + flowid
        ]

        return SubprocessRunner(" ".join(command_list)).run()

    def __set_netem(self, qdisc_major_id):
        command_list = [
            "tc qdisc add",
            "dev " + self.__get_tc_device(),
            "parent {:x}:{:d}".format(
                qdisc_major_id, self.__get_qdisc_minor_id()),
            "handle {:x}:".format(
                self.__get_netem_qdisc_major_id(qdisc_major_id)),
            "netem",
        ]
        if self.packet_loss_rate > 0:
            command_list.append("loss {:f}%".format(self.packet_loss_rate))
        if self.latency_ms > 0:
            command_list.append("delay {:f}ms".format(self.latency_ms))

            if self.latency_distro_ms > 0:
                command_list.append("{:f}ms distribution normal".format(
                    self.latency_distro_ms))

        if self.corruption_rate > 0:
            command_list.append("corrupt {:f}%".format(self.corruption_rate))

        return SubprocessRunner(" ".join(command_list)).run()

    def __set_network_filter(self, qdisc_major_id):
        if all([
            dataproperty.is_empty_string(self.network),
            self.port is None,
        ]):
            return 0

        command_list = [
            "tc filter add",
            "dev " + self.__get_tc_device(),
            "protocol ip",
            "parent {:x}:".format(qdisc_major_id),
            "prio 1 u32",
            "flowid {:x}:{:d}".format(
                qdisc_major_id, self.__get_qdisc_minor_id()),
        ]
        if dataproperty.is_not_empty_string(self.network):
            command_list.append("match ip {:s} {:s}".format(
                self.__get_network_direction_str(), self.network))
        if self.port is not None:
            command_list.append("match ip dport {:d} 0xffff".format(self.port))

        return SubprocessRunner(" ".join(command_list)).run()

    def __set_rate(self, qdisc_major_id):
        if dataproperty.is_empty_string(self.bandwidth_rate):
            return 0

        rate_kbps = thutils.common.humanreadable_to_byte(
            self.bandwidth_rate, kilo_size=1000) / 1000.0
        if rate_kbps <= 0:
            raise ValueError("rate must be greater than zero")

        command_list = [
            "tc qdisc add",
            "dev " + self.__get_tc_device(),
            "parent {:x}:{:d}".format(
                self.__get_netem_qdisc_major_id(qdisc_major_id),
                self.__get_qdisc_minor_id()),
            "handle 20:",
            "tbf",
            "rate {:f}kbit".format(rate_kbps),
            "buffer {:d}".format(
                max(rate_kbps, self.__MIN_BUFFER_BYTE)),  # [byte]
            "limit 10000",
        ]

        return SubprocessRunner(" ".join(command_list)).run()
<commit_msg>Fix tcshow<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
from __future__ import division

import dataproperty
import ipaddress
import six
from subprocrunner import SubprocessRunner
import thutils

from .parser import TcFilterParser
from .parser import TcQdiscParser
from ._common import verify_network_interface
from ._error import TcCommandExecutionError

def _validate_within_min_max(param_name, value, min_value, max_value):
    if value is None:
        return

    if value > max_value:
        raise ValueError(
            "{:s} is too high: expected<={:f}[%], value={:f}[%]".format(
                param_name, max_value, value))

    if value < min_value:
        raise ValueError(
            "{:s} is too low: expected>={:f}[%], value={:f}[%]".format(
                param_name, min_value, value))

class TrafficDirection(object):
    OUTGOING = "outgoing"
    INCOMING = "incoming"
    LIST = [OUTGOING, INCOMING]

class TrafficControl(object):
    __OUT_DEVICE_QDISC_MINOR_ID = 1
    __IN_DEVICE_QDISC_MINOR_ID = 3

    __MIN_PACKET_LOSS_RATE = 0  # [%]
    __MAX_PACKET_LOSS_RATE = 99  # [%]

    __MIN_LATENCY_MS = 0  # [millisecond]
    __MAX_LATENCY_MS = 10000  # [millisecond]

    __MIN_CORRUPTION_RATE = 0  # [%]
    __MAX_CORRUPTION_RATE = 99  # [%]

    __MIN_BUFFER_BYTE = 1600

    __MIN_PORT = 0
    __MAX_PORT = 65535

    @property
    def ifb_device(self):
        return "ifb{:d}".format(self.__get_device_qdisc_major_id())

    def __init__(self, device):
        self.__device = device

        self.direction = None
        self.bandwidth_rate = None  # bandwidth string [G/M/K bps]
        self.latency_ms = None  # [milliseconds]
        self.latency_distro_ms = None  # [milliseconds]
        self.packet_loss_rate = None  # [%]
        self.curruption_rate = None  # [%]
        self.network = None
        self.port = None

    def validate(self):
        verify_network_interface(self.__device)
        self.__validate_bandwidth_rate()
        self.__validate_network_delay()
        self.__validate_packet_loss_rate()
        self.__validate_curruption_rate()
        self.network = self.__validate_network()
        self.__validate_port()

    def __get_device_qdisc_major_id(self):
        import hashlib

        base_device_hash = hashlib.md5(six.b(self.__device)).hexdigest()[:3]
        device_hash_prefix = "1"

        return int(device_hash_prefix + base_device_hash, 16)

    def set_tc(self):
        self.__setup_ifb()

        qdisc_major_id = self.__get_device_qdisc_major_id()
        self.__make_qdisc(qdisc_major_id)
        self.__set_pre_network_filter(qdisc_major_id)
        self.__set_netem(qdisc_major_id)
        self.__set_network_filter(qdisc_major_id)
        self.__set_rate(qdisc_major_id)

    def delete_tc(self):
        return_code_list = []
        command_list = [
            "tc qdisc del dev {:s} root".format(self.__device),
            "tc qdisc del dev {:s} ingress".format(self.__device),
            "tc qdisc del dev {:s} root".format(self.ifb_device),
            "ip link set dev {:s} down".format(self.ifb_device),
            "ip link delete {:s} type ifb".format(self.ifb_device),
        ]

        for command in command_list:
            return_code_list.append(SubprocessRunner(command).run() != 0)

        return -1 if all(return_code_list) else 0

    def get_tc_parameter(self):
        return {
            self.__device: {
                "outgoing": self.__get_filter(self.__device),
                "incoming": self.__get_filter(
                    self.__get_ifb_from_device(self.__device)),
            },
        }

    def __setup_ifb(self):
        if self.direction != TrafficDirection.INCOMING:
            return

        if dataproperty.is_empty_string(self.ifb_device):
            return

        return_code = 0

        command = "modprobe ifb"
        return_code |= SubprocessRunner(command).run()

        command = "ip link add {:s} type ifb".format(self.ifb_device)
        return_code |= SubprocessRunner(command).run()

        command = "ip link set dev {:s} up".format(self.ifb_device)
        return_code |= SubprocessRunner(command).run()

        command = "tc qdisc add dev {:s} ingress".format(self.__device)
        return_code |= SubprocessRunner(command).run()

        command_list = [
            "tc filter add",
            "dev " + self.__device,
            "parent ffff: protocol ip u32 match u32 0 0",
            "flowid {:x}:".format(self.__get_device_qdisc_major_id()),
            "action mirred egress redirect",
            "dev " + self.ifb_device,
        ]
        return_code |= SubprocessRunner(" ".join(command_list)).run()

        return return_code

    def __validate_bandwidth_rate(self):
        if dataproperty.is_empty_string(self.bandwidth_rate):
            return

        rate = thutils.common.humanreadable_to_byte(self.bandwidth_rate)
        if rate <= 0:
            raise ValueError("rate must be greater than zero")

    def __validate_network_delay(self):
        _validate_within_min_max(
            thutils.common.get_var_name(self.latency_ms, locals()),
            self.latency_ms,
            self.__MIN_LATENCY_MS, self.__MAX_LATENCY_MS)

        _validate_within_min_max(
            thutils.common.get_var_name(self.latency_distro_ms, locals()),
            self.latency_distro_ms,
            self.__MIN_LATENCY_MS, self.__MAX_LATENCY_MS)

    def __validate_packet_loss_rate(self):
        _validate_within_min_max(
            thutils.common.get_var_name(self.packet_loss_rate, locals()),
            self.packet_loss_rate,
            self.__MIN_PACKET_LOSS_RATE, self.__MAX_PACKET_LOSS_RATE)

    def __validate_curruption_rate(self):
        _validate_within_min_max(
            thutils.common.get_var_name(self.curruption_rate, locals()),
            self.curruption_rate,
            self.__MIN_CORRUPTION_RATE, self.__MAX_CORRUPTION_RATE)

    def __validate_network(self):
        if dataproperty.is_empty_string(self.network):
            return ""

        try:
            ipaddress.IPv4Address(six.u(self.network))
            return self.network + "/32"
        except ipaddress.AddressValueError:
            pass

        ipaddress.IPv4Network(six.u(self.network))
        return self.network

        raise ValueError("unrecognizable network: " + self.network)

    def __validate_port(self):
        _validate_within_min_max(
            thutils.common.get_var_name(self.port, locals()),
            self.port, self.__MIN_PORT, self.__MAX_PORT)

    def __make_qdisc(self, qdisc_major_id):
        command_list = [
            "tc qdisc add",
            "dev " + self.__get_tc_device(),
            "root",
            "handle {:x}:".format(qdisc_major_id),
            "prio",
        ]

        return SubprocessRunner(" ".join(command_list)).run()

    def __get_tc_device(self):
        if self.direction == TrafficDirection.OUTGOING:
            return self.__device

        if self.direction == TrafficDirection.INCOMING:
            return self.ifb_device

        raise ValueError("unknown direction: " + self.direction)

    def __get_ifb_from_device(self, device):
        filter_parser = TcFilterParser()
        command = "tc filter show dev {:s} root".format(device)
        filter_runner = SubprocessRunner(command)
        filter_runner.run()

        return filter_parser.parse_incoming_device(filter_runner.stdout)

    def __get_network_direction_str(self):
        if self.direction == TrafficDirection.OUTGOING:
            return "dst"

        if self.direction == TrafficDirection.INCOMING:
            return "src"

        raise ValueError("unknown direction: " + self.direction)

    def __get_netem_qdisc_major_id(self, base_qdisc_major_id):
        base_offset = 10

        if self.direction == TrafficDirection.OUTGOING:
            direction_offset = 0
        elif self.direction == TrafficDirection.INCOMING:
            direction_offset = 1

        return base_qdisc_major_id + base_offset + direction_offset

    def __get_qdisc_minor_id(self):
        if self.direction == TrafficDirection.OUTGOING:
            return self.__OUT_DEVICE_QDISC_MINOR_ID

        if self.direction == TrafficDirection.INCOMING:
            return self.__IN_DEVICE_QDISC_MINOR_ID

        raise ValueError("unknown direction: " + self.direction)

    def __get_filter(self, device):
        if dataproperty.is_empty_string(device):
            return {}

        qdisc_parser = TcQdiscParser()
        filter_parser = TcFilterParser()

        # parse qdisc ---
        command = "tc qdisc show dev {:s}".format(device)
        qdisk_show_runner = SubprocessRunner(command)
        if qdisk_show_runner.returncode != 0:
            raise TcCommandExecutionError(qdisk_show_runner.stderr)
        qdisk_show_runner.run()
        qdisc_param = qdisc_parser.parse(qdisk_show_runner.stdout)

        # parse filter ---
        command = "tc filter show dev {:s}".format(device)
        filter_show_runner = SubprocessRunner(command)
        filter_show_runner.run()

        filter_table = {}

        for filter_param in filter_parser.parse_filter(filter_show_runner.stdout):
            key_item_list = []

            if dataproperty.is_not_empty_string(filter_param.get("network")):
                key_item_list.append("network=" + filter_param.get("network"))

            if dataproperty.is_integer(filter_param.get("port")):
                key_item_list.append(
                    "port={:d}".format(filter_param.get("port")))

            filter_key = ", ".join(key_item_list)
            filter_table[filter_key] = {}
            if filter_param.get("flowid") == qdisc_param.get("parent"):
                work_qdisc_param = dict(qdisc_param)
                del work_qdisc_param["parent"]
                filter_table[filter_key] = work_qdisc_param

        return filter_table

    def __set_pre_network_filter(self, qdisc_major_id):
        if all([
            dataproperty.is_empty_string(self.network),
            not dataproperty.is_integer(self.port),
        ]):
            flowid = "{:x}:{:d}".format(
                qdisc_major_id, self.__get_qdisc_minor_id())
        else:
            flowid = "{:x}:2".format(qdisc_major_id)

        command_list = [
            "tc filter add",
            "dev " + self.__get_tc_device(),
            "protocol ip",
            "parent {:x}:".format(qdisc_major_id),
            "prio 2 u32 match ip {:s} 0.0.0.0/0".format(
                self.__get_network_direction_str()),
            "flowid " + flowid
        ]

        return SubprocessRunner(" ".join(command_list)).run()

    def __set_netem(self, qdisc_major_id):
        command_list = [
            "tc qdisc add",
            "dev " + self.__get_tc_device(),
            "parent {:x}:{:d}".format(
                qdisc_major_id, self.__get_qdisc_minor_id()),
            "handle {:x}:".format(
                self.__get_netem_qdisc_major_id(qdisc_major_id)),
            "netem",
        ]
        if self.packet_loss_rate > 0:
            command_list.append("loss {:f}%".format(self.packet_loss_rate))
        if self.latency_ms > 0:
            command_list.append("delay {:f}ms".format(self.latency_ms))

            if self.latency_distro_ms > 0:
                command_list.append("{:f}ms distribution normal".format(
                    self.latency_distro_ms))

        if self.corruption_rate > 0:
            command_list.append("corrupt {:f}%".format(self.corruption_rate))

        return SubprocessRunner(" ".join(command_list)).run()

    def __set_network_filter(self, qdisc_major_id):
        if all([
            dataproperty.is_empty_string(self.network),
            self.port is None,
        ]):
            return 0

        command_list = [
            "tc filter add",
            "dev " + self.__get_tc_device(),
            "protocol ip",
            "parent {:x}:".format(qdisc_major_id),
            "prio 1 u32",
            "flowid {:x}:{:d}".format(
                qdisc_major_id, self.__get_qdisc_minor_id()),
        ]
        if dataproperty.is_not_empty_string(self.network):
            command_list.append("match ip {:s} {:s}".format(
                self.__get_network_direction_str(), self.network))
        if self.port is not None:
            command_list.append("match ip dport {:d} 0xffff".format(self.port))

        return SubprocessRunner(" ".join(command_list)).run()

    def __set_rate(self, qdisc_major_id):
        if dataproperty.is_empty_string(self.bandwidth_rate):
            return 0

        rate_kbps = thutils.common.humanreadable_to_byte(
            self.bandwidth_rate, kilo_size=1000) / 1000.0
        if rate_kbps <= 0:
            raise ValueError("rate must be greater than zero")

        command_list = [
            "tc qdisc add",
            "dev " + self.__get_tc_device(),
            "parent {:x}:{:d}".format(
                self.__get_netem_qdisc_major_id(qdisc_major_id),
                self.__get_qdisc_minor_id()),
            "handle 20:",
            "tbf",
            "rate {:f}kbit".format(rate_kbps),
            "buffer {:d}".format(
                max(rate_kbps, self.__MIN_BUFFER_BYTE)),  # [byte]
            "limit 10000",
        ]

        return SubprocessRunner(" ".join(command_list)).run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""TcEx Framework Module for working with DataStore in the ThreatConnect Platform."""
from json import JSONDecodeError

class DataStore:
    """TcEx DataStore Class.

    **domain**

    This can be either “system”, “organization”, or “local”. Any string not defined as these
    3 will return a “Resource Not Found” error response.

    **dataType**

    This is a free form index type name as allowed by ElasticSearch. The ThreatConnect API will
    use this resource verbatim for ElasticSearch calls.

    **searchCommand**

    The URI string after the typeName is assumed to be ElasticSearch relevant information and is
    passed directly to ElasticSearch.

    """

    def __init__(self, tcex, domain, data_type, mapping=None):
        """Initialize class properties.

        Args:
            tcex ([type]): [description]
            domain (str): A value of “system”, “organization”, or “local”.
            data_type (str): A free form type name for the data.
            mapping (dict, optional): Defaults to None. Elasticsearch mappings data.

        Raises:
            RuntimeError: [description]
        """
        self.tcex = tcex
        self.domain = domain
        self.data_type = data_type
        self.mapping = mapping or {'dynamic': False}

        # properties
        self._create_index()  # create the initial index.
        self._update_mappings()  # update mappings

    def _create_index(self):
        """Create index if it doesn't exist."""
        if not self.index_exists:
            rid = 'temp-index-create'
            index_settings = {}
            headers = {'Content-Type': 'application/json', 'DB-Method': 'POST'}
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
            r = self.tcex.session.post(url, json=index_settings, headers=headers)

            if not r.ok:
                error = r.text or r.reason
                self.tcex.handle_error(800, [r.status_code, error])
            self.tcex.log.debug(
                f'creating index. status_code: {r.status_code}, response: "{r.text}".'
            )
            # delete temporary record
            self.delete(rid, False)

    def _token_available(self):
        """Raise an error if token is not available.

        This check needs to be made after args are parsed to ensure token module has initialized.
        """
        # This module requires a token.
        if self.tcex.token.token is None:  # pragma: no cover
            raise RuntimeError(
                'The DataModel TcEx Module requires a Token to interact with the '
                'ThreatConnect platform.'
            )

    def _update_mappings(self):
        """Update the mappings for the current index."""
        headers = {'Content-Type': 'application/json', 'DB-Method': 'PUT'}
        url = f'/v2/exchange/db/{self.domain}/{self.data_type}/_mappings'
        r = self.tcex.session.post(url, json=self.mapping, headers=headers)
        self.tcex.log.debug(f'update mapping. status_code: {r.status_code}, response: "{r.text}".')

    @property
    def index_exists(self):
        """Check to see if index exists."""
        headers = {'Content-Type': 'application/json', 'DB-Method': 'GET'}
        url = f'/v2/exchange/db/{self.domain}/{self.data_type}/_search'
        r = self.tcex.session.post(url, headers=headers)
        if not r.ok:
            self.tcex.log.warning(f'The provided index was not found ({r.text}).')
            return False
        return True

    def add(self, rid, data, raise_on_error=True):
        """Write data to the DataStore. Alias for post() method.

        Args:
            rid (str): The record identifier.
            data (dict): The record data.
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        return self.post(rid, data, raise_on_error)

    def delete(self, rid, raise_on_error=True):
        """Delete a record from the index using provide Id.

        Args:
            rid (str): The record identifier.
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        self._token_available()  # ensure a token is available

        response_data = None
        headers = {'Content-Type': 'application/json', 'DB-Method': 'DELETE'}
        url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
        r = self.tcex.session.post(url, headers=headers)
        self.tcex.log.debug(f'datastore delete status code: {r.status_code}')
        if r.ok and 'application/json' in r.headers.get('content-type', ''):
            response_data = r.json()
        else:
            error = r.text or r.reason
            self.tcex.handle_error(805, ['delete', r.status_code, error], raise_on_error)
        return response_data

    def get(self, rid=None, data=None, raise_on_error=True):
        """Get data from the DataStore.

        Args:
            rid (str, Optional): Defaults to None. The record identifier.
            data (dict, Optional): Defaults to None. A search query
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        self._token_available()  # ensure a token is available

        response_data = None
        headers = {'Content-Type': 'application/json', 'DB-Method': 'GET'}
        if rid is None:
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/'
        else:
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
        r = self.tcex.session.post(url, json=data, headers=headers)
        self.tcex.log.debug(f'datastore get status code: {r.status_code}')
        if 'application/json' in r.headers.get('content-type', ''):
            # as long as the content is JSON set the value
            try:
                response_data = r.json()
            except JSONDecodeError:
                self.tcex.log.warning(
                    f'DataStore API returned a non-JSON response, even though content-type was \
                    application/json: {r.content}'
                )
        if not r.ok:
            error = r.text or r.reason
            self.tcex.handle_error(805, ['get', r.status_code, error], raise_on_error)
        return response_data

    def post(self, rid, data, raise_on_error=True):
        """Write data to the DataStore.

        Args:
            rid (str): The record identifier. If a value of None is passed an identifier elastic
                search will automatically generate a id.
            data (dict): The record data.
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        self._token_available()  # ensure a token is available

        response_data = None
        headers = {'Content-Type': 'application/json', 'DB-Method': 'POST'}
        if rid is None:
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/'
        else:
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
        r = self.tcex.session.post(url, json=data, headers=headers)
        self.tcex.log.debug(f'datastore post status code: {r.status_code}')
        if r.ok and 'application/json' in r.headers.get('content-type', ''):
            response_data = r.json()
        else:
            error = r.text or r.reason
            self.tcex.handle_error(805, ['post', r.status_code, error], raise_on_error)
        return response_data

    def put(self, rid, data, raise_on_error=True):
        """Update the data for the provided Id.

        Args:
            rid (str): The record identifier.
            data (dict): A search query
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        self._token_available()  # ensure a token is available

        response_data = None
        headers = {'Content-Type': 'application/json', 'DB-Method': 'PUT'}
        url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
        r = self.tcex.session.post(url, json=data, headers=headers)
        self.tcex.log.debug(f'datastore put status code: {r.status_code}')
        if r.ok and 'application/json' in r.headers.get('content-type', ''):
            response_data = r.json()
        else:
            error = r.text or r.reason
            self.tcex.handle_error(805, ['put', r.status_code, error], raise_on_error)
        return response_data

    def update(self, rid, data, raise_on_error=True):
        """Update the for the provided Id. Alias for put() method.

        Args:
            rid (str): The record identifier.
            data (dict): The record data.
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        return self.put(rid, data, raise_on_error)
<commit_msg>change error handling for data store error message issue<commit_after># -*- coding: utf-8 -*-
"""TcEx Framework Module for working with DataStore in the ThreatConnect Platform."""

class DataStore:
    """TcEx DataStore Class.

    **domain**

    This can be either “system”, “organization”, or “local”. Any string not defined as these
    3 will return a “Resource Not Found” error response.

    **dataType**

    This is a free form index type name as allowed by ElasticSearch. The ThreatConnect API will
    use this resource verbatim for ElasticSearch calls.

    **searchCommand**

    The URI string after the typeName is assumed to be ElasticSearch relevant information and is
    passed directly to ElasticSearch.

    """

    def __init__(self, tcex, domain, data_type, mapping=None):
        """Initialize class properties.

        Args:
            tcex ([type]): [description]
            domain (str): A value of “system”, “organization”, or “local”.
            data_type (str): A free form type name for the data.
            mapping (dict, optional): Defaults to None. Elasticsearch mappings data.

        Raises:
            RuntimeError: [description]
        """
        self.tcex = tcex
        self.domain = domain
        self.data_type = data_type
        self.mapping = mapping or {'dynamic': False}

        # properties
        self._create_index()  # create the initial index.
        self._update_mappings()  # update mappings

    def _create_index(self):
        """Create index if it doesn't exist."""
        if not self.index_exists:
            rid = 'temp-index-create'
            index_settings = {}
            headers = {'Content-Type': 'application/json', 'DB-Method': 'POST'}
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
            r = self.tcex.session.post(url, json=index_settings, headers=headers)

            if not r.ok:
                error = r.text or r.reason
                self.tcex.handle_error(800, [r.status_code, error])
            self.tcex.log.debug(
                f'creating index. status_code: {r.status_code}, response: "{r.text}".'
            )
            # delete temporary record
            self.delete(rid, False)

    def _token_available(self):
        """Raise an error if token is not available.

        This check needs to be made after args are parsed to ensure token module has initialized.
        """
        # This module requires a token.
        if self.tcex.token.token is None:  # pragma: no cover
            raise RuntimeError(
                'The DataModel TcEx Module requires a Token to interact with the '
                'ThreatConnect platform.'
            )

    def _update_mappings(self):
        """Update the mappings for the current index."""
        headers = {'Content-Type': 'application/json', 'DB-Method': 'PUT'}
        url = f'/v2/exchange/db/{self.domain}/{self.data_type}/_mappings'
        r = self.tcex.session.post(url, json=self.mapping, headers=headers)
        self.tcex.log.debug(f'update mapping. status_code: {r.status_code}, response: "{r.text}".')

    @property
    def index_exists(self):
        """Check to see if index exists."""
        headers = {'Content-Type': 'application/json', 'DB-Method': 'GET'}
        url = f'/v2/exchange/db/{self.domain}/{self.data_type}/_search'
        r = self.tcex.session.post(url, headers=headers)
        if not r.ok:
            self.tcex.log.warning(f'The provided index was not found ({r.text}).')
            return False
        return True

    def add(self, rid, data, raise_on_error=True):
        """Write data to the DataStore. Alias for post() method.

        Args:
            rid (str): The record identifier.
            data (dict): The record data.
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        return self.post(rid, data, raise_on_error)

    def delete(self, rid, raise_on_error=True):
        """Delete a record from the index using provide Id.

        Args:
            rid (str): The record identifier.
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        self._token_available()  # ensure a token is available

        response_data = None
        headers = {'Content-Type': 'application/json', 'DB-Method': 'DELETE'}
        url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
        r = self.tcex.session.post(url, headers=headers)
        self.tcex.log.debug(f'datastore delete status code: {r.status_code}')
        if r.ok and 'application/json' in r.headers.get('content-type', ''):
            response_data = r.json()
        else:
            error = r.text or r.reason
            self.tcex.handle_error(805, ['delete', r.status_code, error], raise_on_error)
        return response_data

    def get(self, rid=None, data=None, raise_on_error=True):
        """Get data from the DataStore.

        Args:
            rid (str, Optional): Defaults to None. The record identifier.
            data (dict, Optional): Defaults to None. A search query
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        self._token_available()  # ensure a token is available

        response_data = None
        headers = {'Content-Type': 'application/json', 'DB-Method': 'GET'}
        if rid is None:
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/'
        else:
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
        r = self.tcex.session.post(url, json=data, headers=headers)
        self.tcex.log.debug(f'datastore get status code: {r.status_code}')
        if 'application/json' in r.headers.get('content-type', ''):
            # as long as the content is JSON set the value
            try:
                response_data = r.json()
            # TODO this issue should be addressed by core.  This is a temporary solution
            except Exception as e:
                self.tcex.log.warning(
                    f'DataStore API returned a non-JSON response, even though content-type was \
                        application/json: {r.content} \
                        error: {e}'
                )
        if not r.ok:
            error = r.text or r.reason
            self.tcex.handle_error(805, ['get', r.status_code, error], raise_on_error)
        return response_data

    def post(self, rid, data, raise_on_error=True):
        """Write data to the DataStore.

        Args:
            rid (str): The record identifier. If a value of None is passed an identifier elastic
                search will automatically generate a id.
            data (dict): The record data.
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        self._token_available()  # ensure a token is available

        response_data = None
        headers = {'Content-Type': 'application/json', 'DB-Method': 'POST'}
        if rid is None:
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/'
        else:
            url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
        r = self.tcex.session.post(url, json=data, headers=headers)
        self.tcex.log.debug(f'datastore post status code: {r.status_code}')
        if r.ok and 'application/json' in r.headers.get('content-type', ''):
            response_data = r.json()
        else:
            error = r.text or r.reason
            self.tcex.handle_error(805, ['post', r.status_code, error], raise_on_error)
        return response_data

    def put(self, rid, data, raise_on_error=True):
        """Update the data for the provided Id.

        Args:
            rid (str): The record identifier.
            data (dict): A search query
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        self._token_available()  # ensure a token is available

        response_data = None
        headers = {'Content-Type': 'application/json', 'DB-Method': 'PUT'}
        url = f'/v2/exchange/db/{self.domain}/{self.data_type}/{rid}'
        r = self.tcex.session.post(url, json=data, headers=headers)
        self.tcex.log.debug(f'datastore put status code: {r.status_code}')
        if r.ok and 'application/json' in r.headers.get('content-type', ''):
            response_data = r.json()
        else:
            error = r.text or r.reason
            self.tcex.handle_error(805, ['put', r.status_code, error], raise_on_error)
        return response_data

    def update(self, rid, data, raise_on_error=True):
        """Update the for the provided Id. Alias for put() method.

        Args:
            rid (str): The record identifier.
            data (dict): The record data.
            raise_on_error (bool): If True and not r.ok this method will raise a RunTimeError.

        Returns:
            object : Python request response.
        """
        return self.put(rid, data, raise_on_error)
<|endoftext|>


<commit_before>import heapq
import itertools

class SendQueue(object):
    """ Simple send queue. Stores messages and returns a string of all messages.
    """
    def __init__(self, separator=','):
        self.SEPARATOR = separator
        self._queue = []

    def put(self, msg):
        """ puts a new message in the queue. """
        self._queue.append(msg)

    def get(self):
        """ returns enqueued messages in a single string, separator-joined """
        return self.SEPARATOR.join(self._queue)

    def clear(self):
        """ empties the queue. """
        self._queue = []

    def is_empty(self):
        """ check if the queue is empty. """
        return not self._queue

class PriorityQueue(object):
    def __init__(self):
        self._queue = []

        self.counter = itertools.count()   # needed to preserve insertion order
                                           # in elements with the same priority

    def put(self, el):
        count = next(self.counter)
        heapq.heappush(self._queue, (el, count))

    def peek(self):
        return self._queue[0][0]

    def pop(self):
        return heapq.heappop(self._queue)[0]

    def is_empty(self):
        return not self._queue

<commit_msg>added simplistic priority queue<commit_after>import heapq
import itertools

class SendQueue(object):
    """ Simple send queue. Stores messages and returns a string of all messages.
    """
    def __init__(self, separator=','):
        self.SEPARATOR = separator
        self._queue = []

    def put(self, msg):
        """ puts a new message in the queue. """
        self._queue.append(msg)

    def get(self):
        """ returns enqueued messages in a single string, separator-joined """
        return self.SEPARATOR.join(self._queue)

    def clear(self):
        """ empties the queue. """
        self._queue = []

    def is_empty(self):
        """ check if the queue is empty. """
        return not self._queue

class PriorityQueue(object):
    """ Simplistic priority queue.
    """
    def __init__(self):
        self._queue = []

        self.counter = itertools.count()   # needed to preserve insertion order
                                           # in elements with the same priority

    def put(self, el):
        """ Put a new element in the queue. """
        count = next(self.counter)
        heapq.heappush(self._queue, (el, count))

    def peek(self):
        """ Returns the highest priority elemente from the queue without
        removing it. """
        return self._queue[0][0]

    def pop(self):
        """ Remove and returns the highest priority element form the queue. """
        return heapq.heappop(self._queue)[0]

    def is_empty(self):
        """ Checks if the queue is empty. """
        return not self._queue

<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
Personality of the fish
"""

import json

class FishPersonality:

    gender = "Female" # random initialization. Note that this isn't binary
    myers_briggs = "ISTP" # random initialization
    personalityFile = "fish_personality.json"

    def __init__(self):
        # check for existence of personalityFile
        try:
            with open(self.personalityFile, 'r', encoding='utf-8') as filePath:
                basic_entry = json.load(filePath)
                self.gender = basic_entry['gender']
                self.myers_briggs = basic_entry['myers_briggs']
        except IOError as e:
            self.updatePersonalityFile() #Does not exist OR no read permissions

    def updatePersonalityFile(self):
        basic_entry = {}
        basic_entry['gender'] = self.gender
        basic_entry['myers_briggs'] = self.myers_briggs

        with open(self.personalityFile, mode='w', encoding='utf-8') as filePath:
            json.dump(basic_entry, filePath, indent=2)

    def getGender(self):
        return self.gender

    def setGender(self,newGender):
        self.gender = newGender
        self.updatePersonalityFile()
<commit_msg>putting son at an absolute path<commit_after>#!/usr/bin/env python3
"""
Personality of the fish
"""

import json

class FishPersonality:

    gender = "Female" # random initialization. Note that this isn't binary
    myers_briggs = "ISTP" # random initialization
    personalityFile = "/home/pi/rubberfish/fish_personality.json"

    def __init__(self):
        # check for existence of personalityFile
        try:
            with open(self.personalityFile, 'r', encoding='utf-8') as filePath:
                basic_entry = json.load(filePath)
                self.gender = basic_entry['gender']
                self.myers_briggs = basic_entry['myers_briggs']
        except IOError as e:
            self.updatePersonalityFile() #Does not exist OR no read permissions

    def updatePersonalityFile(self):
        basic_entry = {}
        basic_entry['gender'] = self.gender
        basic_entry['myers_briggs'] = self.myers_briggs

        with open(self.personalityFile, mode='w', encoding='utf-8') as filePath:
            json.dump(basic_entry, filePath, indent=2)

    def getGender(self):
        return self.gender

    def setGender(self,newGender):
        self.gender = newGender
        self.updatePersonalityFile()
<|endoftext|>


<commit_before>"""Evaluates a sorting model, adding scalar summaries to tensorboard.

It also outputs evaluation metrics as numpy scalars.
The flag hparam has to be passed as a string of comma separated statements of
the form hparam=value, where the hparam's are any of the listed in the
dictionary DEFAULT_HPARAMS.
See the README.md file for further compilation and running instructions.

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import time

import tensorflow as tf

import sorting_model

gfile = tf.gfile
flags = tf.app.flags
flags.DEFINE_string("hparams", "", "Hyperparameters")
flags.DEFINE_string("batch_transform_type",
                    None, "Options: None, WeakMix, StrongMix")
flags.DEFINE_boolean("evaluate_all",
                     True, "If false evaluate only the last checkpoint")
flags.DEFINE_string("exp_log_dir", "/tmp/sorting/",
                    "Directory where to write event logs.")
flags.DEFINE_boolean("eval_once", True,
                     "If true, only evaluate the model once")
flags.DEFINE_integer("sec_sleep", 30,
                     "If no new checkpoint, sleep for some seconds")
flags.DEFINE_integer("secs_run_for", 10000,
                     "Maximum time (seconds) the program will be active")

FLAGS = tf.app.flags.FLAGS
DEFAULT_HPARAMS = tf.contrib.training.HParams(n_numbers=50,
                                              lr=0.1,
                                              temperature=1.0,
                                              batch_size=50,
                                              prob_inc=1.0,
                                              samples_per_num=5,
                                              n_iter_sinkhorn=10,
                                              n_units=32,
                                              noise_factor=0.0,
                                              optimizer="adam",
                                              keep_prob=1.)

def log(s):
  tf.logging.info(s)
  print(s)

def wait_for_new_checkpoint(saver, sess, logdir, global_step,
                            last_step_evaluated, sleep_secs):
  while True:
    if restore_checkpoint_if_exists(saver, sess, logdir):
      step = sess.run(global_step)
      if step <= last_step_evaluated:
        log("Found old checkpoint, sleeping %ds" % sleep_secs)
        time.sleep(sleep_secs)
      else:
        return step
    else:
      log("Checkpoint not found in %s,"
          "sleeping for %ds" % (logdir, sleep_secs))
      time.sleep(sleep_secs)

def restore_checkpoint_if_exists(saver, sess, logdir):
  ckpt = tf.train.get_checkpoint_state(logdir)
  if ckpt:
    log("Restoring checking point from %s" % ckpt.model_checkpoint_path)
    ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
    full_checkpoint_path = os.path.join(logdir, ckpt_name)
    saver.restore(sess, full_checkpoint_path)
    return True
  return False

# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

def main(_):
  time_start = time.time()
  hparams = DEFAULT_HPARAMS
  hparams.parse(FLAGS.hparams)

  if not gfile.Exists(FLAGS.exp_log_dir):
    gfile.MakeDirs(FLAGS.exp_log_dir)

  g = tf.Graph()
  model = sorting_model.SortingModel(g, hparams)

  with model.graph.as_default():
    model.set_input()
    model.build_network()
    model.build_hard_losses()
    model.add_summaries_eval()
    summaries_eval = tf.summary.merge_all()
    saver = tf.train.Saver()
    writer = tf.summary.FileWriter(FLAGS.exp_log_dir, model.graph)
    last_step_evaluated = -1
    with tf.Session() as session:
      while time.time() - time_start < FLAGS.secs_run_for:
        wait_for_new_checkpoint(
            saver, session, FLAGS.exp_log_dir,
            model.global_step, last_step_evaluated, FLAGS.sec_sleep)
        (summaries, step, eval_measures) = session.run([
            summaries_eval, model.global_step, model.get_eval_measures()])
        (l1_diff, l2sh_diff, kendall_tau,
         prop_wrong, prop_any_wrong) = eval_measures
        log("Frequency of mistakes was %s" % prop_wrong)
        log("Frequency of series with at least an error was %s" %
            prop_any_wrong)
        log("Kendall's tau was %s" % kendall_tau)
        log(("Mean L2 squared difference between true and inferred series "
             " was %s") % l2sh_diff)
        log("Mean L1 difference between true and inferred series was %s"
            % l1_diff)
        writer.add_summary(summaries, global_step=step)
        last_step_evaluated = step
        if FLAGS.eval_once is True:
          break

if __name__ == "__main__":
  tf.app.run()
<commit_msg>Update sorting_eval.py<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Evaluates a sorting model, adding scalar summaries to tensorboard.

It also outputs evaluation metrics as numpy scalars.
The flag hparam has to be passed as a string of comma separated statements of
the form hparam=value, where the hparam's are any of the listed in the
dictionary DEFAULT_HPARAMS.
See the README.md file for further compilation and running instructions.

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import time

import tensorflow as tf

import sorting_model

gfile = tf.gfile
flags = tf.app.flags
flags.DEFINE_string("hparams", "", "Hyperparameters")
flags.DEFINE_string("batch_transform_type",
                    None, "Options: None, WeakMix, StrongMix")
flags.DEFINE_boolean("evaluate_all",
                     True, "If false evaluate only the last checkpoint")
flags.DEFINE_string("exp_log_dir", "/tmp/sorting/",
                    "Directory where to write event logs.")
flags.DEFINE_boolean("eval_once", True,
                     "If true, only evaluate the model once")
flags.DEFINE_integer("sec_sleep", 30,
                     "If no new checkpoint, sleep for some seconds")
flags.DEFINE_integer("secs_run_for", 10000,
                     "Maximum time (seconds) the program will be active")

FLAGS = tf.app.flags.FLAGS
DEFAULT_HPARAMS = tf.contrib.training.HParams(n_numbers=50,
                                              lr=0.1,
                                              temperature=1.0,
                                              batch_size=50,
                                              prob_inc=1.0,
                                              samples_per_num=5,
                                              n_iter_sinkhorn=10,
                                              n_units=32,
                                              noise_factor=0.0,
                                              optimizer="adam",
                                              keep_prob=1.)

def log(s):
  tf.logging.info(s)
  print(s)

def wait_for_new_checkpoint(saver, sess, logdir, global_step,
                            last_step_evaluated, sleep_secs):
  while True:
    if restore_checkpoint_if_exists(saver, sess, logdir):
      step = sess.run(global_step)
      if step <= last_step_evaluated:
        log("Found old checkpoint, sleeping %ds" % sleep_secs)
        time.sleep(sleep_secs)
      else:
        return step
    else:
      log("Checkpoint not found in %s,"
          "sleeping for %ds" % (logdir, sleep_secs))
      time.sleep(sleep_secs)

def restore_checkpoint_if_exists(saver, sess, logdir):
  ckpt = tf.train.get_checkpoint_state(logdir)
  if ckpt:
    log("Restoring checking point from %s" % ckpt.model_checkpoint_path)
    ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
    full_checkpoint_path = os.path.join(logdir, ckpt_name)
    saver.restore(sess, full_checkpoint_path)
    return True
  return False

# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

def main(_):
  time_start = time.time()
  hparams = DEFAULT_HPARAMS
  hparams.parse(FLAGS.hparams)

  if not gfile.Exists(FLAGS.exp_log_dir):
    gfile.MakeDirs(FLAGS.exp_log_dir)

  g = tf.Graph()
  model = sorting_model.SortingModel(g, hparams)

  with model.graph.as_default():
    model.set_input()
    model.build_network()
    model.build_hard_losses()
    model.add_summaries_eval()
    summaries_eval = tf.summary.merge_all()
    saver = tf.train.Saver()
    writer = tf.summary.FileWriter(FLAGS.exp_log_dir, model.graph)
    last_step_evaluated = -1
    with tf.Session() as session:
      while time.time() - time_start < FLAGS.secs_run_for:
        wait_for_new_checkpoint(
            saver, session, FLAGS.exp_log_dir,
            model.global_step, last_step_evaluated, FLAGS.sec_sleep)
        (summaries, step, eval_measures) = session.run([
            summaries_eval, model.global_step, model.get_eval_measures()])
        (l1_diff, l2sh_diff, kendall_tau,
         prop_wrong, prop_any_wrong) = eval_measures
        log("Frequency of mistakes was %s" % prop_wrong)
        log("Frequency of series with at least an error was %s" %
            prop_any_wrong)
        log("Kendall's tau was %s" % kendall_tau)
        log(("Mean L2 squared difference between true and inferred series "
             " was %s") % l2sh_diff)
        log("Mean L1 difference between true and inferred series was %s"
            % l1_diff)
        writer.add_summary(summaries, global_step=step)
        last_step_evaluated = step
        if FLAGS.eval_once is True:
          break

if __name__ == "__main__":
  tf.app.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import geojson, csv, dateutil, datetime, time, os, zipfile, pytz, json, shutil, urllib, math, subprocess
from housepy import config, log, util, strings, emailer, net
from mongo import db

EMAILS = ",".join(config['geo_emails']) if config['geo_emails'] is not None else None

def main():
    log.info("beacon_sender...")
    if config['geo_emails'] is None or not len(config['geo_emails']):
        log.info("--> no emails")
        return
    try:
        # core_sat = config['satellites'][0]
        # last_beacon = list(db.features.find({'properties.FeatureType': "beacon", 'properties.Satellite': {'$eq': core_sat}}).sort('properties.t_utc', -1).limit(1))[0]
        last_beacon = list(db.features.find({'properties.FeatureType': "beacon"}).sort('properties.t_utc', -1).limit(1))[0]
        datetime = last_beacon['properties']['DateTime']
        lon, lat = last_beacon['geometry']['coordinates']
        satellite = last_beacon['properties']['Satellite']
        text = "Satellite:\n%s\n\nTime:\n%s\n\nLat,Lon:\n%f,%f" % (satellite, datetime, lat, lon)
        log.info("--> last reported beacon (%s on %s) at: %f,%f" % (satellite, datetime, lat, lon))
    except Exception as e:
        log.error("Could not get update: %s" % log.exc(e))
    try:
        log.info("Emailing to %s..." % EMAILS)
        emailer.send(EMAILS, "Core beacon location", text)
    except Exception as e:
        log.error("Could not email: %s" % log.exc(e))

main()
<commit_msg>updating sat emailer<commit_after>#!/usr/bin/env python3

import geojson, csv, dateutil, datetime, time, os, zipfile, pytz, json, shutil, urllib, math, subprocess
from housepy import config, log, util, strings, emailer, net
from mongo import db

EMAILS = ",".join(config['geo_emails']) if config['geo_emails'] is not None else None

def main():
    log.info("beacon_sender...")
    if config['geo_emails'] is None or not len(config['geo_emails']):
        log.info("--> no emails")
        return
    text = []
    try:
        for satellite in config['satellites']:
            last_beacon = list(db.features.find({'properties.FeatureType': "beacon", 'properties.Satellite': {'$eq': satellite}}).sort('properties.t_utc', -1).limit(1))[0]
            datetime = last_beacon['properties']['DateTime']
            lon, lat = last_beacon['geometry']['coordinates']
            satellite = last_beacon['properties']['Satellite']
            text.append("%s\n%s\n%f,%f" % (satellite, datetime, lat, lon))
            log.info("--> last reported beacon (%s on %s) at: %f,%f" % (satellite, datetime, lat, lon))
    except Exception as e:
        log.error("Could not get update: %s" % log.exc(e))
    try:
        log.info("Emailing to %s..." % EMAILS)
        text = "\n\n".join(text)
        emailer.send(EMAILS, "Core beacon location", text)
    except Exception as e:
        log.error("Could not email: %s" % log.exc(e))

main()
<|endoftext|>


<commit_before><commit_msg>iteritems<commit_after><|endoftext|>


<commit_before>"""
When this module is imported, we load all the .json files and insert them as
module attributes using locals().  It's a magical and wonderful process.
"""
import json
import os

# During install, we need this to pass even without Django present.
try:
    from django.conf import settings
except ImportError:
    settings = None

from . import settings_defaults

VERSION = (0, 3)
__version__ = '.'.join(map(str, VERSION))
__all__ = ['VERSION', '__version__']

def settings_fallback(key):
    """Grab user-defined settings, or fall back to default."""
    try:
        return getattr(settings, key)
    except (AttributeError, ImportError):
        return getattr(settings_defaults, key)

json_dir = settings_fallback('PROD_DETAILS_DIR')

for filename in os.listdir(json_dir):
    if filename.endswith('.json'):
        name = os.path.splitext(filename)[0]
        path = os.path.join(json_dir, filename)
        locals()[name] = json.load(open(path))
        __all__.append(name)
<commit_msg>Do not break the django app when JSON files are not present (yet). Closes #2.<commit_after>"""
When this module is imported, we load all the .json files and insert them as
module attributes using locals().  It's a magical and wonderful process.
"""
import json
import logging
import os

# During `pip install`, we need this to pass even without Django present.
try:
    from django.conf import settings
except ImportError:
    settings = None

from . import settings_defaults

VERSION = (0, 4)
__version__ = '.'.join(map(str, VERSION))
__all__ = ['VERSION', '__version__', 'product_details']

log = logging.getLogger('product_details')
log.setLevel(logging.WARNING)

class ProductDetails(object):
    """
    Main product details class. Implements the JSON files' content as
    attributes, e.g.: product_details.firefox_version_history .
    """
    json_data = {}

    def __init__(self):
        """Load JSON files and keep them in memory."""

        json_dir = self._settings_fallback('PROD_DETAILS_DIR')

        for filename in os.listdir(json_dir):
            if filename.endswith('.json'):
                name = os.path.splitext(filename)[0]
                path = os.path.join(json_dir, filename)
                self.json_data[name] = json.load(open(path))

    def _settings_fallback(self, key):
        """Grab user-defined settings, or fall back to default."""
        try:
            return getattr(settings, key)
        except (AttributeError, ImportError):
            return getattr(settings_defaults, key)

    def __getattr__(self, key):
        """Catch-all for access to JSON files."""
        try:
            return self.json_data[key]
        except KeyError:
            log.warn('Requested product details file %s not found!' % key)
            return FakeDict()

product_details = ProductDetails()

class FakeDict(dict):
    """Fake dictionary that'll pretend to know every key."""
    def __getitem__(self, key):
        return None
<|endoftext|>


<commit_before>""" ASOS Daily Summary Message Parser ingestor """

# Twisted Python imports
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/dsm_parser', facility=LOG_LOCAL2)
from twisted.python import log

# Twisted Python imports
from twisted.internet import reactor

# Standard Python modules
import re

# Python 3rd Party Add-Ons
import datetime

# pyWWA stuff
from pyldm import ldmbridge
import common

DBPOOL = common.get_database("iem", cp_max=1)

class MyProductIngestor(ldmbridge.LDMProductReceiver):
    """ I receive products from ldmbridge and process them 1 by 1 :) """

    def connectionLost(self, reason):
        log.msg('connectionLost')
        log.err(reason)
        reactor.callLater(5, self.shutdown)

    def shutdown(self):
        reactor.callWhenRunning(reactor.stop)

    def process_data(self, buf):
        """ Process the product """
        try:
            real_parser(buf)
        except Exception, myexp:
            common.email_error(myexp, buf)

def real_parser(buf):
    # KAMW DS 19/07 761539/ 540532// 76/ 54//0062028/00/00/00/00/00/00/00/
    # 00/00/00/00/00/00/00/00/00/00/00/00/00/00/00/00/00/00/25/12091717/
    # 14131452=
    # Split lines
    raw = buf.replace("\015\015\012", "\n")
    lines = raw.split("\n")
    if len(lines[3]) < 10:
        meat = ("".join(lines[4:])).split("=")
    else:
        meat = ("".join(lines[3:])).split("=")
    for data in meat:
        if data == "":
            continue
        process_dsm(data)

PARSER_RE = re.compile("""^(?P<id>[A-Z][A-Z0-9]{3})\s+
   DS\s+
   (COR\s)?
   ([0-9]{4}\s)?
   (?P<day>\d\d)/(?P<month>\d\d)\s?
   ((?P<highmiss>M)|((?P<high>(-?\d+))(?P<hightime>[0-9]{4})))/\s?
   ((?P<lowmiss>M)|((?P<low>(-?\d+))(?P<lowtime>[0-9]{4})))//\s?
   (?P<coophigh>(-?\d+|M))/\s?
   (?P<cooplow>(-?\d+|M))//
   (?P<minslp>M|[0-9]{3,4})(?P<slptime>[0-9]{4})?/
   (?P<precip>T|M|[0-9]{,4})/
 (?P<p01>T|M|\-|[0-9]{,4})/(?P<p02>T|M|\-|[0-9]{,4})/(?P<p03>T|M|\-|[0-9]{,4})/
 (?P<p04>T|M|\-|[0-9]{,4})/(?P<p05>T|M|\-|[0-9]{,4})/(?P<p06>T|M|\-|[0-9]{,4})/
 (?P<p07>T|M|\-|[0-9]{,4})/(?P<p08>T|M|\-|[0-9]{,4})/(?P<p09>T|M|\-|[0-9]{,4})/
 (?P<p10>T|M|\-|[0-9]{,4})/(?P<p11>T|M|\-|[0-9]{,4})/(?P<p12>T|M|\-|[0-9]{,4})/
 (?P<p13>T|M|\-|[0-9]{,4})/(?P<p14>T|M|\-|[0-9]{,4})/(?P<p15>T|M|\-|[0-9]{,4})/
 (?P<p16>T|M|\-|[0-9]{,4})/(?P<p17>T|M|\-|[0-9]{,4})/(?P<p18>T|M|\-|[0-9]{,4})/
 (?P<p19>T|M|\-|[0-9]{,4})/(?P<p20>T|M|\-|[0-9]{,4})/(?P<p21>T|M|\-|[0-9]{,4})/
 (?P<p22>T|M|\-|[0-9]{,4})/(?P<p23>T|M|\-|[0-9]{,4})/(?P<p24>T|M|\-|[0-9]{,4})/
   (?P<avg_sped>[0-9]{3})?/?
   (?P<drct_sped_max>[0-9]{2})?(?P<sped_max>[0-9]{2})?(?P<time_sped_max>[0-9]{4})?/?
   (?P<drct_gust_max>[0-9]{2})?(?P<sped_gust_max>[0-9]{2})?(?P<time_gust_max>[0-9]{4})?
""", re.VERBOSE)

def process_dsm(data):
    m = PARSER_RE.match(data)
    if m is None:
        log.msg("FAIL ||%s||" % (data, ))
        common.email_error("DSM RE Match Failure", data)
        return
    d = m.groupdict()
    # Only parse United States Sites
    if d['id'][0] != "K":
        return
    # Figure out the timestamp
    now = datetime.datetime.now()
    if d['day'] == '00' or d['month'] == '00':
        common.email_error("ERROR: Invalid Timestamp", data)
        return
    ts = now.replace(day=int(d['day']), month=int(d['month']))
    if ts.month == 12 and now.month == 1:
        ts -= datetime.timedelta(days=360)
        ts = ts.replace(day=int(d['day']))
    updater = []
    if d['high'] is not None and d['high'] != "M":
        updater.append("max_tmpf = %s" % (d['high'],))
    if d['low'] is not None and d['low'] != "M":
        updater.append("min_tmpf = %s" % (d['low'],))

    if d['precip'] is not None and d['precip'] not in ["M", "T"]:
        updater.append("pday = %s" % (float(d['precip']) / 100.0,))
    elif d['precip'] == "T":
        updater.append("pday = 0.0001")

    sql = """
        UPDATE summary_%s s SET %s FROM stations t WHERE t.iemid = s.iemid
        and t.id = '%s' and day = '%s'
        """ % (ts.year, " , ".join(updater), d['id'][1:],
               ts.strftime("%Y-%m-%d"))
    log.msg("%s %s H:%s L:%s P:%s" % (d['id'], ts.strftime("%Y-%m-%d"),
                                      d['high'], d['low'], d['precip']))
    if len(updater) > 0:
        defer = DBPOOL.runOperation(sql)
        defer.addErrback(common.email_error, sql)

if __name__ == '__main__':
    ldm = ldmbridge.LDMProductFactory(MyProductIngestor())
    reactor.run()
<commit_msg>allow negative minslp for now<commit_after>""" ASOS Daily Summary Message Parser ingestor """

# Twisted Python imports
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/dsm_parser', facility=LOG_LOCAL2)
from twisted.python import log

# Twisted Python imports
from twisted.internet import reactor

# Standard Python modules
import re

# Python 3rd Party Add-Ons
import datetime

# pyWWA stuff
from pyldm import ldmbridge
import common

DBPOOL = common.get_database("iem", cp_max=1)

class MyProductIngestor(ldmbridge.LDMProductReceiver):
    """ I receive products from ldmbridge and process them 1 by 1 :) """

    def connectionLost(self, reason):
        log.msg('connectionLost')
        log.err(reason)
        reactor.callLater(5, self.shutdown)

    def shutdown(self):
        reactor.callWhenRunning(reactor.stop)

    def process_data(self, buf):
        """ Process the product """
        try:
            real_parser(buf)
        except Exception, myexp:
            common.email_error(myexp, buf)

def real_parser(buf):
    # KAMW DS 19/07 761539/ 540532// 76/ 54//0062028/00/00/00/00/00/00/00/
    # 00/00/00/00/00/00/00/00/00/00/00/00/00/00/00/00/00/00/25/12091717/
    # 14131452=
    # Split lines
    raw = buf.replace("\015\015\012", "\n")
    lines = raw.split("\n")
    if len(lines[3]) < 10:
        meat = ("".join(lines[4:])).split("=")
    else:
        meat = ("".join(lines[3:])).split("=")
    for data in meat:
        if data == "":
            continue
        process_dsm(data)

PARSER_RE = re.compile("""^(?P<id>[A-Z][A-Z0-9]{3})\s+
   DS\s+
   (COR\s)?
   ([0-9]{4}\s)?
   (?P<day>\d\d)/(?P<month>\d\d)\s?
   ((?P<highmiss>M)|((?P<high>(-?\d+))(?P<hightime>[0-9]{4})))/\s?
   ((?P<lowmiss>M)|((?P<low>(-?\d+))(?P<lowtime>[0-9]{4})))//\s?
   (?P<coophigh>(-?\d+|M))/\s?
   (?P<cooplow>(-?\d+|M))//
   (?P<minslp>M|[\-0-9]{3,4})(?P<slptime>[0-9]{4})?/
   (?P<precip>T|M|[0-9]{,4})/
 (?P<p01>T|M|\-|[0-9]{,4})/(?P<p02>T|M|\-|[0-9]{,4})/(?P<p03>T|M|\-|[0-9]{,4})/
 (?P<p04>T|M|\-|[0-9]{,4})/(?P<p05>T|M|\-|[0-9]{,4})/(?P<p06>T|M|\-|[0-9]{,4})/
 (?P<p07>T|M|\-|[0-9]{,4})/(?P<p08>T|M|\-|[0-9]{,4})/(?P<p09>T|M|\-|[0-9]{,4})/
 (?P<p10>T|M|\-|[0-9]{,4})/(?P<p11>T|M|\-|[0-9]{,4})/(?P<p12>T|M|\-|[0-9]{,4})/
 (?P<p13>T|M|\-|[0-9]{,4})/(?P<p14>T|M|\-|[0-9]{,4})/(?P<p15>T|M|\-|[0-9]{,4})/
 (?P<p16>T|M|\-|[0-9]{,4})/(?P<p17>T|M|\-|[0-9]{,4})/(?P<p18>T|M|\-|[0-9]{,4})/
 (?P<p19>T|M|\-|[0-9]{,4})/(?P<p20>T|M|\-|[0-9]{,4})/(?P<p21>T|M|\-|[0-9]{,4})/
 (?P<p22>T|M|\-|[0-9]{,4})/(?P<p23>T|M|\-|[0-9]{,4})/(?P<p24>T|M|\-|[0-9]{,4})/
   (?P<avg_sped>[0-9]{3})?/?
   (?P<drct_sped_max>[0-9]{2})?(?P<sped_max>[0-9]{2})?(?P<time_sped_max>[0-9]{4})?/?
   (?P<drct_gust_max>[0-9]{2})?(?P<sped_gust_max>[0-9]{2})?(?P<time_gust_max>[0-9]{4})?
""", re.VERBOSE)

def process_dsm(data):
    m = PARSER_RE.match(data)
    if m is None:
        log.msg("FAIL ||%s||" % (data, ))
        common.email_error("DSM RE Match Failure", data)
        return
    d = m.groupdict()
    # Only parse United States Sites
    if d['id'][0] != "K":
        return
    # Figure out the timestamp
    now = datetime.datetime.now()
    if d['day'] == '00' or d['month'] == '00':
        common.email_error("ERROR: Invalid Timestamp", data)
        return
    ts = now.replace(day=int(d['day']), month=int(d['month']))
    if ts.month == 12 and now.month == 1:
        ts -= datetime.timedelta(days=360)
        ts = ts.replace(day=int(d['day']))
    updater = []
    if d['high'] is not None and d['high'] != "M":
        updater.append("max_tmpf = %s" % (d['high'],))
    if d['low'] is not None and d['low'] != "M":
        updater.append("min_tmpf = %s" % (d['low'],))

    if d['precip'] is not None and d['precip'] not in ["M", "T"]:
        updater.append("pday = %s" % (float(d['precip']) / 100.0,))
    elif d['precip'] == "T":
        updater.append("pday = 0.0001")

    sql = """
        UPDATE summary_%s s SET %s FROM stations t WHERE t.iemid = s.iemid
        and t.id = '%s' and day = '%s'
        """ % (ts.year, " , ".join(updater), d['id'][1:],
               ts.strftime("%Y-%m-%d"))
    log.msg("%s %s H:%s L:%s P:%s" % (d['id'], ts.strftime("%Y-%m-%d"),
                                      d['high'], d['low'], d['precip']))
    if len(updater) > 0:
        defer = DBPOOL.runOperation(sql)
        defer.addErrback(common.email_error, sql)

if __name__ == '__main__':
    ldm = ldmbridge.LDMProductFactory(MyProductIngestor())
    reactor.run()
<|endoftext|>