<commit_before>import argparse

from dvc.command.base import append_doc_link, fix_subparsers
from dvc.command.config import CmdConfig
from dvc.ui import ui
from dvc.utils import format_link

class CmdRemote(CmdConfig):
    def __init__(self, args):
        super().__init__(args)

        if getattr(self.args, "name", None):
            self.args.name = self.args.name.lower()

    def _check_exists(self, conf):
        from dvc.config import ConfigError

        if self.args.name not in conf["remote"]:
            raise ConfigError(f"remote '{self.args.name}' doesn't exist.")

class CmdRemoteAdd(CmdRemote):
    def run(self):
        from dvc.config import ConfigError

        if self.args.default:
            ui.write(f"Setting '{self.args.name}' as a default remote.")

        with self.config.edit(self.args.level) as conf:
            if self.args.name in conf["remote"] and not self.args.force:
                raise ConfigError(
                    "remote '{}' already exists. Use `-f|--force` to "
                    "overwrite it.".format(self.args.name)
                )

            conf["remote"][self.args.name] = {"url": self.args.url}
            if self.args.default:
                conf["core"]["remote"] = self.args.name

        return 0

class CmdRemoteRemove(CmdRemote):
    def run(self):
        with self.config.edit(self.args.level) as conf:
            self._check_exists(conf)
            del conf["remote"][self.args.name]

        up_to_level = self.args.level or "repo"
        # Remove core.remote refs to this remote in any shadowing configs
        for level in reversed(self.config.LEVELS):
            with self.config.edit(level) as conf:
                if conf["core"].get("remote") == self.args.name:
                    del conf["core"]["remote"]

            if level == up_to_level:
                break

        return 0

class CmdRemoteModify(CmdRemote):
    def run(self):
        from dvc.config import merge

        with self.config.edit(self.args.level) as conf:
            merged = self.config.load_config_to_level(self.args.level)
            merge(merged, conf)
            self._check_exists(merged)

            if self.args.name not in conf["remote"]:
                conf["remote"][self.args.name] = {}
            section = conf["remote"][self.args.name]
            if self.args.unset:
                section.pop(self.args.option, None)
            else:
                section[self.args.option] = self.args.value
        return 0

class CmdRemoteDefault(CmdRemote):
    def run(self):
        from dvc.config import ConfigError

        if self.args.name is None and not self.args.unset:
            conf = self.config.read(self.args.level)
            try:
                print(conf["core"]["remote"])
            except KeyError:
                ui.write("No default remote set")
                return 1
        else:
            with self.config.edit(self.args.level) as conf:
                if self.args.unset:
                    conf["core"].pop("remote", None)
                else:
                    merged_conf = self.config.load_config_to_level(
                        self.args.level
                    )
                    if (
                        self.args.name in conf["remote"]
                        or self.args.name in merged_conf["remote"]
                    ):
                        conf["core"]["remote"] = self.args.name
                    else:
                        raise ConfigError(
                            "default remote must be present in remote list."
                        )
        return 0

class CmdRemoteList(CmdRemote):
    def run(self):
        conf = self.config.read(self.args.level)
        for name, conf in conf["remote"].items():
            ui.write(name, conf["url"], sep="\t")
        return 0

class CmdRemoteRename(CmdRemote):
    def _rename_default(self, conf):
        if conf["core"].get("remote") == self.args.name:
            conf["core"]["remote"] = self.args.new

    def run(self):
        from dvc.config import ConfigError

        all_config = self.config.load_config_to_level(None)
        if self.args.new in all_config.get("remote", {}):
            raise ConfigError(
                "Rename failed. Remote name '{}' already exists.".format(
                    {self.args.new}
                )
            )

        with self.config.edit(self.args.level) as conf:
            self._check_exists(conf)
            conf["remote"][self.args.new] = conf["remote"][self.args.name]
            del conf["remote"][self.args.name]
            self._rename_default(conf)

        up_to_level = self.args.level or "repo"
        for level in reversed(self.config.LEVELS):
            if level == up_to_level:
                break
            with self.config.edit(level) as level_conf:
                self._rename_default(level_conf)

        return 0

def add_parser(subparsers, parent_parser):
    from dvc.command.config import parent_config_parser

    REMOTE_HELP = "Set up and manage data remotes."
    remote_parser = subparsers.add_parser(
        "remote",
        parents=[parent_parser],
        description=append_doc_link(REMOTE_HELP, "remote"),
        help=REMOTE_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )

    remote_subparsers = remote_parser.add_subparsers(
        dest="cmd",
        help="Use `dvc remote CMD --help` for " "command-specific help.",
    )

    fix_subparsers(remote_subparsers)

    REMOTE_ADD_HELP = "Add a new data remote."
    remote_add_parser = remote_subparsers.add_parser(
        "add",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_ADD_HELP, "remote/add"),
        help=REMOTE_ADD_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_add_parser.add_argument("name", help="Name of the remote")
    remote_add_parser.add_argument(
        "url",
        help="Remote location. See full list of supported URLs at {}".format(
            format_link("https://man.dvc.org/remote")
        ),
    )
    remote_add_parser.add_argument(
        "-d",
        "--default",
        action="store_true",
        default=False,
        help="Set as default remote.",
    )
    remote_add_parser.add_argument(
        "-f",
        "--force",
        action="store_true",
        default=False,
        help="Force overwriting existing configs",
    )
    remote_add_parser.set_defaults(func=CmdRemoteAdd)

    REMOTE_DEFAULT_HELP = "Set/unset the default data remote."
    remote_default_parser = remote_subparsers.add_parser(
        "default",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote/default"),
        help=REMOTE_DEFAULT_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_default_parser.add_argument(
        "name", nargs="?", help="Name of the remote"
    )
    remote_default_parser.add_argument(
        "-u",
        "--unset",
        action="store_true",
        default=False,
        help="Unset default remote.",
    )
    remote_default_parser.set_defaults(func=CmdRemoteDefault)

    REMOTE_MODIFY_HELP = "Modify the configuration of a data remote."
    remote_modify_parser = remote_subparsers.add_parser(
        "modify",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_MODIFY_HELP, "remote/modify"),
        help=REMOTE_MODIFY_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_modify_parser.add_argument("name", help="Name of the remote")
    remote_modify_parser.add_argument(
        "option", help="Name of the option to modify."
    )
    remote_modify_parser.add_argument(
        "value", nargs="?", help="(optional) Value of the option."
    )
    remote_modify_parser.add_argument(
        "-u",
        "--unset",
        default=False,
        action="store_true",
        help="Unset option.",
    )
    remote_modify_parser.set_defaults(func=CmdRemoteModify)

    REMOTE_LIST_HELP = "List all available data remotes."
    remote_list_parser = remote_subparsers.add_parser(
        "list",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_LIST_HELP, "remote/list"),
        help=REMOTE_LIST_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_list_parser.set_defaults(func=CmdRemoteList)

    REMOTE_REMOVE_HELP = "Remove a data remote."
    remote_remove_parser = remote_subparsers.add_parser(
        "remove",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
        help=REMOTE_REMOVE_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_remove_parser.add_argument(
        "name", help="Name of the remote to remove."
    )
    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
    REMOTE_RENAME_HELP = "Rename a DVC remote"
    remote_rename_parser = remote_subparsers.add_parser(
        "rename",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
        help=REMOTE_RENAME_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_rename_parser.add_argument("name", help="Remote to be renamed")
    remote_rename_parser.add_argument("new", help="New name of the remote")
    remote_rename_parser.set_defaults(func=CmdRemoteRename)
<commit_msg>command: Should not consume 'print' to output default remote info (#6650)<commit_after>import argparse

from dvc.command.base import append_doc_link, fix_subparsers
from dvc.command.config import CmdConfig
from dvc.ui import ui
from dvc.utils import format_link

class CmdRemote(CmdConfig):
    def __init__(self, args):
        super().__init__(args)

        if getattr(self.args, "name", None):
            self.args.name = self.args.name.lower()

    def _check_exists(self, conf):
        from dvc.config import ConfigError

        if self.args.name not in conf["remote"]:
            raise ConfigError(f"remote '{self.args.name}' doesn't exist.")

class CmdRemoteAdd(CmdRemote):
    def run(self):
        from dvc.config import ConfigError

        if self.args.default:
            ui.write(f"Setting '{self.args.name}' as a default remote.")

        with self.config.edit(self.args.level) as conf:
            if self.args.name in conf["remote"] and not self.args.force:
                raise ConfigError(
                    "remote '{}' already exists. Use `-f|--force` to "
                    "overwrite it.".format(self.args.name)
                )

            conf["remote"][self.args.name] = {"url": self.args.url}
            if self.args.default:
                conf["core"]["remote"] = self.args.name

        return 0

class CmdRemoteRemove(CmdRemote):
    def run(self):
        with self.config.edit(self.args.level) as conf:
            self._check_exists(conf)
            del conf["remote"][self.args.name]

        up_to_level = self.args.level or "repo"
        # Remove core.remote refs to this remote in any shadowing configs
        for level in reversed(self.config.LEVELS):
            with self.config.edit(level) as conf:
                if conf["core"].get("remote") == self.args.name:
                    del conf["core"]["remote"]

            if level == up_to_level:
                break

        return 0

class CmdRemoteModify(CmdRemote):
    def run(self):
        from dvc.config import merge

        with self.config.edit(self.args.level) as conf:
            merged = self.config.load_config_to_level(self.args.level)
            merge(merged, conf)
            self._check_exists(merged)

            if self.args.name not in conf["remote"]:
                conf["remote"][self.args.name] = {}
            section = conf["remote"][self.args.name]
            if self.args.unset:
                section.pop(self.args.option, None)
            else:
                section[self.args.option] = self.args.value
        return 0

class CmdRemoteDefault(CmdRemote):
    def run(self):
        from dvc.config import ConfigError

        if self.args.name is None and not self.args.unset:
            conf = self.config.read(self.args.level)
            try:
                ui.write(conf["core"]["remote"])
            except KeyError:
                ui.write("No default remote set")
                return 1
        else:
            with self.config.edit(self.args.level) as conf:
                if self.args.unset:
                    conf["core"].pop("remote", None)
                else:
                    merged_conf = self.config.load_config_to_level(
                        self.args.level
                    )
                    if (
                        self.args.name in conf["remote"]
                        or self.args.name in merged_conf["remote"]
                    ):
                        conf["core"]["remote"] = self.args.name
                    else:
                        raise ConfigError(
                            "default remote must be present in remote list."
                        )
        return 0

class CmdRemoteList(CmdRemote):
    def run(self):
        conf = self.config.read(self.args.level)
        for name, conf in conf["remote"].items():
            ui.write(name, conf["url"], sep="\t")
        return 0

class CmdRemoteRename(CmdRemote):
    def _rename_default(self, conf):
        if conf["core"].get("remote") == self.args.name:
            conf["core"]["remote"] = self.args.new

    def run(self):
        from dvc.config import ConfigError

        all_config = self.config.load_config_to_level(None)
        if self.args.new in all_config.get("remote", {}):
            raise ConfigError(
                "Rename failed. Remote name '{}' already exists.".format(
                    {self.args.new}
                )
            )

        with self.config.edit(self.args.level) as conf:
            self._check_exists(conf)
            conf["remote"][self.args.new] = conf["remote"][self.args.name]
            del conf["remote"][self.args.name]
            self._rename_default(conf)

        up_to_level = self.args.level or "repo"
        for level in reversed(self.config.LEVELS):
            if level == up_to_level:
                break
            with self.config.edit(level) as level_conf:
                self._rename_default(level_conf)

        return 0

def add_parser(subparsers, parent_parser):
    from dvc.command.config import parent_config_parser

    REMOTE_HELP = "Set up and manage data remotes."
    remote_parser = subparsers.add_parser(
        "remote",
        parents=[parent_parser],
        description=append_doc_link(REMOTE_HELP, "remote"),
        help=REMOTE_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )

    remote_subparsers = remote_parser.add_subparsers(
        dest="cmd",
        help="Use `dvc remote CMD --help` for " "command-specific help.",
    )

    fix_subparsers(remote_subparsers)

    REMOTE_ADD_HELP = "Add a new data remote."
    remote_add_parser = remote_subparsers.add_parser(
        "add",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_ADD_HELP, "remote/add"),
        help=REMOTE_ADD_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_add_parser.add_argument("name", help="Name of the remote")
    remote_add_parser.add_argument(
        "url",
        help="Remote location. See full list of supported URLs at {}".format(
            format_link("https://man.dvc.org/remote")
        ),
    )
    remote_add_parser.add_argument(
        "-d",
        "--default",
        action="store_true",
        default=False,
        help="Set as default remote.",
    )
    remote_add_parser.add_argument(
        "-f",
        "--force",
        action="store_true",
        default=False,
        help="Force overwriting existing configs",
    )
    remote_add_parser.set_defaults(func=CmdRemoteAdd)

    REMOTE_DEFAULT_HELP = "Set/unset the default data remote."
    remote_default_parser = remote_subparsers.add_parser(
        "default",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_DEFAULT_HELP, "remote/default"),
        help=REMOTE_DEFAULT_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_default_parser.add_argument(
        "name", nargs="?", help="Name of the remote"
    )
    remote_default_parser.add_argument(
        "-u",
        "--unset",
        action="store_true",
        default=False,
        help="Unset default remote.",
    )
    remote_default_parser.set_defaults(func=CmdRemoteDefault)

    REMOTE_MODIFY_HELP = "Modify the configuration of a data remote."
    remote_modify_parser = remote_subparsers.add_parser(
        "modify",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_MODIFY_HELP, "remote/modify"),
        help=REMOTE_MODIFY_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_modify_parser.add_argument("name", help="Name of the remote")
    remote_modify_parser.add_argument(
        "option", help="Name of the option to modify."
    )
    remote_modify_parser.add_argument(
        "value", nargs="?", help="(optional) Value of the option."
    )
    remote_modify_parser.add_argument(
        "-u",
        "--unset",
        default=False,
        action="store_true",
        help="Unset option.",
    )
    remote_modify_parser.set_defaults(func=CmdRemoteModify)

    REMOTE_LIST_HELP = "List all available data remotes."
    remote_list_parser = remote_subparsers.add_parser(
        "list",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_LIST_HELP, "remote/list"),
        help=REMOTE_LIST_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_list_parser.set_defaults(func=CmdRemoteList)

    REMOTE_REMOVE_HELP = "Remove a data remote."
    remote_remove_parser = remote_subparsers.add_parser(
        "remove",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
        help=REMOTE_REMOVE_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_remove_parser.add_argument(
        "name", help="Name of the remote to remove."
    )
    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
    REMOTE_RENAME_HELP = "Rename a DVC remote"
    remote_rename_parser = remote_subparsers.add_parser(
        "rename",
        parents=[parent_config_parser, parent_parser],
        description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
        help=REMOTE_RENAME_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    remote_rename_parser.add_argument("name", help="Remote to be renamed")
    remote_rename_parser.add_argument("new", help="New name of the remote")
    remote_rename_parser.set_defaults(func=CmdRemoteRename)
<|endoftext|>


<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import
from __future__ import unicode_literals

from collections import OrderedDict
import ipaddress
import json
import re

import typepy

import pyparsing as pp

from .._const import Tc
from .._logger import logger
from ._interface import AbstractParser

class TcFilterParser(AbstractParser):

    class FilterMatchIdIpv4(object):
        INCOMING_NETWORK = 12
        OUTGOING_NETWORK = 16
        PORT = 20

    class FilterMatchIdIpv6(object):
        INCOMING_NETWORK_LIST = [8, 12, 16, 20]
        OUTGOING_NETWORK_LIST = [24, 28, 32, 36]
        PORT = 40

    __FILTER_FLOWID_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("flowid", include=True) +
        pp.Word(pp.hexnums + ":")
    )
    __FILTER_PROTOCOL_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("protocol", include=True) +
        pp.Word(pp.alphanums)
    )
    __FILTER_PRIORITY_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("pref", include=True) +
        pp.Word(pp.nums))
    __FILTER_ID_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("fh", include=True) +
        pp.Word(pp.hexnums + ":")
    )
    __FILTER_MATCH_PATTERN = (
        pp.Literal("match") +
        pp.Word(pp.alphanums + "/") +
        pp.Literal("at") +
        pp.Word(pp.nums)
    )
    __FILTER_MANGLE_MARK_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("handle", include=True) +
        pp.Word(pp.hexnums) +
        pp.SkipTo("classid", include=True) +
        pp.Word(pp.hexnums + ":")
    )

    @property
    def protocol(self):
        return self.__protocol

    @property
    def _tc_subcommand(self):
        return Tc.Subcommand.FILTER

    def __init__(self, con, ip_version):
        super(TcFilterParser, self).__init__()

        self.__con = con
        self.__ip_version = ip_version
        self.__buffer = None
        self.__parse_idx = 0

        self.__protocol = None

        self._clear()

    def parse(self, device, text):
        self._clear()

        if typepy.is_null_string(text):
            return []

        filter_data_matrix = []
        self.__buffer = self._to_unicode(text).splitlines()
        self.__parse_idx = 0

        while self.__parse_idx < len(self.__buffer):
            line = self._to_unicode(self.__buffer[self.__parse_idx].strip())
            self.__parse_idx += 1

            if typepy.is_null_string(line):
                continue

            self.__device = device

            try:
                self.__parse_mangle_mark(line)
            except pp.ParseException:
                logger.debug("failed to parse mangle: {}".format(line))
            else:
                filter_data_matrix.append({
                    Tc.Param.DEVICE: self.__device,
                    Tc.Param.CLASS_ID: self.__classid,
                    Tc.Param.HANDLE: self.__handle,
                })
                self._clear()
                continue

            tc_filter = self.__get_filter()

            try:
                self.__parse_flow_id(line)
                self.__parse_protocol(line)
                self.__parse_priority(line)
                self.__parse_filter_id(line)

                if tc_filter.get(Tc.Param.FLOW_ID):
                    logger.debug("store filter: {}".format(tc_filter))
                    filter_data_matrix.append(tc_filter)
                    self._clear()

                    self.__parse_flow_id(line)
                    self.__parse_protocol(line)
                    self.__parse_priority(line)
                    self.__parse_filter_id(line)

                continue
            except pp.ParseException:
                logger.debug("failed to parse flow id: {}".format(line))

            try:
                if self.__ip_version == 4:
                    self.__parse_filter_ipv4(line)
                elif self.__ip_version == 6:
                    self.__parse_filter_ipv6(line)
                else:
                    raise ValueError(
                        "unknown ip version: {}".format(self.__ip_version))
            except pp.ParseException:
                logger.debug("failed to parse filter: {}".format(line))

        if self.__flow_id:
            filter_data_matrix.append(self.__get_filter())

        if filter_data_matrix:
            self.__con.create_table_from_data_matrix(
                table_name=self._tc_subcommand,
                attr_name_list=list(self.__get_filter()),
                data_matrix=filter_data_matrix)

        logger.debug("tc {:s} parse result: {}".format(
            self._tc_subcommand, json.dumps(filter_data_matrix, indent=4)))

        return filter_data_matrix

    def parse_incoming_device(self, text):
        if typepy.is_null_string(text):
            return None

        match = re.search(
            "Egress Redirect to device ifb[\d]+",
            self._to_unicode(text), re.MULTILINE)
        if match is None:
            return None

        return re.search("ifb[\d]+", match.group()).group()

    def _clear(self):
        self.__device = None
        self.__filter_id = None
        self.__flow_id = None
        self.__protocol = None
        self.__priority = None
        self.__filter_src_network = None
        self.__filter_dst_network = None
        self.__filter_src_port = None
        self.__filter_dst_port = None

        self.__handle = None
        self.__classid = None

    def __get_filter(self):
        tc_filter = OrderedDict()
        tc_filter[Tc.Param.DEVICE] = self.__device
        tc_filter[Tc.Param.FILTER_ID] = self.__filter_id
        tc_filter[Tc.Param.FLOW_ID] = self.__flow_id
        tc_filter[Tc.Param.PROTOCOL] = self.protocol
        tc_filter[Tc.Param.PRIORITY] = self.__priority
        tc_filter[Tc.Param.SRC_NETWORK] = self.__filter_src_network
        tc_filter[Tc.Param.DST_NETWORK] = self.__filter_dst_network
        tc_filter[Tc.Param.SRC_PORT] = self.__filter_src_port
        tc_filter[Tc.Param.DST_PORT] = self.__filter_dst_port

        return tc_filter

    def __parse_flow_id(self, line):
        parsed_list = self.__FILTER_FLOWID_PATTERN.parseString(line)
        self.__flow_id = parsed_list[-1]
        logger.debug("succeed to parse flow id: flow-id={}, line={}".format(
            self.__flow_id, line))

    def __parse_protocol(self, line):
        parsed_list = self.__FILTER_PROTOCOL_PATTERN.parseString(line)
        self.__protocol = parsed_list[-1]
        logger.debug("succeed to parse protocol: protocol={}, line={}".format(
            self.__protocol, line))

    def __parse_priority(self, line):
        parsed_list = self.__FILTER_PRIORITY_PATTERN.parseString(line)
        self.__priority = int(parsed_list[-1])
        logger.debug("succeed to parse priority: priority={}, line={}".format(
            self.__priority, line))

    def __parse_filter_id(self, line):
        parsed_list = self.__FILTER_ID_PATTERN.parseString(line)
        self.__filter_id = parsed_list[-1]
        logger.debug("succeed to parse filter id: filter-id={}, line={}".format(
            self.__filter_id, line))

    def __parse_mangle_mark(self, line):
        parsed_list = self.__FILTER_MANGLE_MARK_PATTERN.parseString(line)
        self.__classid = parsed_list[-1]
        self.__handle = int("0" + parsed_list[-3], 16)
        logger.debug(
            "succeed to parse mangle mark: "
            "classid={}, handle={}, line={}".format(
                self.__classid, self.__handle, line))

    def __parse_filter_ip_line(self, line):
        parsed_list = self.__FILTER_MATCH_PATTERN.parseString(line)
        value_hex, mask_hex = parsed_list[1].split("/")
        match_id = int(parsed_list[3])

        return (value_hex, mask_hex, match_id)

    def __parse_filter_ipv4_network(self, value_hex, mask_hex, match_id):
        ipaddr = ".".join([
            str(int(value_hex[i: i + 2], 16))
            for i in range(0, len(value_hex), 2)
        ])
        netmask = bin(int(mask_hex, 16)).count("1")
        network = "{:s}/{:d}".format(ipaddr, netmask)

        if match_id == self.FilterMatchIdIpv4.INCOMING_NETWORK:
            self.__filter_src_network = network
        elif match_id == self.FilterMatchIdIpv4.OUTGOING_NETWORK:
            self.__filter_dst_network = network
        else:
            logger.warn("unknown match id: {}".format(match_id))

    def __parse_filter_ipv6_network(self, value_hex, mask_hex, match_id):
        from collections import namedtuple

        Ipv6Entry = namedtuple("Ipv6Entry", "match_id octet_list mask_hex")

        OCTET_LEN = 4
        ipv6_entry_list = [
            Ipv6Entry(
                match_id=match_id,
                octet_list=[
                    value_hex[i: i + OCTET_LEN]
                    for i in range(0, len(value_hex), OCTET_LEN)
                ],
                mask_hex=mask_hex),
        ]

        while True:
            try:
                line = self.__buffer[self.__parse_idx].strip()
            except IndexError:
                break

            try:
                value_hex, mask_hex, match_id = self.__parse_filter_ip_line(
                    line)
            except pp.ParseException:
                break

            if (match_id in self.FilterMatchIdIpv6.INCOMING_NETWORK_LIST or
                    match_id in self.FilterMatchIdIpv6.OUTGOING_NETWORK_LIST):
                ipv6_entry_list.append(
                    Ipv6Entry(
                        match_id=match_id,
                        octet_list=[
                            value_hex[i: i + OCTET_LEN]
                            for i in range(0, len(value_hex), OCTET_LEN)
                        ],
                        mask_hex=mask_hex))
            else:
                break

            self.__parse_idx += 1

        src_octet_list = []
        dst_octet_list = []
        src_netmask = 0
        dst_netmask = 0

        for ipv6_entry in ipv6_entry_list:
            part_netmask = bin(int(ipv6_entry.mask_hex, 16)).count("1")

            if ipv6_entry.match_id in self.FilterMatchIdIpv6.INCOMING_NETWORK_LIST:
                src_octet_list.extend(ipv6_entry.octet_list)
                src_netmask += part_netmask
            elif ipv6_entry.match_id in self.FilterMatchIdIpv6.OUTGOING_NETWORK_LIST:
                dst_octet_list.extend(ipv6_entry.octet_list)
                dst_netmask += part_netmask
            else:
                raise ValueError(
                    "unexpected ipv6 entry: {}".format(ipv6_entry))

        while len(src_octet_list) < 8:
            src_octet_list.append("0000")
        while len(dst_octet_list) < 8:
            dst_octet_list.append("0000")

        self.__filter_dst_network = ipaddress.IPv6Network("{:s}/{:d}".format(
            ":".join(dst_octet_list), dst_netmask)).compressed
        self.__filter_src_network = ipaddress.IPv6Network("{:s}/{:d}".format(
            ":".join(src_octet_list), src_netmask)).compressed

    def __parse_filter_port(self, value_hex):
        # Port filter consists eight hex digits.
        # The upper-half represents source port filter and
        # the bottom-half represents destination port filter.

        if len(value_hex) != 8:
            raise ValueError("invalid port filter value: {}".format(value_hex))

        src_port_hex = value_hex[:4]
        dst_port_hex = value_hex[4:]

        logger.debug(
            "parse ipv4 port: src-port-hex={}, dst-port-hex={}".format(
                src_port_hex, dst_port_hex))

        src_port_decimal = int(src_port_hex, 16)
        self.__filter_src_port = (
            src_port_decimal if src_port_decimal != 0 else None)

        dst_port_decimal = int(dst_port_hex, 16)
        self.__filter_dst_port = (
            dst_port_decimal if dst_port_decimal != 0 else None)

    def __parse_filter_ipv4(self, line):
        value_hex, mask_hex, match_id = self.__parse_filter_ip_line(line)

        if match_id in [
                self.FilterMatchIdIpv4.INCOMING_NETWORK,
                self.FilterMatchIdIpv4.OUTGOING_NETWORK,
        ]:
            self.__parse_filter_ipv4_network(value_hex, mask_hex, match_id)
        elif match_id == self.FilterMatchIdIpv4.PORT:
            self.__parse_filter_port(value_hex)
        elif match_id in (self.FilterMatchIdIpv6.INCOMING_NETWORK_LIST +
                          self.FilterMatchIdIpv6.OUTGOING_NETWORK_LIST +
                          [self.FilterMatchIdIpv6.PORT]):
            logger.warn(
                "unknown match id for an IPv4 filter: "
                "might be an IPv6 filter. try to use --ipv6 option. "
                "(id={})".format(match_id))
            return
        else:
            logger.debug("unknown match id: {}".format(match_id))
            return

        logger.debug(
            "succeed to parse ipv4 filter: " + ", ".join([
                "src_network={}".format(self.__filter_src_network),
                "dst_network={}".format(self.__filter_dst_network),
                "src_port={}".format(self.__filter_src_port),
                "dst_port={}".format(self.__filter_dst_port),
                "line={}".format(line)
            ]))

    def __parse_filter_ipv6(self, line):
        value_hex, mask_hex, match_id = self.__parse_filter_ip_line(line)

        if (match_id in self.FilterMatchIdIpv6.INCOMING_NETWORK_LIST or
                match_id in self.FilterMatchIdIpv6.OUTGOING_NETWORK_LIST):
            self.__parse_filter_ipv6_network(value_hex, mask_hex, match_id)
        elif match_id == self.FilterMatchIdIpv6.PORT:
            self.__parse_filter_port(value_hex)
        else:
            logger.debug("unknown match id: {}".format(match_id))
            return

        logger.debug(
            "succeed to parse ipv6 filter: " + ", ".join([
                "src_network={}".format(self.__filter_src_network),
                "dst_network={}".format(self.__filter_dst_network),
                "src_port={}".format(self.__filter_src_port),
                "dst_port={}".format(self.__filter_dst_port),
                "line={}".format(line)
            ]))
<commit_msg>Improve filter parse result consistency<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import
from __future__ import unicode_literals

from collections import OrderedDict
import ipaddress
import json
import re

import typepy

import pyparsing as pp

from .._common import sanitize_network
from .._const import Tc
from .._logger import logger
from ._interface import AbstractParser

class TcFilterParser(AbstractParser):

    class FilterMatchIdIpv4(object):
        INCOMING_NETWORK = 12
        OUTGOING_NETWORK = 16
        PORT = 20

    class FilterMatchIdIpv6(object):
        INCOMING_NETWORK_LIST = [8, 12, 16, 20]
        OUTGOING_NETWORK_LIST = [24, 28, 32, 36]
        PORT = 40

    __FILTER_FLOWID_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("flowid", include=True) +
        pp.Word(pp.hexnums + ":")
    )
    __FILTER_PROTOCOL_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("protocol", include=True) +
        pp.Word(pp.alphanums)
    )
    __FILTER_PRIORITY_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("pref", include=True) +
        pp.Word(pp.nums))
    __FILTER_ID_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("fh", include=True) +
        pp.Word(pp.hexnums + ":")
    )
    __FILTER_MATCH_PATTERN = (
        pp.Literal("match") +
        pp.Word(pp.alphanums + "/") +
        pp.Literal("at") +
        pp.Word(pp.nums)
    )
    __FILTER_MANGLE_MARK_PATTERN = (
        pp.Literal("filter parent") +
        pp.SkipTo("handle", include=True) +
        pp.Word(pp.hexnums) +
        pp.SkipTo("classid", include=True) +
        pp.Word(pp.hexnums + ":")
    )

    @property
    def protocol(self):
        return self.__protocol

    @property
    def _tc_subcommand(self):
        return Tc.Subcommand.FILTER

    def __init__(self, con, ip_version):
        super(TcFilterParser, self).__init__()

        self.__con = con
        self.__ip_version = ip_version
        self.__buffer = None
        self.__parse_idx = 0

        self.__protocol = None

        self._clear()

    def parse(self, device, text):
        self._clear()

        if typepy.is_null_string(text):
            return []

        filter_data_matrix = []
        self.__buffer = self._to_unicode(text).splitlines()
        self.__parse_idx = 0

        while self.__parse_idx < len(self.__buffer):
            line = self._to_unicode(self.__buffer[self.__parse_idx].strip())
            self.__parse_idx += 1

            if typepy.is_null_string(line):
                continue

            self.__device = device

            try:
                self.__parse_mangle_mark(line)
            except pp.ParseException:
                logger.debug("failed to parse mangle: {}".format(line))
            else:
                filter_data_matrix.append({
                    Tc.Param.DEVICE: self.__device,
                    Tc.Param.CLASS_ID: self.__classid,
                    Tc.Param.HANDLE: self.__handle,
                })
                self._clear()
                continue

            tc_filter = self.__get_filter()

            try:
                self.__parse_flow_id(line)
                self.__parse_protocol(line)
                self.__parse_priority(line)
                self.__parse_filter_id(line)

                if tc_filter.get(Tc.Param.FLOW_ID):
                    logger.debug("store filter: {}".format(tc_filter))
                    filter_data_matrix.append(tc_filter)
                    self._clear()

                    self.__parse_flow_id(line)
                    self.__parse_protocol(line)
                    self.__parse_priority(line)
                    self.__parse_filter_id(line)

                continue
            except pp.ParseException:
                logger.debug("failed to parse flow id: {}".format(line))

            try:
                if self.__ip_version == 4:
                    self.__parse_filter_ipv4(line)
                elif self.__ip_version == 6:
                    self.__parse_filter_ipv6(line)
                else:
                    raise ValueError(
                        "unknown ip version: {}".format(self.__ip_version))
            except pp.ParseException:
                logger.debug("failed to parse filter: {}".format(line))

        if self.__flow_id:
            filter_data_matrix.append(self.__get_filter())

        if filter_data_matrix:
            self.__con.create_table_from_data_matrix(
                table_name=self._tc_subcommand,
                attr_name_list=list(self.__get_filter()),
                data_matrix=filter_data_matrix)

        logger.debug("tc {:s} parse result: {}".format(
            self._tc_subcommand, json.dumps(filter_data_matrix, indent=4)))

        return filter_data_matrix

    def parse_incoming_device(self, text):
        if typepy.is_null_string(text):
            return None

        match = re.search(
            "Egress Redirect to device ifb[\d]+",
            self._to_unicode(text), re.MULTILINE)
        if match is None:
            return None

        return re.search("ifb[\d]+", match.group()).group()

    def _clear(self):
        self.__device = None
        self.__filter_id = None
        self.__flow_id = None
        self.__protocol = None
        self.__priority = None
        self.__filter_src_network = None
        self.__filter_dst_network = None
        self.__filter_src_port = None
        self.__filter_dst_port = None

        self.__handle = None
        self.__classid = None

    def __get_filter(self):
        tc_filter = OrderedDict()
        tc_filter[Tc.Param.DEVICE] = self.__device
        tc_filter[Tc.Param.FILTER_ID] = self.__filter_id
        tc_filter[Tc.Param.FLOW_ID] = self.__flow_id
        tc_filter[Tc.Param.PROTOCOL] = self.protocol
        tc_filter[Tc.Param.PRIORITY] = self.__priority
        tc_filter[Tc.Param.SRC_NETWORK] = sanitize_network(
            self.__filter_src_network, self.__ip_version)
        tc_filter[Tc.Param.DST_NETWORK] = sanitize_network(
            self.__filter_dst_network, self.__ip_version)
        tc_filter[Tc.Param.SRC_PORT] = self.__filter_src_port
        tc_filter[Tc.Param.DST_PORT] = self.__filter_dst_port

        return tc_filter

    def __parse_flow_id(self, line):
        parsed_list = self.__FILTER_FLOWID_PATTERN.parseString(line)
        self.__flow_id = parsed_list[-1]
        logger.debug("succeed to parse flow id: flow-id={}, line={}".format(
            self.__flow_id, line))

    def __parse_protocol(self, line):
        parsed_list = self.__FILTER_PROTOCOL_PATTERN.parseString(line)
        self.__protocol = parsed_list[-1]
        logger.debug("succeed to parse protocol: protocol={}, line={}".format(
            self.__protocol, line))

    def __parse_priority(self, line):
        parsed_list = self.__FILTER_PRIORITY_PATTERN.parseString(line)
        self.__priority = int(parsed_list[-1])
        logger.debug("succeed to parse priority: priority={}, line={}".format(
            self.__priority, line))

    def __parse_filter_id(self, line):
        parsed_list = self.__FILTER_ID_PATTERN.parseString(line)
        self.__filter_id = parsed_list[-1]
        logger.debug("succeed to parse filter id: filter-id={}, line={}".format(
            self.__filter_id, line))

    def __parse_mangle_mark(self, line):
        parsed_list = self.__FILTER_MANGLE_MARK_PATTERN.parseString(line)
        self.__classid = parsed_list[-1]
        self.__handle = int("0" + parsed_list[-3], 16)
        logger.debug(
            "succeed to parse mangle mark: "
            "classid={}, handle={}, line={}".format(
                self.__classid, self.__handle, line))

    def __parse_filter_ip_line(self, line):
        parsed_list = self.__FILTER_MATCH_PATTERN.parseString(line)
        value_hex, mask_hex = parsed_list[1].split("/")
        match_id = int(parsed_list[3])

        return (value_hex, mask_hex, match_id)

    def __parse_filter_ipv4_network(self, value_hex, mask_hex, match_id):
        ipaddr = ".".join([
            str(int(value_hex[i: i + 2], 16))
            for i in range(0, len(value_hex), 2)
        ])
        netmask = bin(int(mask_hex, 16)).count("1")
        network = "{:s}/{:d}".format(ipaddr, netmask)

        if match_id == self.FilterMatchIdIpv4.INCOMING_NETWORK:
            self.__filter_src_network = network
        elif match_id == self.FilterMatchIdIpv4.OUTGOING_NETWORK:
            self.__filter_dst_network = network
        else:
            logger.warn("unknown match id: {}".format(match_id))

    def __parse_filter_ipv6_network(self, value_hex, mask_hex, match_id):
        from collections import namedtuple

        Ipv6Entry = namedtuple("Ipv6Entry", "match_id octet_list mask_hex")

        OCTET_LEN = 4
        ipv6_entry_list = [
            Ipv6Entry(
                match_id=match_id,
                octet_list=[
                    value_hex[i: i + OCTET_LEN]
                    for i in range(0, len(value_hex), OCTET_LEN)
                ],
                mask_hex=mask_hex),
        ]

        while True:
            try:
                line = self.__buffer[self.__parse_idx].strip()
            except IndexError:
                break

            try:
                value_hex, mask_hex, match_id = self.__parse_filter_ip_line(
                    line)
            except pp.ParseException:
                break

            if (match_id in self.FilterMatchIdIpv6.INCOMING_NETWORK_LIST or
                    match_id in self.FilterMatchIdIpv6.OUTGOING_NETWORK_LIST):
                ipv6_entry_list.append(
                    Ipv6Entry(
                        match_id=match_id,
                        octet_list=[
                            value_hex[i: i + OCTET_LEN]
                            for i in range(0, len(value_hex), OCTET_LEN)
                        ],
                        mask_hex=mask_hex))
            else:
                break

            self.__parse_idx += 1

        src_octet_list = []
        dst_octet_list = []
        src_netmask = 0
        dst_netmask = 0

        for ipv6_entry in ipv6_entry_list:
            part_netmask = bin(int(ipv6_entry.mask_hex, 16)).count("1")

            if ipv6_entry.match_id in self.FilterMatchIdIpv6.INCOMING_NETWORK_LIST:
                src_octet_list.extend(ipv6_entry.octet_list)
                src_netmask += part_netmask
            elif ipv6_entry.match_id in self.FilterMatchIdIpv6.OUTGOING_NETWORK_LIST:
                dst_octet_list.extend(ipv6_entry.octet_list)
                dst_netmask += part_netmask
            else:
                raise ValueError(
                    "unexpected ipv6 entry: {}".format(ipv6_entry))

        while len(src_octet_list) < 8:
            src_octet_list.append("0000")
        while len(dst_octet_list) < 8:
            dst_octet_list.append("0000")

        self.__filter_dst_network = ipaddress.IPv6Network("{:s}/{:d}".format(
            ":".join(dst_octet_list), dst_netmask)).compressed
        self.__filter_src_network = ipaddress.IPv6Network("{:s}/{:d}".format(
            ":".join(src_octet_list), src_netmask)).compressed

    def __parse_filter_port(self, value_hex):
        # Port filter consists eight hex digits.
        # The upper-half represents source port filter and
        # the bottom-half represents destination port filter.

        if len(value_hex) != 8:
            raise ValueError("invalid port filter value: {}".format(value_hex))

        src_port_hex = value_hex[:4]
        dst_port_hex = value_hex[4:]

        logger.debug(
            "parse ipv4 port: src-port-hex={}, dst-port-hex={}".format(
                src_port_hex, dst_port_hex))

        src_port_decimal = int(src_port_hex, 16)
        self.__filter_src_port = (
            src_port_decimal if src_port_decimal != 0 else None)

        dst_port_decimal = int(dst_port_hex, 16)
        self.__filter_dst_port = (
            dst_port_decimal if dst_port_decimal != 0 else None)

    def __parse_filter_ipv4(self, line):
        value_hex, mask_hex, match_id = self.__parse_filter_ip_line(line)

        if match_id in [
                self.FilterMatchIdIpv4.INCOMING_NETWORK,
                self.FilterMatchIdIpv4.OUTGOING_NETWORK,
        ]:
            self.__parse_filter_ipv4_network(value_hex, mask_hex, match_id)
        elif match_id == self.FilterMatchIdIpv4.PORT:
            self.__parse_filter_port(value_hex)
        elif match_id in (self.FilterMatchIdIpv6.INCOMING_NETWORK_LIST +
                          self.FilterMatchIdIpv6.OUTGOING_NETWORK_LIST +
                          [self.FilterMatchIdIpv6.PORT]):
            logger.warn(
                "unknown match id for an IPv4 filter: "
                "might be an IPv6 filter. try to use --ipv6 option. "
                "(id={})".format(match_id))
            return
        else:
            logger.debug("unknown match id: {}".format(match_id))
            return

        logger.debug(
            "succeed to parse ipv4 filter: " + ", ".join([
                "src_network={}".format(self.__filter_src_network),
                "dst_network={}".format(self.__filter_dst_network),
                "src_port={}".format(self.__filter_src_port),
                "dst_port={}".format(self.__filter_dst_port),
                "line={}".format(line)
            ]))

    def __parse_filter_ipv6(self, line):
        value_hex, mask_hex, match_id = self.__parse_filter_ip_line(line)

        if (match_id in self.FilterMatchIdIpv6.INCOMING_NETWORK_LIST or
                match_id in self.FilterMatchIdIpv6.OUTGOING_NETWORK_LIST):
            self.__parse_filter_ipv6_network(value_hex, mask_hex, match_id)
        elif match_id == self.FilterMatchIdIpv6.PORT:
            self.__parse_filter_port(value_hex)
        else:
            logger.debug("unknown match id: {}".format(match_id))
            return

        logger.debug(
            "succeed to parse ipv6 filter: " + ", ".join([
                "src_network={}".format(self.__filter_src_network),
                "dst_network={}".format(self.__filter_dst_network),
                "src_port={}".format(self.__filter_src_port),
                "dst_port={}".format(self.__filter_dst_port),
                "line={}".format(line)
            ]))
<|endoftext|>


<commit_before>from django.shortcuts import get_object_or_404, render
from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_protect
from django.http import HttpResponseRedirect, HttpResponse
from django.http import Http404, HttpResponseBadRequest
from django.core.urlresolvers import reverse
from django.db.models import Count
from dwitter.models import Dweet
from django.contrib.auth.decorators import login_required
from django.utils import timezone
from functools import wraps
from datetime import datetime, timedelta
from math import log
import json

def epoch_seconds(date):
    epoch = datetime(2015,5,5)
    naive = date.replace(tzinfo=None)
    td = naive - epoch
    return td.days * 86400 + td.seconds + (float(td.microseconds) / 1000000)

def hot(likes, date):
    order = log(max(abs(likes), 1), 2)
    return round(order + epoch_seconds(date)/86400)

def ajax_login_required(view_func):
    @wraps(view_func)
    def wrapper(request, *args, **kwargs):
        if request.user.is_authenticated():
            return view_func(request, *args, **kwargs)
        json_resp = json.dumps({'not_authenticated': True})
        return HttpResponse(json_resp, content_type='application/json')
    return wrapper

def feed(request, page_nr, sort):
    page = int(page_nr)
    dweets_per_page = 10
    first = (page - 1) * dweets_per_page
    last = page * dweets_per_page
    dweet_count = Dweet.objects.count()

    if(first < 0 or first > dweet_count):
        raise Http404("No such page")
    if(last >= dweet_count):
        last = dweet_count

    if(sort == "top"):
        dweet_list = (Dweet.objects.annotate(num_likes=Count('likes'))
                      .order_by('-num_likes', '-posted')[first:last])

        next_url = reverse('top_feed_page', kwargs={'page_nr': page + 1})
        prev_url = reverse('top_feed_page', kwargs={'page_nr': page - 1})
    elif (sort == "new"):
        dweet_list = Dweet.objects.order_by('-posted')[first:last]
        next_url = reverse('new_feed_page', kwargs={'page_nr': page + 1})
        prev_url = reverse('new_feed_page', kwargs={'page_nr': page - 1})
    elif (sort == "hot"):
        dweet_list = (Dweet.objects.annotate(num_likes=Count('likes'))
                      .order_by('-hotscore', '-posted')[first:last])
        next_url = reverse('hot_feed_page', kwargs={'page_nr': page + 1})
        prev_url = reverse('hot_feed_page', kwargs={'page_nr': page - 1})
    else:
        raise Http404("No such sorting method " + sort)

    context = {'dweet_list': dweet_list,
               'header_title': 'Dwitter',
               'feed_type': 'all',
               'page_nr': page,
               'next_url': next_url,
               'prev_url': prev_url,
               'sort': sort,
               }
    return render(request, 'feed/feed.html', context)

def dweet_show(request, dweet_id):
    dweet = get_object_or_404(Dweet, id=dweet_id)

    context = {
               'dweet': dweet,
               'header_title': 'Dwitter',
               }
    return render(request, 'feed/permalink.html', context)

@login_required
def dweet(request):
    code = request.POST['code']

    if(len(code.replace('\r\n', '\n')) > 140):
        return HttpResponseBadRequest("Dweet code too long! Code: " + code)

    d = Dweet(code=code,
              author=request.user,
              posted=timezone.now())

    d.likes.add(request.user)
    d.hotscore = hot(1, dweet.posted)

    d.save()
    return HttpResponseRedirect(reverse('root'))

@login_required
def dweet_reply(request, dweet_id):
    reply_to = get_object_or_404(Dweet, id=dweet_id)
    d = Dweet(code=request.POST['code'],
              reply_to=reply_to,
              author=request.user,
              posted=timezone.now())
    d.save()
    return HttpResponseRedirect(reverse('root'))

@csrf_protect
@login_required
@require_POST
def dweet_delete(request, dweet_id):
    dweet = get_object_or_404(Dweet, id=dweet_id)
    if(request.user == dweet.author or request.user.is_staff):
        dweet.delete()
        return HttpResponseRedirect(reverse('root'))

    return HttpResponse("Not authorized to delete the dweet.")

@ajax_login_required
def like(request, dweet_id):
    dweet = get_object_or_404(Dweet, id=dweet_id)

    if(dweet.likes.filter(id=request.user.id).exists()):
        liked = False
        dweet.likes.remove(request.user)
    else:
        liked = True
        dweet.likes.add(request.user)

    likes = dweet.likes.count()
    dweet.hotscore = hot(likes, dweet.posted)
    dweet.save()
    json_resp = json.dumps({'likes': likes, 'liked': liked})
    return HttpResponse(json_resp, content_type='application/json')
<commit_msg>Bugfix on dweet creation<commit_after>from django.shortcuts import get_object_or_404, render
from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_protect
from django.http import HttpResponseRedirect, HttpResponse
from django.http import Http404, HttpResponseBadRequest
from django.core.urlresolvers import reverse
from django.db.models import Count
from dwitter.models import Dweet
from django.contrib.auth.decorators import login_required
from django.utils import timezone
from functools import wraps
from datetime import datetime, timedelta
from math import log
import json

def epoch_seconds(date):
    epoch = datetime(2015,5,5)
    naive = date.replace(tzinfo=None)
    td = naive - epoch
    return td.days * 86400 + td.seconds + (float(td.microseconds) / 1000000)

def hot(likes, date):
    order = log(max(abs(likes), 1), 2)
    return round(order + epoch_seconds(date)/86400)

def ajax_login_required(view_func):
    @wraps(view_func)
    def wrapper(request, *args, **kwargs):
        if request.user.is_authenticated():
            return view_func(request, *args, **kwargs)
        json_resp = json.dumps({'not_authenticated': True})
        return HttpResponse(json_resp, content_type='application/json')
    return wrapper

def feed(request, page_nr, sort):
    page = int(page_nr)
    dweets_per_page = 10
    first = (page - 1) * dweets_per_page
    last = page * dweets_per_page
    dweet_count = Dweet.objects.count()

    if(first < 0 or first > dweet_count):
        raise Http404("No such page")
    if(last >= dweet_count):
        last = dweet_count

    if(sort == "top"):
        dweet_list = (Dweet.objects.annotate(num_likes=Count('likes'))
                      .order_by('-num_likes', '-posted')[first:last])

        next_url = reverse('top_feed_page', kwargs={'page_nr': page + 1})
        prev_url = reverse('top_feed_page', kwargs={'page_nr': page - 1})
    elif (sort == "new"):
        dweet_list = Dweet.objects.order_by('-posted')[first:last]
        next_url = reverse('new_feed_page', kwargs={'page_nr': page + 1})
        prev_url = reverse('new_feed_page', kwargs={'page_nr': page - 1})
    elif (sort == "hot"):
        dweet_list = (Dweet.objects.annotate(num_likes=Count('likes'))
                      .order_by('-hotscore', '-posted')[first:last])
        next_url = reverse('hot_feed_page', kwargs={'page_nr': page + 1})
        prev_url = reverse('hot_feed_page', kwargs={'page_nr': page - 1})
    else:
        raise Http404("No such sorting method " + sort)

    context = {'dweet_list': dweet_list,
               'header_title': 'Dwitter',
               'feed_type': 'all',
               'page_nr': page,
               'next_url': next_url,
               'prev_url': prev_url,
               'sort': sort,
               }
    return render(request, 'feed/feed.html', context)

def dweet_show(request, dweet_id):
    dweet = get_object_or_404(Dweet, id=dweet_id)

    context = {
               'dweet': dweet,
               'header_title': 'Dwitter',
               }
    return render(request, 'feed/permalink.html', context)

@login_required
def dweet(request):
    code = request.POST['code']

    if(len(code.replace('\r\n', '\n')) > 140):
        return HttpResponseBadRequest("Dweet code too long! Code: " + code)

    d = Dweet(code=code,
              author=request.user,
              posted=timezone.now())
    d.likes.add(d.author)
    d.hotscore = hot(1, d.posted)
    d.save()
    return HttpResponseRedirect(reverse('root'))

@login_required
def dweet_reply(request, dweet_id):
    reply_to = get_object_or_404(Dweet, id=dweet_id)
    d = Dweet(code=request.POST['code'],
              reply_to=reply_to,
              author=request.user,
              posted=timezone.now())
    d.save()
    return HttpResponseRedirect(reverse('root'))

@csrf_protect
@login_required
@require_POST
def dweet_delete(request, dweet_id):
    dweet = get_object_or_404(Dweet, id=dweet_id)
    if(request.user == dweet.author or request.user.is_staff):
        dweet.delete()
        return HttpResponseRedirect(reverse('root'))

    return HttpResponse("Not authorized to delete the dweet.")

@ajax_login_required
def like(request, dweet_id):
    dweet = get_object_or_404(Dweet, id=dweet_id)

    if(dweet.likes.filter(id=request.user.id).exists()):
        liked = False
        dweet.likes.remove(request.user)
    else:
        liked = True
        dweet.likes.add(request.user)

    likes = dweet.likes.count()
    dweet.hotscore = hot(likes, dweet.posted)
    dweet.save()
    json_resp = json.dumps({'likes': likes, 'liked': liked})
    return HttpResponse(json_resp, content_type='application/json')
<|endoftext|>


<commit_before>from bokeh.plotting import figure, curdoc, vplot,  hplot
from bokeh.models.widgets import Button, Toggle, Slider, VBoxForm, HBox, VBox, CheckboxGroup
from bokeh.driving import linear
import numpy as np

# Forward & reverse propagating waves. Use global variables to pass values into function.
def forward_wave():
    return np.exp(-alpha*(z - zmin)) * np.cos(twopi * (current_time - z_norm))
def reverse_wave():
    return (np.exp(-alpha*(z[-1] - zmin)) * reflection_coef) * \
           np.exp(alpha*z) * np.cos(twopi * (current_time + z_norm))

# Initializations
freq_Hz = 100e6
velocity_mps = 2e8
wavelength_m = velocity_mps / freq_Hz
twopi = 2.0*np.pi
n_samp_one_temporal_cycle = 30
alpha = 0.0
reflection_coef = 1.0
current_time = 0.0
zmin = -20
zmax = 0
numpnts = 500
periodic_callback_time_ms = 16
z = np.linspace(zmin,zmax,numpnts)
z_norm = z/wavelength_m
color_forward_wave = 'blue'
color_reverse_wave = 'red'
v1 = forward_wave()
v2 = reverse_wave()

ii = 0

# Set up plot
p = figure(plot_width=600, plot_height=400, x_range=(zmin,zmax), y_range=(-2.1,2.1), \
           title="Forward & Reverse Sinusoidal Voltages", \
           tools="pan,box_zoom,resize,save,reset")
l_forward = p.line(z, v1, line_width=2, color=color_forward_wave, line_alpha=0.5)
l_reverse = p.line(z, v2, line_width=2, color=color_reverse_wave, line_alpha=0.5)
p.xaxis.axis_label = "z (m)"
p.yaxis.axis_label = "Voltage (V)"
t1 = p.text(zmin+1.5, 1.0, text=['{} {}'.format(l_forward.glyph.line_color,l_reverse.glyph.line_color)],
            text_align="left", text_font_size="10pt")
#t2 = p.text(zmin+1.5, 1.0-0.08, text=['t = {} s'.format(current_time)], text_align="left", text_font_size="10pt")
t3 = p.text(zmin+1.5, 1.0-0.16, text=['Stopped'], text_align="left", text_font_size="10pt")

# Set up toggle button & callback function
def toggle_handler(active):
    if active:
        toggle.label = 'Stop'
        t3.data_source.data["text"] = ['Running']
        curdoc().add_periodic_callback(update, periodic_callback_time_ms)
    else:
        toggle.label = 'Start'
        t3.data_source.data["text"] = ['Stopped']
        curdoc().remove_periodic_callback(update)
toggle = Toggle(label="Start", type="success")
toggle.on_click(toggle_handler)
toggle.active = False

# Set up reset button
def reset_handler():
    global ii, current_time
    ii = 0
    current_time = 0
    l_forward.data_source.data["y"] = forward_wave()
    l_reverse.data_source.data["y"] = reverse_wave()
    #t2.data_source.data["text"] = ['t = {:.3f} s'.format(current_time)]
button_reset = Button(label="Reset", type="success")
button_reset.on_click(reset_handler)

# Set up checkboxes to show/hide forward & reverse propagating waves
def checkbox_group_handler(active):
    if not toggle.active:
        if 0 in checkbox_group.active:
            l_forward.glyph.line_alpha = 0.5
            #l_forward.glyph.line_color = color_forward_wave
        else:
            l_forward.glyph.line_alpha = 0.0
        if 1 in checkbox_group.active:
            l_reverse.glyph.line_alpha = 0.5
            #l_reverse.glyph.line_color = color_reverse_wave
        else:
            l_reverse.glyph.line_alpha = 0.0
        t1.data_source.data["text"] = ['{} {}'.format(l_forward.glyph.line_color,l_reverse.glyph.line_color)]
checkbox_group = CheckboxGroup(
        labels=["Forward Propagating Wave", "Reverse Propagating Wave"], active=[0, 1])
checkbox_group.on_click(checkbox_group_handler)

# Set up slider & callback function
def update_alpha(attrname, old, new):
    global alpha
    alpha = alpha_slider.value
    if not toggle.active:
        l_forward.data_source.data["y"] = forward_wave()
        l_reverse.data_source.data["y"] = reverse_wave()
alpha_slider = Slider(title="Alpha (1/m)", value=0.0, start=0.0, end=0.25, step=0.005)
alpha_slider.on_change('value', update_alpha)

# Set up layout
layout = hplot(p, VBox(toggle, button_reset, checkbox_group, alpha_slider, height=480), width=900)

# Create callback function for periodic callback
def update():
    global ii, current_time
    if toggle.active:
        ii += 1
        current_time = ii / n_samp_one_temporal_cycle
        l_forward.data_source.data["y"] = forward_wave()
        l_reverse.data_source.data["y"] = reverse_wave()
        #t2.data_source.data["text"] = ['t = {:.3f} s'.format(current_time)]
<commit_msg>fixed checkbox problem by setting to disabled when animation is running<commit_after>from bokeh.plotting import figure, curdoc, vplot,  hplot
from bokeh.models.widgets import Button, Toggle, Slider, VBoxForm, HBox, VBox, CheckboxGroup
from bokeh.driving import linear
import numpy as np

# Forward & reverse propagating waves. Use global variables to pass values into function.
def forward_wave():
    return np.exp(-alpha*(z - zmin)) * np.cos(twopi * (current_time - z_norm))
def reverse_wave():
    return (np.exp(-alpha*(z[-1] - zmin)) * reflection_coef) * \
           np.exp(alpha*z) * np.cos(twopi * (current_time + z_norm))

# Initializations
freq_Hz = 100e6
velocity_mps = 2e8
wavelength_m = velocity_mps / freq_Hz
twopi = 2.0*np.pi
n_samp_one_temporal_cycle = 30
alpha = 0.0
reflection_coef = 1.0
current_time = 0.0
zmin = -20
zmax = 0
numpnts = 500
periodic_callback_time_ms = 16
z = np.linspace(zmin,zmax,numpnts)
z_norm = z/wavelength_m
color_forward_wave = 'blue'
color_reverse_wave = 'red'
v1 = forward_wave()
v2 = reverse_wave()

ii = 0

# Set up plot
p = figure(plot_width=600, plot_height=400, x_range=(zmin,zmax), y_range=(-2.1,2.1), \
           title="Forward & Reverse Sinusoidal Voltages", \
           tools="pan,box_zoom,resize,save,reset")
l_forward = p.line(z, v1, line_width=2, color=color_forward_wave, line_alpha=0.5)
l_reverse = p.line(z, v2, line_width=2, color=color_reverse_wave, line_alpha=0.5)
p.xaxis.axis_label = "z (m)"
p.yaxis.axis_label = "Voltage (V)"
t1 = p.text(zmin+1.5, 1.0, text=['{} {}'.format(l_forward.glyph.line_alpha,l_reverse.glyph.line_alpha)],
            text_align="left", text_font_size="10pt")
#t2 = p.text(zmin+1.5, 1.0-0.08, text=['t = {} s'.format(current_time)], text_align="left", text_font_size="10pt")
t3 = p.text(zmin+1.5, 1.0-0.16, text=['Stopped'], text_align="left", text_font_size="10pt")

# Set up toggle button & callback function
def toggle_handler(active):
    if active:
        toggle.label = 'Stop'
        checkbox_group.disabled = True
        t3.data_source.data["text"] = ['Running']
        curdoc().add_periodic_callback(update, periodic_callback_time_ms)
    else:
        toggle.label = 'Start'
        checkbox_group.disabled = False
        t3.data_source.data["text"] = ['Stopped']
        curdoc().remove_periodic_callback(update)
toggle = Toggle(label="Start", type="success")
toggle.on_click(toggle_handler)
toggle.active = False

# Set up reset button
def reset_handler():
    global ii, current_time
    ii = 0
    current_time = 0
    l_forward.data_source.data["y"] = forward_wave()
    l_reverse.data_source.data["y"] = reverse_wave()
    #t2.data_source.data["text"] = ['t = {:.3f} s'.format(current_time)]
button_reset = Button(label="Reset", type="success")
button_reset.on_click(reset_handler)

# Set up checkboxes to show/hide forward & reverse propagating waves
def checkbox_group_handler(active):
    if 0 in active:
        l_forward.glyph.line_alpha = 0.5
        #l_forward.glyph.line_color = color_forward_wave
    else:
        l_forward.glyph.line_alpha = 0.0
    if 1 in active:
        l_reverse.glyph.line_alpha = 0.5
        #l_reverse.glyph.line_color = color_reverse_wave
    else:
        l_reverse.glyph.line_alpha = 0.0
    t1.data_source.data["text"] = ['{} {}'.format(l_forward.glyph.line_alpha,l_reverse.glyph.line_alpha)]
checkbox_group = CheckboxGroup(
        labels=["Forward Propagating Wave", "Reverse Propagating Wave"], active=[0, 1])
checkbox_group.on_click(checkbox_group_handler)

# Set up slider & callback function
def update_alpha(attrname, old, new):
    global alpha
    alpha = alpha_slider.value
    if not toggle.active:
        l_forward.data_source.data["y"] = forward_wave()
        l_reverse.data_source.data["y"] = reverse_wave()
alpha_slider = Slider(title="Alpha (1/m)", value=0.0, start=0.0, end=0.25, step=0.005)
alpha_slider.on_change('value', update_alpha)

# Set up layout
layout = hplot(p, VBox(toggle, button_reset, checkbox_group, alpha_slider, height=480), width=900)

# Create callback function for periodic callback
def update():
    global ii, current_time
    if toggle.active:
        ii += 1
        current_time = ii / n_samp_one_temporal_cycle
        l_forward.data_source.data["y"] = forward_wave()
        l_reverse.data_source.data["y"] = reverse_wave()
        #t2.data_source.data["text"] = ['t = {:.3f} s'.format(current_time)]
<|endoftext|>


<commit_before>from cuttsum.resources import MultiProcessWorker
from cuttsum.trecdata import SCChunkResource
import signal
import urllib3
import os
import Queue
import gzip
import re
import streamcorpus as sc
from datetime import datetime

class ArticlesResource(MultiProcessWorker):
    def __init__(self):
        #Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'articles')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def streamitem_iter(self, event, corpus, extractor):
        for hour in event.list_event_hours():
            path = self.get_chunk_path(event, extractor, hour)
            if os.path.exists(path):

                with sc.Chunk(path=path, mode="rb",
                        message=corpus.sc_msg()) as chunk:
                    for si in chunk:
                        yield hour, path, si

    def get_job_units(self, event, corpus, **kwargs):

        extractor = kwargs.get("extractor", "gold")
        overwrite = kwargs.get("overwrite", False)
        data_dir = os.path.join(self.dir_, extractor, event.fs_name())
        if not os.path.exists(data_dir):
            os.makedirs(data_dir)
        chunks_resource = SCChunkResource()

        if extractor == "gold":

            import cuttsum.judgements

            all_matches = cuttsum.judgements.get_matches()
            matches = all_matches[all_matches["query id"] == event.query_id]

            hours = set([datetime.utcfromtimestamp(int(update_id.split("-")[0])).replace(
                             minute=0, second=0)
                         for update_id in matches["update id"].tolist()])
            hours = sorted(list(hours))

            units = []
            for h, hour in enumerate(hours):
                output_path = self.get_chunk_path(event, extractor, hour)
                if overwrite is True or not os.path.exists(output_path):
                    units.append(h)
            return units

        else:
            raise Exception("extractor: {} not implemented!".format(extractor))

    def do_job_unit(self, event, corpus, unit, **kwargs):

        extractor = kwargs.get("extractor", "gold")
        data_dir = os.path.join(self.dir_, extractor, event.fs_name())
        chunks_resource = SCChunkResource()

        if extractor == "gold":
            import cuttsum.judgements
            all_matches = cuttsum.judgements.get_matches()
            matches = all_matches[all_matches["query id"] == event.query_id]
            stream_ids = set(
                matches["update id"].apply(lambda x: "-".join(x.split("-")[:-1])).tolist())

            hours = set([datetime.utcfromtimestamp(int(update_id.split("-")[0])).replace(
                             minute=0, second=0)
                         for update_id in matches["update id"].tolist()])
            hours = sorted(list(hours))
            hour = hours[unit]
            output_path = self.get_chunk_path(event, extractor, hour)
            gold_si = []
            for path in chunks_resource.get_chunks_for_hour(hour, corpus):
                with sc.Chunk(path=path, mode="rb", message=corpus.sc_msg()) as chunk:
                    for si in chunk:
                        if si.stream_id in stream_ids:
                            gold_si.append(si)
            gold_si.sort(key=lambda x: x.stream_id)
            for si in gold_si:
                print si.stream_id

            if os.path.exists(output_path):
                os.remove(path)
            with sc.Chunk(path=output_path, mode="wb", message=corpus.sc_msg()) as chunk:
                for si in gold_si:
                    chunk.add(si)

        else:
            raise Exception("extractor: {} not implemented!".format(extractor))

    def check_coverage(self, event, corpus, **kwargs):
        data_dir = os.path.join(self.dir_, event.fs_name())
        hours = event.list_event_hours()
        n_hours = len(hours)
        n_covered = 0
        for hour in hours:
            path = os.path.join(data_dir, '{}.sc.gz'.format(
                hour.strftime(u'%Y-%m-%d-%H')))
            if os.path.exists(path):
                n_covered += 1
        if n_hours == 0:
            return 0
        else:
            return n_covered / float(n_hours)

    def get_chunk_path(self, event, extractor, hour):
        data_dir = os.path.join(self.dir_, extractor, event.fs_name())
        return os.path.join(data_dir, u'{}.sc.gz'.format(
            hour.strftime(u'%Y-%m-%d-%H')))

#    def get(self, event, corpus, **kwargs):
#        data_dir = os.path.join(self.dir_, event.fs_name())
#        if not os.path.exists(data_dir):
#            os.makedirs(data_dir)
#        chunks = SCChunkResource()
#
#
#        overwrite = kwargs.get(u'overwrite', False)
#        hours = event.list_event_hours()
#
#        jobs = []
#        for hour in hours:
#            path = self.get_chunk_path(event, hour)
#            if os.path.exists(path) and overwrite is False:
#                continue
#            chunk_paths = chunks.get_chunks_for_hour(hour, corpus)
#            jobs.append((path, chunk_paths))
#
#        n_procs = kwargs.get(u'n_procs', 1)
#        progress_bar = kwargs.get(u'progress_bar', False)
#        self.do_work(_article_resource_worker, jobs, n_procs, progress_bar,
#                     event=event, corpus=corpus)
#
#    def dependencies(self):
#        return tuple(['SCChunkResource'])
#
    def __unicode__(self):
        return unicode(self.__class__.__name__)

    def __str__(self):
        return self.__class__.__name__

#def _article_resource_worker(job_queue, result_queue, **kwargs):
#    signal.signal(signal.SIGINT, signal.SIG_IGN)
#    event = kwargs.get(u'event')
#    corpus = kwargs.get(u'corpus')
#    while not job_queue.empty():
#        try:
#            opath, chunk_paths = job_queue.get(block=False)
#            artcl_detect = ArticleDetector(event)
#            patt = event.regex_pattern()
#            with sc.Chunk(path=opath, mode='wb', message=corpus.sc_msg()) as ochunk:
#                for path in chunk_paths:
#                    for si in sc.Chunk(path=path, message=corpus.sc_msg()):
#                        if si.body.clean_visible is None:
#                            continue
#
#                        elif patt.search(si.body.clean_visible, re.I):
#
#                            #if corpus.annotator() not in si.body.sentences:
#                            #    continue
#                            sentences = corpus.get_sentences(si)
#                            sent_idxs = artcl_detect.find_articles(
#                                sentences)
#                            if len(sent_idxs) > 0:
#                                rel_sents = []
#                                for sent_idx in sent_idxs:
#                                    #for token in sentences[sent_idx].tokens:
#                                    #    print token.token,
#                                    #print
#                                    rel_sents.append(sentences[sent_idx])
#                                si.body.sentences[u'article-clf'] = rel_sents
#                                ochunk.add(si)
#
#
#            result_queue.put(None)
#        except Queue.Empty:
#            pass
#
#

<commit_msg>Formatting.<commit_after>from cuttsum.resources import MultiProcessWorker
from cuttsum.trecdata import SCChunkResource
import signal
import urllib3
import os
import Queue
import gzip
import re
import streamcorpus as sc
from datetime import datetime

class ArticlesResource(MultiProcessWorker):
    def __init__(self):
        #Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'articles')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def streamitem_iter(self, event, corpus, extractor):
        for hour in event.list_event_hours():
            path = self.get_chunk_path(event, extractor, hour)
            if os.path.exists(path):

                with sc.Chunk(path=path, mode="rb",
                        message=corpus.sc_msg()) as chunk:
                    for si in chunk:
                        yield hour, path, si

    def get_job_units(self, event, corpus, **kwargs):

        extractor = kwargs.get("extractor", "gold")
        overwrite = kwargs.get("overwrite", False)
        data_dir = os.path.join(self.dir_, extractor, event.fs_name())
        if not os.path.exists(data_dir):
            os.makedirs(data_dir)
        chunks_resource = SCChunkResource()

        if extractor == "gold":

            import cuttsum.judgements

            all_matches = cuttsum.judgements.get_matches()
            matches = all_matches[all_matches["query id"] == event.query_id]

            hours = set([datetime.utcfromtimestamp(int(update_id.split("-")[0])).replace(
                             minute=0, second=0)
                         for update_id in matches["update id"].tolist()])
            hours = sorted(list(hours))

            units = []
            for h, hour in enumerate(hours):
                output_path = self.get_chunk_path(event, extractor, hour)
                if overwrite is True or not os.path.exists(output_path):
                    units.append(h)
            return units

        else:
            raise Exception("extractor: {} not implemented!".format(extractor))

    def do_job_unit(self, event, corpus, unit, **kwargs):

        extractor = kwargs.get("extractor", "gold")
        data_dir = os.path.join(self.dir_, extractor, event.fs_name())
        chunks_resource = SCChunkResource()

        if extractor == "gold":
            import cuttsum.judgements
            all_matches = cuttsum.judgements.get_matches()
            matches = all_matches[all_matches["query id"] == event.query_id]
            stream_ids = set(
                matches["update id"].apply(
                    lambda x: "-".join(x.split("-")[:-1])).tolist())

            hours = set([datetime.utcfromtimestamp(
                            int(update_id.split("-")[0])).replace(
                             minute=0, second=0)
                         for update_id in matches["update id"].tolist()])
            hours = sorted(list(hours))
            hour = hours[unit]
            output_path = self.get_chunk_path(event, extractor, hour)
            gold_si = []
            for path in chunks_resource.get_chunks_for_hour(hour, corpus):
                with sc.Chunk(path=path, mode="rb", message=corpus.sc_msg()) as chunk:
                    for si in chunk:
                        if si.stream_id in stream_ids:
                            gold_si.append(si)
            gold_si.sort(key=lambda x: x.stream_id)
            for si in gold_si:
                print si.stream_id

            if os.path.exists(output_path):
                os.remove(path)
            with sc.Chunk(path=output_path, mode="wb", message=corpus.sc_msg()) as chunk:
                for si in gold_si:
                    chunk.add(si)

        else:
            raise Exception("extractor: {} not implemented!".format(extractor))

    def check_coverage(self, event, corpus, **kwargs):
        data_dir = os.path.join(self.dir_, event.fs_name())
        hours = event.list_event_hours()
        n_hours = len(hours)
        n_covered = 0
        for hour in hours:
            path = os.path.join(data_dir, '{}.sc.gz'.format(
                hour.strftime(u'%Y-%m-%d-%H')))
            if os.path.exists(path):
                n_covered += 1
        if n_hours == 0:
            return 0
        else:
            return n_covered / float(n_hours)

    def get_chunk_path(self, event, extractor, hour):
        data_dir = os.path.join(self.dir_, extractor, event.fs_name())
        return os.path.join(data_dir, u'{}.sc.gz'.format(
            hour.strftime(u'%Y-%m-%d-%H')))

#    def get(self, event, corpus, **kwargs):
#        data_dir = os.path.join(self.dir_, event.fs_name())
#        if not os.path.exists(data_dir):
#            os.makedirs(data_dir)
#        chunks = SCChunkResource()
#
#
#        overwrite = kwargs.get(u'overwrite', False)
#        hours = event.list_event_hours()
#
#        jobs = []
#        for hour in hours:
#            path = self.get_chunk_path(event, hour)
#            if os.path.exists(path) and overwrite is False:
#                continue
#            chunk_paths = chunks.get_chunks_for_hour(hour, corpus)
#            jobs.append((path, chunk_paths))
#
#        n_procs = kwargs.get(u'n_procs', 1)
#        progress_bar = kwargs.get(u'progress_bar', False)
#        self.do_work(_article_resource_worker, jobs, n_procs, progress_bar,
#                     event=event, corpus=corpus)
#
#    def dependencies(self):
#        return tuple(['SCChunkResource'])
#
    def __unicode__(self):
        return unicode(self.__class__.__name__)

    def __str__(self):
        return self.__class__.__name__

#def _article_resource_worker(job_queue, result_queue, **kwargs):
#    signal.signal(signal.SIGINT, signal.SIG_IGN)
#    event = kwargs.get(u'event')
#    corpus = kwargs.get(u'corpus')
#    while not job_queue.empty():
#        try:
#            opath, chunk_paths = job_queue.get(block=False)
#            artcl_detect = ArticleDetector(event)
#            patt = event.regex_pattern()
#            with sc.Chunk(path=opath, mode='wb', message=corpus.sc_msg()) as ochunk:
#                for path in chunk_paths:
#                    for si in sc.Chunk(path=path, message=corpus.sc_msg()):
#                        if si.body.clean_visible is None:
#                            continue
#
#                        elif patt.search(si.body.clean_visible, re.I):
#
#                            #if corpus.annotator() not in si.body.sentences:
#                            #    continue
#                            sentences = corpus.get_sentences(si)
#                            sent_idxs = artcl_detect.find_articles(
#                                sentences)
#                            if len(sent_idxs) > 0:
#                                rel_sents = []
#                                for sent_idx in sent_idxs:
#                                    #for token in sentences[sent_idx].tokens:
#                                    #    print token.token,
#                                    #print
#                                    rel_sents.append(sentences[sent_idx])
#                                si.body.sentences[u'article-clf'] = rel_sents
#                                ochunk.add(si)
#
#
#            result_queue.put(None)
#        except Queue.Empty:
#            pass
#
#

<|endoftext|>


<commit_before>import logging

from experiment_groups.iteration_managers.base import BaseIterationManger
from experiment_groups.schemas import HyperbandIterationConfig
from polyaxon_schemas.polyaxonfile.specification import ExperimentSpecification
from polyaxon_schemas.utils import Optimization

logger = logging.getLogger('polyaxon.experiments_groups.iteration_manager')

class HyperbandIterationManager(BaseIterationManger):
    def create_iteration(self, experiment_ids=None):
        """Create an iteration for the experiment group."""
        from experiment_groups.models import ExperimentGroupIteration

        search_manager = self.experiment_group.search_manager
        iteration_config = self.experiment_group.iteration_config

        if iteration_config is None:
            iteration = 1
            bracket_iteration = 0
        else:
            should_reschedule = search_manager.should_reschedule(
                iteration=iteration_config.iteration,
                bracket_iteration=iteration_config.bracket_iteration)
            should_reduce_configs = search_manager.should_reduce_configs(
                iteration=iteration_config.iteration,
                bracket_iteration=iteration_config.bracket_iteration)
            if should_reschedule:
                iteration = iteration_config.iteration + 1
                bracket_iteration = 0
            elif should_reduce_configs:
                iteration = iteration_config.iteration
                bracket_iteration = iteration_config.bracket_iteration
            else:
                raise ValueError(
                    'Hyperband create iteration failed for `{}`, '
                    'could not reschedule ot reduce configs'.format(self.experiment_group.id))

        # Create a new iteration config
        iteration_config = HyperbandIterationConfig(
            iteration=iteration,
            bracket_iteration=bracket_iteration)
        if experiment_ids:
            iteration_config.experiment_ids = experiment_ids
        return ExperimentGroupIteration.objects.create(
            experiment_group=self.experiment_group,
            data=iteration_config.to_dict())

    def get_iteration_config(self):
        iteration_config = self.experiment_group.iteration_config
        if iteration_config is None:
            logger.warning(
                'Experiment group `%s` attempt to update, but has no iteration',
                self.experiment_group.id)
            return None
        return iteration_config

    def _update_config(self, iteration_config):
        iteration = self.experiment_group.iteration
        iteration.data = iteration_config.to_dict()
        iteration.save()

    def add_iteration_experiments(self, experiment_ids):
        iteration_config = self.get_iteration_config()
        if not iteration_config:
            return

        iteration_config.experiment_ids = experiment_ids
        self._update_config(iteration_config)

    def update_iteration(self):
        """Update the last experiment group's iteration with experiment performance."""
        iteration_config = self.get_iteration_config()
        if not iteration_config:
            return
        experiments_metrics = self.experiment_group.get_experiments_metrics(
            experiment_ids=iteration_config.experiment_ids,
            metric=self.experiment_group.params_config.hyperband.metric.name,
        )
        iteration_config.experiments_metrics = [m for m in experiments_metrics if m[1] is not None]
        self._update_config(iteration_config)

    def get_reduced_configs(self):
        """Reduce the experiments to restart."""
        iteration_config = self.experiment_group.iteration_config
        if iteration_config is None:
            logger.warning(
                'Experiment group `%s` attempt to update, but has no iteration',
                self.experiment_group.id)
            return
        search_manager = self.experiment_group.search_manager

        # Get the number of experiments to keep
        n_configs_to_keep = search_manager.get_n_config_to_keep_for_iteration(
            iteration=iteration_config.iteration,
            bracket_iteration=iteration_config.bracket_iteration)

        # Get the last group's experiments metrics
        experiments_metrics = self.experiment_group.iteration_config.experiments_metrics

        # Order the experiments
        reverse = Optimization.maximize(
            self.experiment_group.params_config.hyperband.metric.optimization)
        experiments_metrics = sorted(experiments_metrics, key=lambda x: x[1], reverse=reverse)

        # Keep n experiments
        return [xp[0] for xp in experiments_metrics[:n_configs_to_keep]]

    def reduce_configs(self):
        """Reduce the experiments to restart."""
        experiment_ids = self.get_reduced_configs()
        experiments = self.experiment_group.experiments.filter(id__in=experiment_ids)
        self.create_iteration(experiment_ids=experiment_ids)
        iteration_config = self.experiment_group.iteration_config
        params_config = self.experiment_group.params_config
        resource_value = self.experiment_group.search_manager.get_resources_for_iteration(
            iteration=iteration_config.iteration)
        resource_name = params_config.hyperband.resource.name
        resource_value = params_config.hyperband.resource.cast_value(resource_value)

        # Check if we need to resume or restart the experiments
        for experiment in experiments:
            declarations = experiment.declarations
            declarations[resource_name] = resource_value
            specification = ExperimentSpecification(
                values=[experiment.specification.parsed_data, declarations])
            status_message = 'Hyperband iteration: {}, bracket iteration: {}'.format(
                iteration_config.iteration,
                iteration_config.bracket_iteration)

            if params_config.hyperband.resume:
                experiment.resume(
                    declarations=declarations,
                    config=specification.parsed_data,
                    message=status_message)
            else:
                experiment.restart(
                    experiment_group=self.experiment_group,
                    config=specification.parsed_data,
                    declarations=declarations)
<commit_msg>Fix declarations overriding<commit_after>import logging

from experiment_groups.iteration_managers.base import BaseIterationManger
from experiment_groups.schemas import HyperbandIterationConfig
from polyaxon_schemas.polyaxonfile.specification import ExperimentSpecification
from polyaxon_schemas.utils import Optimization

logger = logging.getLogger('polyaxon.experiments_groups.iteration_manager')

class HyperbandIterationManager(BaseIterationManger):
    def create_iteration(self, experiment_ids=None):
        """Create an iteration for the experiment group."""
        from experiment_groups.models import ExperimentGroupIteration

        search_manager = self.experiment_group.search_manager
        iteration_config = self.experiment_group.iteration_config

        if iteration_config is None:
            iteration = 1
            bracket_iteration = 0
        else:
            should_reschedule = search_manager.should_reschedule(
                iteration=iteration_config.iteration,
                bracket_iteration=iteration_config.bracket_iteration)
            should_reduce_configs = search_manager.should_reduce_configs(
                iteration=iteration_config.iteration,
                bracket_iteration=iteration_config.bracket_iteration)
            if should_reschedule:
                iteration = iteration_config.iteration + 1
                bracket_iteration = 0
            elif should_reduce_configs:
                iteration = iteration_config.iteration
                bracket_iteration = iteration_config.bracket_iteration
            else:
                raise ValueError(
                    'Hyperband create iteration failed for `{}`, '
                    'could not reschedule ot reduce configs'.format(self.experiment_group.id))

        # Create a new iteration config
        iteration_config = HyperbandIterationConfig(
            iteration=iteration,
            bracket_iteration=bracket_iteration)
        if experiment_ids:
            iteration_config.experiment_ids = experiment_ids
        return ExperimentGroupIteration.objects.create(
            experiment_group=self.experiment_group,
            data=iteration_config.to_dict())

    def get_iteration_config(self):
        iteration_config = self.experiment_group.iteration_config
        if iteration_config is None:
            logger.warning(
                'Experiment group `%s` attempt to update, but has no iteration',
                self.experiment_group.id)
            return None
        return iteration_config

    def _update_config(self, iteration_config):
        iteration = self.experiment_group.iteration
        iteration.data = iteration_config.to_dict()
        iteration.save()

    def add_iteration_experiments(self, experiment_ids):
        iteration_config = self.get_iteration_config()
        if not iteration_config:
            return

        iteration_config.experiment_ids = experiment_ids
        self._update_config(iteration_config)

    def update_iteration(self):
        """Update the last experiment group's iteration with experiment performance."""
        iteration_config = self.get_iteration_config()
        if not iteration_config:
            return
        experiments_metrics = self.experiment_group.get_experiments_metrics(
            experiment_ids=iteration_config.experiment_ids,
            metric=self.experiment_group.params_config.hyperband.metric.name,
        )
        iteration_config.experiments_metrics = [m for m in experiments_metrics if m[1] is not None]
        self._update_config(iteration_config)

    def get_reduced_configs(self):
        """Reduce the experiments to restart."""
        iteration_config = self.experiment_group.iteration_config
        if iteration_config is None:
            logger.warning(
                'Experiment group `%s` attempt to update, but has no iteration',
                self.experiment_group.id)
            return
        search_manager = self.experiment_group.search_manager

        # Get the number of experiments to keep
        n_configs_to_keep = search_manager.get_n_config_to_keep_for_iteration(
            iteration=iteration_config.iteration,
            bracket_iteration=iteration_config.bracket_iteration)

        # Get the last group's experiments metrics
        experiments_metrics = self.experiment_group.iteration_config.experiments_metrics

        # Order the experiments
        reverse = Optimization.maximize(
            self.experiment_group.params_config.hyperband.metric.optimization)
        experiments_metrics = sorted(experiments_metrics, key=lambda x: x[1], reverse=reverse)

        # Keep n experiments
        return [xp[0] for xp in experiments_metrics[:n_configs_to_keep]]

    def reduce_configs(self):
        """Reduce the experiments to restart."""
        experiment_ids = self.get_reduced_configs()
        experiments = self.experiment_group.experiments.filter(id__in=experiment_ids)
        self.create_iteration(experiment_ids=experiment_ids)
        iteration_config = self.experiment_group.iteration_config
        params_config = self.experiment_group.params_config
        resource_value = self.experiment_group.search_manager.get_resources_for_iteration(
            iteration=iteration_config.iteration)
        resource_name = params_config.hyperband.resource.name
        resource_value = params_config.hyperband.resource.cast_value(resource_value)

        # Check if we need to resume or restart the experiments
        for experiment in experiments:
            declarations = experiment.declarations
            declarations[resource_name] = resource_value
            declarations = {'declarations': declarations}
            specification = ExperimentSpecification(
                values=[experiment.specification.parsed_data, declarations])
            status_message = 'Hyperband iteration: {}, bracket iteration: {}'.format(
                iteration_config.iteration,
                iteration_config.bracket_iteration)

            if params_config.hyperband.resume:
                experiment.resume(
                    declarations=declarations,
                    config=specification.parsed_data,
                    message=status_message)
            else:
                experiment.restart(
                    experiment_group=self.experiment_group,
                    config=specification.parsed_data,
                    declarations=declarations)
<|endoftext|>