<commit_before># 2ping - A bi-directional ping utility
# Copyright (C) 2010-2020 Ryan Finnie
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

import argparse
import os
import socket
import sys

from . import __version__
from .utils import _, AES

def parse_args(argv=None):
    if argv is None:
        argv = sys.argv

    if argv[0].endswith("2ping6"):
        ipv6_default = True
    else:
        ipv6_default = False

    parser = argparse.ArgumentParser(
        description="2ping ({})".format(__version__),
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        prog=os.path.basename(argv[0]),
    )
    parser.add_argument(
        "--version",
        "-V",
        action="version",
        version=__version__,
        help=_("report the program version"),
    )

    # Positionals
    parser.add_argument(
        "host", type=str, default=None, nargs="*", help=_("host to ping")
    )

    # ping-compatible options
    parser.add_argument(
        "--audible", "-a", dest="audible", action="store_true", help=_("audible ping")
    )
    parser.add_argument(
        "--adaptive",
        "-A",
        dest="adaptive",
        action="store_true",
        help=_("adaptive RTT ping"),
    )
    parser.add_argument(
        "--count", "-c", dest="count", type=int, help=_("number of pings to send")
    )
    parser.add_argument(
        "--flood", "-f", dest="flood", action="store_true", help=_("flood mode")
    )
    parser.add_argument(
        "--interval",
        "-i",
        dest="interval",
        type=float,
        default=1.0,
        help=_("seconds between pings"),
        metavar="SECONDS",
    )
    parser.add_argument(
        "--interface-address",
        "-I",
        dest="interface_address",
        type=str,
        action="append",
        help=_("interface bind address"),
        metavar="ADDRESS",
    )
    parser.add_argument(
        "--preload",
        "-l",
        dest="preload",
        type=int,
        default=1,
        help=_("number of pings to send at start"),
        metavar="COUNT",
    )
    parser.add_argument(
        "--pattern",
        "-p",
        dest="pattern",
        type=str,
        help=_("hex pattern for padding"),
        metavar="HEX_BYTES",
    )
    parser.add_argument(
        "--quiet", "-q", dest="quiet", action="store_true", help=_("quiet mode")
    )
    parser.add_argument(
        "--packetsize-compat",
        "-s",
        dest="packetsize_compat",
        type=int,
        help=_("packet size (ping compatible)"),
        metavar="BYTES",
    )
    parser.add_argument(
        "--verbose", "-v", dest="verbose", action="store_true", help=_("verbose mode")
    )
    parser.add_argument(
        "--deadline",
        "-w",
        dest="deadline",
        type=float,
        help=_("maximum run time"),
        metavar="SECONDS",
    )

    # 2ping options
    parser.add_argument("--all-interfaces", action="store_true", help=_("deprecated"))
    parser.add_argument(
        "--auth", type=str, help=_("HMAC authentication key"), metavar="KEY"
    )
    parser.add_argument(
        "--auth-digest",
        type=str,
        default="hmac-md5",
        choices=["hmac-md5", "hmac-sha1", "hmac-sha256", "hmac-crc32", "hmac-sha512"],
        help=_("HMAC authentication digest"),
        metavar="DIGEST",
    )
    parser.add_argument("--debug", action="store_true", help=_("debug mode"))
    parser.add_argument("--encrypt", type=str, help=_("Encryption key"), metavar="KEY")
    parser.add_argument(
        "--encrypt-method",
        type=str,
        default="hkdf-aes256-cbc",
        choices=["hkdf-aes256-cbc"],
        help=_("Encryption method"),
        metavar="METHOD",
    )
    parser.add_argument(
        "--fuzz", type=float, help=_("incoming fuzz percentage"), metavar="PERCENT"
    )
    parser.add_argument(
        "--inquire-wait",
        type=float,
        default=10.0,
        help=_("maximum time before loss inquiries"),
        metavar="SECONDS",
    )
    parser.add_argument("--ipv4", "-4", action="store_true", help=_("force IPv4"))
    parser.add_argument(
        "--ipv6", "-6", action="store_true", default=ipv6_default, help=_("force IPv6")
    )
    parser.add_argument("--listen", action="store_true", help=_("listen mode"))
    parser.add_argument(
        "--loopback", action="store_true", help=_("UNIX loopback test mode")
    )
    parser.add_argument(
        "--loopback-pairs",
        type=int,
        default=1,
        help=_("number of loopback pairs to create"),
        metavar="PAIRS",
    )
    parser.add_argument(
        "--max-packet-size",
        type=int,
        default=512,
        help=_("maximum packet size"),
        metavar="BYTES",
    )
    parser.add_argument(
        "--min-packet-size",
        type=int,
        default=128,
        help=_("minimum packet size"),
        metavar="BYTES",
    )
    parser.add_argument(
        "--nagios",
        type=str,
        help=_("nagios-compatible output"),
        metavar="WRTA,WLOSS%,CRTA,CLOSS%",
    )
    parser.add_argument(
        "--no-3way", action="store_true", help=_("do not send 3-way pings")
    )
    parser.add_argument(
        "--no-match-packet-size",
        action="store_true",
        help=_("do not match packet size of peer"),
    )
    parser.add_argument(
        "--no-send-version",
        action="store_true",
        help=_("do not send program version to peers"),
    )
    parser.add_argument(
        "--notice", type=str, help=_("arbitrary notice text"), metavar="TEXT"
    )
    parser.add_argument(
        "--packet-loss",
        type=str,
        help=_("percentage simulated packet loss"),
        metavar="OUT:IN",
    )
    parser.add_argument(
        "--port", type=str, default="15998", help=_("port to connect / bind to")
    )
    parser.add_argument(
        "--send-monotonic-clock",
        action="store_true",
        help=_("send monotonic clock to peers"),
    )
    parser.add_argument(
        "--send-random", type=int, help=_("send random data to peers"), metavar="BYTES"
    )
    parser.add_argument(
        "--send-time", action="store_true", help=_("send wall clock time to peers")
    )
    parser.add_argument(
        "--stats", type=float, help=_("print recurring statistics"), metavar="SECONDS"
    )
    parser.add_argument(
        "--srv", action="store_true", help=_("lookup SRV records in client mode")
    )
    parser.add_argument(
        "--srv-service",
        type=str,
        default="2ping",
        help=_("service name for SRV lookups"),
    )
    parser.add_argument(
        "--subtract-peer-host-latency",
        action="store_true",
        help=_("subtract peer host latency from RTT calculations, if sent"),
    )

    # ping-compatible ignored options
    for opt in "b|B|d|L|n|R|r|U".split("|"):
        parser.add_argument(
            "-{}".format(opt),
            action="store_true",
            dest="ignored_{}".format(opt),
            help=argparse.SUPPRESS,
        )
    for opt in "F|Q|S|t|T|M|W".split("|"):
        parser.add_argument(
            "-{}".format(opt),
            type=str,
            default=None,
            dest="ignored_{}".format(opt),
            help=argparse.SUPPRESS,
        )

    args = parser.parse_args(args=argv[1:])

    if (not args.listen) and (not args.host) and (not args.loopback):
        parser.print_help()
        parser.exit()
    if args.loopback and not hasattr(socket, "AF_UNIX"):
        parser.error("--loopback not supported on non-UNIX platforms")
    if args.nagios:
        args.quiet = True
        if (not args.count) and (not args.deadline):
            args.count = 5
        nagios_opts = args.nagios.split(",")
        if len(nagios_opts) != 4:
            parser.error(_("Invalid limits"))
        (
            args.nagios_warn_rta,
            args.nagios_warn_loss,
            args.nagios_crit_rta,
            args.nagios_crit_loss,
        ) = nagios_opts
        if args.nagios_warn_loss[-1:] != "%":
            parser.error(_("Invalid limits"))
        if args.nagios_crit_loss[-1:] != "%":
            parser.error(_("Invalid limits"))
        try:
            args.nagios_warn_loss = float(args.nagios_warn_loss[:-1])
            args.nagios_crit_loss = float(args.nagios_crit_loss[:-1])
            args.nagios_warn_rta = float(args.nagios_warn_rta)
            args.nagios_crit_rta = float(args.nagios_crit_rta)
        except ValueError as e:
            parser.error(e.message)
    if args.packetsize_compat:
        args.min_packet_size = args.packetsize_compat + 8
    if args.max_packet_size < args.min_packet_size:
        parser.error(_("Maximum packet size must be at least minimum packet size"))
    if args.max_packet_size < 64:
        parser.error(_("Maximum packet size must be at least 64"))
    args.packet_loss_in = 0
    args.packet_loss_out = 0
    if args.packet_loss:
        if ":" in args.packet_loss:
            (v_out, v_in) = args.packet_loss.split(":", 1)
            try:
                args.packet_loss_out = float(v_out)
                args.packet_loss_in = float(v_in)
            except ValueError as e:
                parser.error(e.message)
        else:
            try:
                args.packet_loss_in = args.packet_loss_out = float(args.packet_loss)
            except ValueError as e:
                parser.error(_("Packet loss: {error}").format(error=e.message))
    if args.pattern:
        if len(args.pattern) & 1:
            parser.error(_("Pattern must be full bytes"))
        if len(args.pattern) > 32:
            parser.error(_("Pattern must be 16 bytes or less"))
        args.pattern_bytes = b""
        for i in range(int(len(args.pattern) / 2)):
            a = args.pattern[(i * 2) : (i * 2 + 2)]
            try:
                b = int(a, 16)
            except ValueError as e:
                parser.error(_("Pattern: {error}").format(error=e.message))
            args.pattern_bytes += bytes([b])
    else:
        args.pattern_bytes = b"\x00"

    hmac_id_map = {
        "hmac-md5": 1,
        "hmac-sha1": 2,
        "hmac-sha256": 3,
        "hmac-crc32": 4,
        "hmac-sha512": 5,
    }
    args.auth_digest_index = hmac_id_map[args.auth_digest]

    if args.encrypt and isinstance(AES, ImportError):
        parser.error(_("Crypto module required for encryption"))

    encrypt_id_map = {"hkdf-aes256-cbc": 1}
    args.encrypt_method_index = encrypt_id_map[args.encrypt_method]

    if args.debug:
        args.verbose = True

    return args
<commit_msg>--interface-address default to [] instead of None<commit_after># 2ping - A bi-directional ping utility
# Copyright (C) 2010-2020 Ryan Finnie
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

import argparse
import os
import socket
import sys

from . import __version__
from .utils import _, AES

def parse_args(argv=None):
    if argv is None:
        argv = sys.argv

    if argv[0].endswith("2ping6"):
        ipv6_default = True
    else:
        ipv6_default = False

    parser = argparse.ArgumentParser(
        description="2ping ({})".format(__version__),
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        prog=os.path.basename(argv[0]),
    )
    parser.add_argument(
        "--version",
        "-V",
        action="version",
        version=__version__,
        help=_("report the program version"),
    )

    # Positionals
    parser.add_argument(
        "host", type=str, default=None, nargs="*", help=_("host to ping")
    )

    # ping-compatible options
    parser.add_argument(
        "--audible", "-a", dest="audible", action="store_true", help=_("audible ping")
    )
    parser.add_argument(
        "--adaptive",
        "-A",
        dest="adaptive",
        action="store_true",
        help=_("adaptive RTT ping"),
    )
    parser.add_argument(
        "--count", "-c", dest="count", type=int, help=_("number of pings to send")
    )
    parser.add_argument(
        "--flood", "-f", dest="flood", action="store_true", help=_("flood mode")
    )
    parser.add_argument(
        "--interval",
        "-i",
        dest="interval",
        type=float,
        default=1.0,
        help=_("seconds between pings"),
        metavar="SECONDS",
    )
    parser.add_argument(
        "--interface-address",
        "-I",
        dest="interface_address",
        type=str,
        action="append",
        default=[],
        help=_("interface bind address"),
        metavar="ADDRESS",
    )
    parser.add_argument(
        "--preload",
        "-l",
        dest="preload",
        type=int,
        default=1,
        help=_("number of pings to send at start"),
        metavar="COUNT",
    )
    parser.add_argument(
        "--pattern",
        "-p",
        dest="pattern",
        type=str,
        help=_("hex pattern for padding"),
        metavar="HEX_BYTES",
    )
    parser.add_argument(
        "--quiet", "-q", dest="quiet", action="store_true", help=_("quiet mode")
    )
    parser.add_argument(
        "--packetsize-compat",
        "-s",
        dest="packetsize_compat",
        type=int,
        help=_("packet size (ping compatible)"),
        metavar="BYTES",
    )
    parser.add_argument(
        "--verbose", "-v", dest="verbose", action="store_true", help=_("verbose mode")
    )
    parser.add_argument(
        "--deadline",
        "-w",
        dest="deadline",
        type=float,
        help=_("maximum run time"),
        metavar="SECONDS",
    )

    # 2ping options
    parser.add_argument("--all-interfaces", action="store_true", help=_("deprecated"))
    parser.add_argument(
        "--auth", type=str, help=_("HMAC authentication key"), metavar="KEY"
    )
    parser.add_argument(
        "--auth-digest",
        type=str,
        default="hmac-md5",
        choices=["hmac-md5", "hmac-sha1", "hmac-sha256", "hmac-crc32", "hmac-sha512"],
        help=_("HMAC authentication digest"),
        metavar="DIGEST",
    )
    parser.add_argument("--debug", action="store_true", help=_("debug mode"))
    parser.add_argument("--encrypt", type=str, help=_("Encryption key"), metavar="KEY")
    parser.add_argument(
        "--encrypt-method",
        type=str,
        default="hkdf-aes256-cbc",
        choices=["hkdf-aes256-cbc"],
        help=_("Encryption method"),
        metavar="METHOD",
    )
    parser.add_argument(
        "--fuzz", type=float, help=_("incoming fuzz percentage"), metavar="PERCENT"
    )
    parser.add_argument(
        "--inquire-wait",
        type=float,
        default=10.0,
        help=_("maximum time before loss inquiries"),
        metavar="SECONDS",
    )
    parser.add_argument("--ipv4", "-4", action="store_true", help=_("force IPv4"))
    parser.add_argument(
        "--ipv6", "-6", action="store_true", default=ipv6_default, help=_("force IPv6")
    )
    parser.add_argument("--listen", action="store_true", help=_("listen mode"))
    parser.add_argument(
        "--loopback", action="store_true", help=_("UNIX loopback test mode")
    )
    parser.add_argument(
        "--loopback-pairs",
        type=int,
        default=1,
        help=_("number of loopback pairs to create"),
        metavar="PAIRS",
    )
    parser.add_argument(
        "--max-packet-size",
        type=int,
        default=512,
        help=_("maximum packet size"),
        metavar="BYTES",
    )
    parser.add_argument(
        "--min-packet-size",
        type=int,
        default=128,
        help=_("minimum packet size"),
        metavar="BYTES",
    )
    parser.add_argument(
        "--nagios",
        type=str,
        help=_("nagios-compatible output"),
        metavar="WRTA,WLOSS%,CRTA,CLOSS%",
    )
    parser.add_argument(
        "--no-3way", action="store_true", help=_("do not send 3-way pings")
    )
    parser.add_argument(
        "--no-match-packet-size",
        action="store_true",
        help=_("do not match packet size of peer"),
    )
    parser.add_argument(
        "--no-send-version",
        action="store_true",
        help=_("do not send program version to peers"),
    )
    parser.add_argument(
        "--notice", type=str, help=_("arbitrary notice text"), metavar="TEXT"
    )
    parser.add_argument(
        "--packet-loss",
        type=str,
        help=_("percentage simulated packet loss"),
        metavar="OUT:IN",
    )
    parser.add_argument(
        "--port", type=str, default="15998", help=_("port to connect / bind to")
    )
    parser.add_argument(
        "--send-monotonic-clock",
        action="store_true",
        help=_("send monotonic clock to peers"),
    )
    parser.add_argument(
        "--send-random", type=int, help=_("send random data to peers"), metavar="BYTES"
    )
    parser.add_argument(
        "--send-time", action="store_true", help=_("send wall clock time to peers")
    )
    parser.add_argument(
        "--stats", type=float, help=_("print recurring statistics"), metavar="SECONDS"
    )
    parser.add_argument(
        "--srv", action="store_true", help=_("lookup SRV records in client mode")
    )
    parser.add_argument(
        "--srv-service",
        type=str,
        default="2ping",
        help=_("service name for SRV lookups"),
    )
    parser.add_argument(
        "--subtract-peer-host-latency",
        action="store_true",
        help=_("subtract peer host latency from RTT calculations, if sent"),
    )

    # ping-compatible ignored options
    for opt in "b|B|d|L|n|R|r|U".split("|"):
        parser.add_argument(
            "-{}".format(opt),
            action="store_true",
            dest="ignored_{}".format(opt),
            help=argparse.SUPPRESS,
        )
    for opt in "F|Q|S|t|T|M|W".split("|"):
        parser.add_argument(
            "-{}".format(opt),
            type=str,
            default=None,
            dest="ignored_{}".format(opt),
            help=argparse.SUPPRESS,
        )

    args = parser.parse_args(args=argv[1:])

    if (not args.listen) and (not args.host) and (not args.loopback):
        parser.print_help()
        parser.exit()
    if args.loopback and not hasattr(socket, "AF_UNIX"):
        parser.error("--loopback not supported on non-UNIX platforms")
    if args.nagios:
        args.quiet = True
        if (not args.count) and (not args.deadline):
            args.count = 5
        nagios_opts = args.nagios.split(",")
        if len(nagios_opts) != 4:
            parser.error(_("Invalid limits"))
        (
            args.nagios_warn_rta,
            args.nagios_warn_loss,
            args.nagios_crit_rta,
            args.nagios_crit_loss,
        ) = nagios_opts
        if args.nagios_warn_loss[-1:] != "%":
            parser.error(_("Invalid limits"))
        if args.nagios_crit_loss[-1:] != "%":
            parser.error(_("Invalid limits"))
        try:
            args.nagios_warn_loss = float(args.nagios_warn_loss[:-1])
            args.nagios_crit_loss = float(args.nagios_crit_loss[:-1])
            args.nagios_warn_rta = float(args.nagios_warn_rta)
            args.nagios_crit_rta = float(args.nagios_crit_rta)
        except ValueError as e:
            parser.error(e.message)
    if args.packetsize_compat:
        args.min_packet_size = args.packetsize_compat + 8
    if args.max_packet_size < args.min_packet_size:
        parser.error(_("Maximum packet size must be at least minimum packet size"))
    if args.max_packet_size < 64:
        parser.error(_("Maximum packet size must be at least 64"))
    args.packet_loss_in = 0
    args.packet_loss_out = 0
    if args.packet_loss:
        if ":" in args.packet_loss:
            (v_out, v_in) = args.packet_loss.split(":", 1)
            try:
                args.packet_loss_out = float(v_out)
                args.packet_loss_in = float(v_in)
            except ValueError as e:
                parser.error(e.message)
        else:
            try:
                args.packet_loss_in = args.packet_loss_out = float(args.packet_loss)
            except ValueError as e:
                parser.error(_("Packet loss: {error}").format(error=e.message))
    if args.pattern:
        if len(args.pattern) & 1:
            parser.error(_("Pattern must be full bytes"))
        if len(args.pattern) > 32:
            parser.error(_("Pattern must be 16 bytes or less"))
        args.pattern_bytes = b""
        for i in range(int(len(args.pattern) / 2)):
            a = args.pattern[(i * 2) : (i * 2 + 2)]
            try:
                b = int(a, 16)
            except ValueError as e:
                parser.error(_("Pattern: {error}").format(error=e.message))
            args.pattern_bytes += bytes([b])
    else:
        args.pattern_bytes = b"\x00"

    hmac_id_map = {
        "hmac-md5": 1,
        "hmac-sha1": 2,
        "hmac-sha256": 3,
        "hmac-crc32": 4,
        "hmac-sha512": 5,
    }
    args.auth_digest_index = hmac_id_map[args.auth_digest]

    if args.encrypt and isinstance(AES, ImportError):
        parser.error(_("Crypto module required for encryption"))

    encrypt_id_map = {"hkdf-aes256-cbc": 1}
    args.encrypt_method_index = encrypt_id_map[args.encrypt_method]

    if args.debug:
        args.verbose = True

    return args
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import srt
import sys
import logging
import itertools
import collections

DEFAULT_ENCODING = 'utf8'

log = logging.getLogger(__name__)

def basic_parser(multi_input=False):
    parser = argparse.ArgumentParser(description=__doc__)

    # Cannot use argparse.FileType as we need to know the encoding from the
    # args

    if multi_input:
        parser.add_argument(
            '--input', '-i', metavar='FILE',
            action='append',
            help='the files to process',
            required=True,
        )
    else:
        parser.add_argument(
            '--input', '-i', metavar='FILE',
            default=sys.stdin,
            help='the file to process (default: stdin)',
        )

    parser.add_argument(
        '--output', '-o', metavar='FILE',
        default=sys.stdout,
        help='the file to write to (default: stdout)',
    )
    parser.add_argument(
        '--no-strict',
        action='store_false', dest='strict',
        help='allow blank lines in output, your media player may explode',
    )
    parser.add_argument(
        '--debug',
        action='store_const', dest='log_level',
        const=logging.DEBUG, default=logging.WARNING,
        help='enable debug logging',
    )

    # Can't set default= here because we need to know whether this was
    # explicitly specified for -e warnings
    parser.add_argument(
        '--encoding', '-e',
        help='the encoding to read/write files in (default: utf8)'
    )
    return parser

def set_basic_args(args):
    encoding_explicitly_specified = True
    if args.encoding is None:
        args.encoding = DEFAULT_ENCODING
        encoding_explicitly_specified = False

    for stream_name in ('input', 'output'):
        stream = getattr(args, stream_name)
        if stream in (sys.stdin, sys.stdout):
            if stream is args.input:
                args.input = srt.parse(args.input.read())
            if encoding_explicitly_specified:
                log.warning(
                    '-e/--encoding has no effect on %s, you need to use '
                    '--input or --output', stream.name,
                )
        else:
            if stream is args.input:
                if isinstance(args.input, collections.MutableSequence):
                    for i, input_fn in enumerate(args.input):
                        with open(input_fn, encoding=args.encoding) as input_f:
                            args.input[i] = srt.parse(input_f.read())
                else:
                    with open(stream, encoding=args.encoding) as input_f:
                        args.input = srt.parse(input_f.read())
            else:
                args.output = open(args.output, 'w+', encoding=args.encoding)

def compose_suggest_on_fail(subs, strict=True):
    try:
        return srt.compose(subs, strict=strict)
    except srt.SRTParseError as thrown_exc:
        log.fatal(
            'Parsing failed, maybe you need to pass a different encoding '
            'with --encoding?'
        )
        raise

def sliding_window(seq, width=2):
    seq_iter = iter(seq)
    sliced = tuple(itertools.islice(seq_iter, width))

    if len(sliced) == width:
        yield sliced

    for elem in seq_iter:
        sliced = sliced[1:] + (elem,)
        yield sliced
<commit_msg>Allow - for stdin/stdout<commit_after>#!/usr/bin/env python

import argparse
import srt
import logging
import sys
import itertools
import collections

DASH_STREAM_MAP = {
    'input': sys.stdin,
    'output': sys.stdout,
}

DEFAULT_ENCODING = 'utf8'

log = logging.getLogger(__name__)

STREAM_ENC_MSG = (
    '-e/--encoding has no effect on %s, you need to use --input or --output '
    'with a real file'
)

def dash_to_stream(arg, arg_type):
    if arg == '-':
        return DASH_STREAM_MAP[arg_type]
    return arg

def basic_parser(multi_input=False):
    parser = argparse.ArgumentParser(description=__doc__)

    # Cannot use argparse.FileType as we need to know the encoding from the
    # args

    if multi_input:
        parser.add_argument(
            '--input', '-i', metavar='FILE',
            action='append',
            type=lambda arg: dash_to_stream(arg, 'input'),
            help='the files to process',
            required=True,
        )
    else:
        parser.add_argument(
            '--input', '-i', metavar='FILE',
            default=sys.stdin,
            type=lambda arg: dash_to_stream(arg, 'input'),
            help='the file to process (default: stdin)',
        )

    parser.add_argument(
        '--output', '-o', metavar='FILE',
        default=sys.stdout,
        type=lambda arg: dash_to_stream(arg, 'output'),
        help='the file to write to (default: stdout)',
    )
    parser.add_argument(
        '--no-strict',
        action='store_false', dest='strict',
        help='allow blank lines in output, your media player may explode',
    )
    parser.add_argument(
        '--debug',
        action='store_const', dest='log_level',
        const=logging.DEBUG, default=logging.WARNING,
        help='enable debug logging',
    )

    # Can't set default= here because we need to know whether this was
    # explicitly specified for -e warnings
    parser.add_argument(
        '--encoding', '-e',
        help='the encoding to read/write files in (default: utf8)'
    )
    return parser

def set_basic_args(args):
    encoding_explicitly_specified = True
    if args.encoding is None:
        args.encoding = DEFAULT_ENCODING
        encoding_explicitly_specified = False

    # TODO: dedupe some of this
    for stream_name in ('input', 'output'):
        log.debug('Processing stream "%s"', stream_name)
        stream = getattr(args, stream_name)
        log.debug('Got %r as stream', stream)
        if stream in DASH_STREAM_MAP.values():
            log.debug('%s in DASH_STREAM_MAP', stream_name)
            if stream is args.input:
                args.input = srt.parse(args.input.read())
            if encoding_explicitly_specified:
                log.warning(STREAM_ENC_MSG, stream.name)
        else:
            log.debug('%s not in DASH_STREAM_MAP', stream_name)
            if stream is args.input:
                if isinstance(args.input, collections.MutableSequence):
                    for i, input_fn in enumerate(args.input):
                        if input_fn in DASH_STREAM_MAP.values():
                            if encoding_explicitly_specified:
                                log.warning(STREAM_ENC_MSG, input_fn.name)
                            if stream is args.input:
                                args.input[i] = srt.parse(input_fn.read())
                        else:
                            with open(input_fn, encoding=args.encoding) as f:
                                args.input[i] = srt.parse(f.read())
                else:
                    with open(stream, encoding=args.encoding) as input_f:
                        args.input = srt.parse(input_f.read())
            else:
                args.output = open(args.output, 'w+', encoding=args.encoding)

def compose_suggest_on_fail(subs, strict=True):
    try:
        return srt.compose(subs, strict=strict)
    except srt.SRTParseError as thrown_exc:
        log.fatal(
            'Parsing failed, maybe you need to pass a different encoding '
            'with --encoding?'
        )
        raise

def sliding_window(seq, width=2):
    seq_iter = iter(seq)
    sliced = tuple(itertools.islice(seq_iter, width))

    if len(sliced) == width:
        yield sliced

    for elem in seq_iter:
        sliced = sliced[1:] + (elem,)
        yield sliced
<|endoftext|>


<commit_before>"""
Code to perform lexing accourding to token definitions
"""

# standard library
import re
import logging

# application specific
from .utils import logger, enforce_types
from .exceptions import InvalidToken

logger = logger.getChild('Lexer')

class Lexer(object):
    """
    Code to perform lexing according to token definitions
    """
    def __init__(self, token_defs: dict):
        self.tokens = {}
        self.TRANS = str.maketrans({
            '(': ' ( ',
            ')': ' ) ',
            '"': ' " ',
            "'": " ' ",
        })

        if token_defs:
            self.load_token_definitions(token_defs)
        else:
            self.tokens_loaded = False

    def match_with(self, left: str) -> tuple:
        """
        Convenience function.

        returns an obj with a match function that compares left with
        the supplied right
        """

        # todo; funtools.partial :D

        def internal(right):
            return left == right
        return type('obj', (object,), {'match': internal})

    @enforce_types
    def load_token_definitions(self, token_defs: dict):
        """
        Iterates through the supplied token_defs dictionary, creates wrappers
        for literals and compiles regex's

        See \
        :py:meth:`GrammarParser.load_token_definitions <tyrian.typarser.grammar_parser.GrammarParser.load_token_definitions>` \
        for format
        """

        for k, v in token_defs['literal'].items():
            k = self.match_with(k)
            self.tokens[k] = v

        for k, v in token_defs['regex'].items():
            k = re.compile(k)
            self.tokens[k] = v

        self.tokens_loaded = True

    @enforce_types
    def lex(self, content: str, filename: str=None) -> list:
        """
        Takes a string to lex according to token definition loaded
        via load_token_definitions
        """

        assert self.tokens_loaded, (
            'Please call load_token_definitions before calling this function')

        lines = content.split('\n')

        tokens = []
        for line_no, line in enumerate(lines, start=1):
            for token in self._lex(line, line_no, filename):
                tokens.append(token)

        return tokens

    @enforce_types
    def _lex(self, line: str, line_no: int, filename: str):
        """
        used internally by lex, does actual lexing
        """

        line = (line.translate(self.TRANS)
                    .strip()
                    .split(' '))

        for current_token in line:
            for definition, name in self.tokens.items():
                logging.debug(definition.match(current_token), current_token)
                if definition.match(current_token):
                    yield {
                        "name": name,
                        "token": current_token,
                        "line_no": line_no,
                        'filename': filename
                    }
                    break
            else:
                if current_token.strip():
                    msg = '"{}" on line {}'.format(current_token, line_no)
                    if filename:
                        msg += ' of file {}'.format(filename)
                    raise InvalidToken(msg)
<commit_msg>Improve match_with, add param defs<commit_after>"""
Code to perform lexing accourding to token definitions
"""

# standard library
import re
import logging
import operator
import functools

# application specific
from .utils import logger, enforce_types
from .exceptions import InvalidToken

logger = logger.getChild('Lexer')

class Lexer(object):
    """
    Code to perform lexing according to token definitions

    :param token_defs: dictionary containing token definitions, see \
    :py:meth:`Lexer.load_token_definitions <tyrian.lexer.Lexer.load_token_definitions>`
    """

    def __init__(self, token_defs: dict):
        self.tokens = {}
        self.TRANS = str.maketrans({
            '(': ' ( ',
            ')': ' ) ',
            '"': ' " ',
            "'": " ' ",
        })

        if token_defs:
            self.load_token_definitions(token_defs)
        else:
            self.tokens_loaded = False

    def match_with(self, left: str) -> tuple:
        """
        Convenience function.

        returns an object with a match attribute partial'ed operator.eq,
        configured to match `left` with the supplied `right`
        """

        match = functools.partial(operator.eq, left)

        return type('obj', (object,), {'match': match})

    @enforce_types
    def load_token_definitions(self, token_defs: dict):
        """
        Iterates through the supplied token_defs dictionary, creates wrappers
        for literals and compiles regex's

        See \
        :py:meth:`GrammarParser.load_token_definitions <tyrian.typarser.grammar_parser.GrammarParser.load_token_definitions>` \
        for format
        """

        for k, v in token_defs['literal'].items():
            k = self.match_with(k)
            self.tokens[k] = v

        for k, v in token_defs['regex'].items():
            k = re.compile(k)
            self.tokens[k] = v

        self.tokens_loaded = True

    @enforce_types
    def lex(self, content: str, filename: str=None) -> list:
        """
        Takes a string to lex according to token definition loaded
        via load_token_definitions
        """

        assert self.tokens_loaded, (
            'Please call load_token_definitions before calling this function')

        lines = content.split('\n')

        tokens = []
        for line_no, line in enumerate(lines, start=1):
            for token in self._lex(line, line_no, filename):
                tokens.append(token)

        return tokens

    @enforce_types
    def _lex(self, line: str, line_no: int, filename: str):
        """
        used internally by lex, does actual lexing
        """

        line = (line.translate(self.TRANS)
                    .strip()
                    .split(' '))

        for current_token in line:
            for definition, name in self.tokens.items():
                logging.debug(definition.match(current_token), current_token)
                if definition.match(current_token):
                    yield {
                        "name": name,
                        "token": current_token,
                        "line_no": line_no,
                        'filename': filename
                    }
                    break
            else:
                if current_token.strip():
                    msg = '"{}" on line {}'.format(current_token, line_no)
                    if filename:
                        msg += ' of file {}'.format(filename)
                    raise InvalidToken(msg)
<|endoftext|>


<commit_before>"""
Pm Project module
=================

Provide functionality for project management.

Commands:
^^^^^^^^^

       ls
         list contents
       init
         initialize a project folder
       add
         add boilerplate code
       compress
         compress files
       clean
         remove files
       du
         calculate disk usage
       deliver
         deliver project data to customer

Synopsis:
---------

The following command creates a directory in the project root
named j_doe_00_00. The '-g' flag adds a git directory to the
project repos, and initializes the project subdirectory j_doe_00_00_git
for use with git.

   pm project init j_doe_00_00 -g

FIXME: Boilerplate code can be added to the project by running

   pm project add j_doe_00_00

The boilerplate code includes makefiles, sbatch templates, and documentation
templates.

Code
====
"""
import os
import sys
import re

from cement.core import controller
from pmtools import AbstractBaseController
from pmtools.utils.misc import query_yes_no, filtered_walk

## Main project controller
class ProjectController(AbstractBaseController):
    """
    Functionality for project management.
    """
    class Meta:
        label = 'project'
        description = 'Manage projects'
        arguments = [
            (['projectid'], dict(help="Scilife project id (e.g. j_doe_00_00)", default="", action="store", nargs="?")),
            (['--pbzip2'], dict(help="Use pbzip2 as compressing device", default=False, action="store_true")),
            (['--pigz'], dict(help="Use pigz as compressing device", default=False, action="store_true")),
            (['-f', '--fastq'], dict(help="Workon fastq files", default=False, action="store_true")),
            (['-p','--pileup'], dict(help="Workon pileup files", default=False, action="store_true")),
            ([,'--txt'], dict(help="Workon txt files", default=False, action="store_true")),
            (['-g', '--git'], dict(help="Initialize git directory in repos and project gitdir", default=False, action="store_true")),
            (['-S', '--sampleid'], dict(help="project sample id", action="store")),
            (['-F', '--flowcellid'], dict(help="project flowcell id", action="store")),
            (['--finished'], dict(help="include finished project listing", action="store_true", default=False)),
            ]
        flowcelldir = None
        compress_opt = "-v"
        compress_prog = "gzip"
        compress_suffix = ".gz"
        project_root = ""
        file_pat = []

    ## FIX ME: would like there to be a hook for this
    def _post_dispatch(self):
        # setup project search space
        if self.app.pargs.finished:
            self._meta.project_root = self.app.config.get("project", "finished")
        else:
            self._meta.project_root = self.app.config.get("project", "root")
        assert os.path.exists(self._meta.project_root), "No such directory {}; check your project config".format(self._meta.project_root)
        ## Assure that project exists
        if self.pargs.projectid:
            assert os.path.exists(os.path.join(self._meta.project_root, self.pargs.projectid)), "no such project '{}' in project directory '{}'".format(self.pargs.projectid, self._meta.project_root)

        ## Setup file patterns to use with ls and compress
        if self.pargs.fastq:
            self._meta.file_pat += [".fastq", "fastq.txt", ".fq"]
        if self.pargs.pileup:
            self._meta.file_pat += [".pileup"]
        if self.pargs.txt:
            self._meta.file_pat += [".txt"]

        ## Setup zip program
        if self.pargs.pbzip2:
            self._meta.compress_prog = "pbzip2"
        elif self.pargs.pigz:
            self._meta.compress_prog = "pigz"

    ## utility functions
    def _assert_project_id(self, msg):
        if not self.pargs.projectid:
            self.log.warn(msg)
            return False
        return True

    ## default
    @controller.expose(hide=True)
    def default(self):
        print self._help_text

    ## ls
    @controller.expose(help="List project folder")
    def ls(self):
        self._post_dispatch()
        if self.pargs.projectid=="":
            self._ls(self._meta.project_root, filter=True)
        else:
            pattern = "|".join(["{}$".format(x) for x in self._meta.file_pat])
            def file_filter(f):
                if not pattern:
                    return
                return re.search(pattern, f) != None
            flist = filtered_walk(os.path.join(self._meta.project_root, self.pargs.projectid), file_filter)
            if flist:
                self.app._output_data["stdout"].write("\n".join(flist))

    ## init
    @controller.expose(help="Initalize project folder")
    def init(self):
        if self.pargs.projectid=="":
            return
        self.log.info("Initalizing project %s" % self.pargs.projectid)
        ## Create directory structure
        dirs = ["%s_git" % self.pargs.projectid, "data", "intermediate"]
        gitdirs = ["config", "sbatch", "doc", "lib"]
        [self.safe_makedir(os.path.join(self._meta.project_root, self.pargs.projectid, x)) for x in dirs]
        [self.safe_makedir(os.path.join(self._meta.project_root, self.pargs.projectid, dirs[0], x)) for x in gitdirs]
        ## Initialize git if repos defined and flag set
        if self.config.get("project", "repos") and self.pargs.git:
            dirs = {
                'repos':os.path.join(self.config.get("project", "repos"), "current", self.pargs.projectid),
                'gitdir':os.path.join(self._meta.project_root, self.pargs.projectid, "%s_git" % self.pargs.projectid)
                    }
            self.safe_makedir(dirs['repos'])
            self.sh(["cd", dirs['repos'], "&& git init --bare"])
            self.sh(["cd", dirs['gitdir'], "&& git init && git remote add origin", dirs['repos']])

    def _flowcells(self):
        self._meta.flowcelldir = os.path.join(self._meta.project_root, self.pargs.projectid, "nobackup", "data")
        if not os.path.exists(self._meta.flowcelldir):
              self._meta.flowcelldir = os.path.join(self._meta.project_root, self.pargs.projectid,"data")
        if not os.path.exists(self._meta.flowcelldir):
            return []
        files = os.listdir(self._meta.flowcelldir)
        return files

    ## add
    @controller.expose(help="Add boilerplate code")
    def add(self):
        self._not_implemented()

    ## clean
    @controller.expose(help="Remove files")
    def clean(self):
        self._not_implemented()

    def _compress(self, pattern, label="compress"):
        def compress_filter(f):
            if not pattern:
                return
            return re.search(pattern, f) != None

        if self.pargs.input_file:
            flist = [self.pargs.input_file]
        else:
            flist = filtered_walk(os.path.join(self._meta.project_root, self.pargs.projectid), compress_filter)

        if len(flist) == 0:
            self.app.log.info("No files matching pattern {} found".format(pattern))
        if len(flist) > 0 and not query_yes_no("Going to {} {} files ({}...). Are you sure you want to continue?".format(label, len(flist), ",".join([os.path.basename(x) for x in flist[0:10]])), force=self.pargs.force):
            sys.exit()
        for f in flist:
            self.log.info("{}ing {}".format(label, f))
            self.app.cmd.command([self._meta.compress_prog, self._meta.compress_opt, "%s" % f], label, ignore_error=True)

    ## decompress
    @controller.expose(help="Decompress files")
    def decompress(self):
        """Decompress files"""
        self._post_dispatch()
        if not self._assert_project_id("Not running decompress function on project root directory"):
            return
        self._meta.compress_opt = "-dv"
        if self.pargs.pbzip2:
            self._meta.compress_suffix = ".bz2"
        pattern = "|".join(["{}{}$".format(x, self._meta.compress_suffix) for x in self._meta.file_pat])
        self._compress(pattern, label="decompress")

    ## compress
    @controller.expose(help="Compress files")
    def compress(self):
        self._post_dispatch()
        if not self._assert_project_id("Not running compress function on project root directory"):
            return
        self._meta.compress_opt = "-v"
        pattern = "|".join(["{}$".format(x) for x in self._meta.file_pat])
        self._compress(pattern)

    ## du
    @controller.expose(help="Calculate disk usage in intermediate and data directories")
    def du(self):
        self._post_dispatch()
        if not self._assert_project_id("Not running du function on project root directory"):
            return
        out = self.app.cmd.command(["du", "-hs", "{}".format(os.path.join(self._meta.project_root, self.pargs.projectid))])
        if out:
            self.app._output_data["stdout"].write(out.rstrip())

    ## deliver
    ##
    ## NOTE: this is a temporary workaround for cases where data has
    ## been removed from analysis directory
    @controller.expose(help="Deliver project data")
    def deliver(self):
        self._post_dispatch()
        if not self.pargs.flowcellid:
            self.log.warn("No flowcellid provided. Please provide a flowcellid from which to deliver. Available options are:\n\t{}".format("\n\t".join(self._flowcells())))
            return

<commit_msg>Fix type<commit_after>"""
Pm Project module
=================

Provide functionality for project management.

Commands:
^^^^^^^^^

       ls
         list contents
       init
         initialize a project folder
       add
         add boilerplate code
       compress
         compress files
       clean
         remove files
       du
         calculate disk usage
       deliver
         deliver project data to customer

Synopsis:
---------

The following command creates a directory in the project root
named j_doe_00_00. The '-g' flag adds a git directory to the
project repos, and initializes the project subdirectory j_doe_00_00_git
for use with git.

   pm project init j_doe_00_00 -g

FIXME: Boilerplate code can be added to the project by running

   pm project add j_doe_00_00

The boilerplate code includes makefiles, sbatch templates, and documentation
templates.

Code
====
"""
import os
import sys
import re

from cement.core import controller
from pmtools import AbstractBaseController
from pmtools.utils.misc import query_yes_no, filtered_walk

## Main project controller
class ProjectController(AbstractBaseController):
    """
    Functionality for project management.
    """
    class Meta:
        label = 'project'
        description = 'Manage projects'
        arguments = [
            (['projectid'], dict(help="Scilife project id (e.g. j_doe_00_00)", default="", action="store", nargs="?")),
            (['--pbzip2'], dict(help="Use pbzip2 as compressing device", default=False, action="store_true")),
            (['--pigz'], dict(help="Use pigz as compressing device", default=False, action="store_true")),
            (['-f', '--fastq'], dict(help="Workon fastq files", default=False, action="store_true")),
            (['-p','--pileup'], dict(help="Workon pileup files", default=False, action="store_true")),
            (['--txt'], dict(help="Workon txt files", default=False, action="store_true")),
            (['-g', '--git'], dict(help="Initialize git directory in repos and project gitdir", default=False, action="store_true")),
            (['-S', '--sampleid'], dict(help="project sample id", action="store")),
            (['-F', '--flowcellid'], dict(help="project flowcell id", action="store")),
            (['--finished'], dict(help="include finished project listing", action="store_true", default=False)),
            ]
        flowcelldir = None
        compress_opt = "-v"
        compress_prog = "gzip"
        compress_suffix = ".gz"
        project_root = ""
        file_pat = []

    ## FIX ME: would like there to be a hook for this
    def _post_dispatch(self):
        # setup project search space
        if self.app.pargs.finished:
            self._meta.project_root = self.app.config.get("project", "finished")
        else:
            self._meta.project_root = self.app.config.get("project", "root")
        assert os.path.exists(self._meta.project_root), "No such directory {}; check your project config".format(self._meta.project_root)
        ## Assure that project exists
        if self.pargs.projectid:
            assert os.path.exists(os.path.join(self._meta.project_root, self.pargs.projectid)), "no such project '{}' in project directory '{}'".format(self.pargs.projectid, self._meta.project_root)

        ## Setup file patterns to use with ls and compress
        if self.pargs.fastq:
            self._meta.file_pat += [".fastq", "fastq.txt", ".fq"]
        if self.pargs.pileup:
            self._meta.file_pat += [".pileup"]
        if self.pargs.txt:
            self._meta.file_pat += [".txt"]

        ## Setup zip program
        if self.pargs.pbzip2:
            self._meta.compress_prog = "pbzip2"
        elif self.pargs.pigz:
            self._meta.compress_prog = "pigz"

    ## utility functions
    def _assert_project_id(self, msg):
        if not self.pargs.projectid:
            self.log.warn(msg)
            return False
        return True

    ## default
    @controller.expose(hide=True)
    def default(self):
        print self._help_text

    ## ls
    @controller.expose(help="List project folder")
    def ls(self):
        self._post_dispatch()
        if self.pargs.projectid=="":
            self._ls(self._meta.project_root, filter=True)
        else:
            pattern = "|".join(["{}$".format(x) for x in self._meta.file_pat])
            def file_filter(f):
                if not pattern:
                    return
                return re.search(pattern, f) != None
            flist = filtered_walk(os.path.join(self._meta.project_root, self.pargs.projectid), file_filter)
            if flist:
                self.app._output_data["stdout"].write("\n".join(flist))

    ## init
    @controller.expose(help="Initalize project folder")
    def init(self):
        if self.pargs.projectid=="":
            return
        self.log.info("Initalizing project %s" % self.pargs.projectid)
        ## Create directory structure
        dirs = ["%s_git" % self.pargs.projectid, "data", "intermediate"]
        gitdirs = ["config", "sbatch", "doc", "lib"]
        [self.safe_makedir(os.path.join(self._meta.project_root, self.pargs.projectid, x)) for x in dirs]
        [self.safe_makedir(os.path.join(self._meta.project_root, self.pargs.projectid, dirs[0], x)) for x in gitdirs]
        ## Initialize git if repos defined and flag set
        if self.config.get("project", "repos") and self.pargs.git:
            dirs = {
                'repos':os.path.join(self.config.get("project", "repos"), "current", self.pargs.projectid),
                'gitdir':os.path.join(self._meta.project_root, self.pargs.projectid, "%s_git" % self.pargs.projectid)
                    }
            self.safe_makedir(dirs['repos'])
            self.sh(["cd", dirs['repos'], "&& git init --bare"])
            self.sh(["cd", dirs['gitdir'], "&& git init && git remote add origin", dirs['repos']])

    def _flowcells(self):
        self._meta.flowcelldir = os.path.join(self._meta.project_root, self.pargs.projectid, "nobackup", "data")
        if not os.path.exists(self._meta.flowcelldir):
              self._meta.flowcelldir = os.path.join(self._meta.project_root, self.pargs.projectid,"data")
        if not os.path.exists(self._meta.flowcelldir):
            return []
        files = os.listdir(self._meta.flowcelldir)
        return files

    ## add
    @controller.expose(help="Add boilerplate code")
    def add(self):
        self._not_implemented()

    ## clean
    @controller.expose(help="Remove files")
    def clean(self):
        self._not_implemented()

    def _compress(self, pattern, label="compress"):
        def compress_filter(f):
            if not pattern:
                return
            return re.search(pattern, f) != None

        if self.pargs.input_file:
            flist = [self.pargs.input_file]
        else:
            flist = filtered_walk(os.path.join(self._meta.project_root, self.pargs.projectid), compress_filter)

        if len(flist) == 0:
            self.app.log.info("No files matching pattern {} found".format(pattern))
        if len(flist) > 0 and not query_yes_no("Going to {} {} files ({}...). Are you sure you want to continue?".format(label, len(flist), ",".join([os.path.basename(x) for x in flist[0:10]])), force=self.pargs.force):
            sys.exit()
        for f in flist:
            self.log.info("{}ing {}".format(label, f))
            self.app.cmd.command([self._meta.compress_prog, self._meta.compress_opt, "%s" % f], label, ignore_error=True)

    ## decompress
    @controller.expose(help="Decompress files")
    def decompress(self):
        """Decompress files"""
        self._post_dispatch()
        if not self._assert_project_id("Not running decompress function on project root directory"):
            return
        self._meta.compress_opt = "-dv"
        if self.pargs.pbzip2:
            self._meta.compress_suffix = ".bz2"
        pattern = "|".join(["{}{}$".format(x, self._meta.compress_suffix) for x in self._meta.file_pat])
        self._compress(pattern, label="decompress")

    ## compress
    @controller.expose(help="Compress files")
    def compress(self):
        self._post_dispatch()
        if not self._assert_project_id("Not running compress function on project root directory"):
            return
        self._meta.compress_opt = "-v"
        pattern = "|".join(["{}$".format(x) for x in self._meta.file_pat])
        self._compress(pattern)

    ## du
    @controller.expose(help="Calculate disk usage in intermediate and data directories")
    def du(self):
        self._post_dispatch()
        if not self._assert_project_id("Not running du function on project root directory"):
            return
        out = self.app.cmd.command(["du", "-hs", "{}".format(os.path.join(self._meta.project_root, self.pargs.projectid))])
        if out:
            self.app._output_data["stdout"].write(out.rstrip())

    ## deliver
    ##
    ## NOTE: this is a temporary workaround for cases where data has
    ## been removed from analysis directory
    @controller.expose(help="Deliver project data")
    def deliver(self):
        self._post_dispatch()
        if not self.pargs.flowcellid:
            self.log.warn("No flowcellid provided. Please provide a flowcellid from which to deliver. Available options are:\n\t{}".format("\n\t".join(self._flowcells())))
            return

<|endoftext|>


<commit_before>import sys, os, io
import sha
import numpy as np
import scipy as sp
import networkx as nx
import traceback
import functools
try:
    import simplejson as js
except:
    import json as js

from time import time, sleep
import random
import logging
import logging.handlers
from utils import getHost

sys.path.append('build') # Yuck!
sys.path.append('.')
sys.path.append('lib')

try:
    from samc import SAMCRun
    from bayesnet import BayesNet
    from bayesnetcpd import BayesNetCPD
    from generator import *
except ImportError as e:
    logger.critical(e)
    logger.critical(sys.path)
    logger.critical("Make sure LD_LIBRARY_PATH is set correctly and that the build"+\
            " directory is populated by waf.")
    sys.exit()

def sample(states, data, ground, template=None, iters=1e4, priorweight=1.0, burn=100000, stepscale=10000, temperature = 1.0):
    nodes = np.arange(data.shape[1])

    b = BayesNetCPD(states, data, template, priorweight)
    ground = BayesNetCPD(states, np.array([]))
    s = SAMCRun(b,ground,burn,stepscale)

    t1 = time()
    detail = s.sample(iters, temperature)
    t2 = time()
    logger.info("SAMC run took %f seconds." , (t2-t1))
    return b,s

if 'SAMC_JOB' in os.environ and 'WORKHASH' in os.environ:
    try:
        syslog_server = os.environ['SYSLOG']
        redis_server = os.environ['REDIS']
    except:
        print "ERROR in worker: Need SYSLOG and REDIS environment variables defined."
        sys.exit(1)

    formatter = logging.Formatter('%(name)s: samc %(levelname)s %(message)s')

    h = logging.handlers.SysLogHandler((syslog_server,10514))
    h.setLevel(logging.INFO)
    h.setFormatter(formatter)

    logger = logging.getLogger(getHost() + '-worker-' + str(os.getpid()))

    hstream = logging.StreamHandler()
    hstream.setLevel(logging.INFO)
    hstream.setFormatter(formatter)

    logger.addHandler(hstream)
    logger.addHandler(h)
    logger.setLevel(logging.DEBUG)

    def log_uncaught_exceptions(ex_cls, ex, tb):
        logger.critical(''.join(traceback.format_tb(tb)))
        logger.critical('{0}: {1}'.format(ex_cls, ex))

    sys.excepthook = log_uncaught_exceptions
    logger.info("Beginning Job")

    import redis
    r = redis.StrictRedis(redis_server)

    ########## Read config from driver.py ########
    config = js.loads(os.environ['SAMC_JOB'])

    N = config['nodes']
    iters = config['samc_iters']
    numdata = config['numdata']
    priorweight = config['priorweight']
    numtemplate = config['numtemplate']
    burn = config.get('burn', 100000)
    stepscale = config.get('stepscale', 10000)
    experiment = config.get('experiment_type', 'single')
    method = config.get('gen_method', 'dirichlet')
    noise = config.get('noise', 0.0)
    truncate = config.get('truncate', None)

    if 'graph' in config:
        assert 'seed' in config, "Seed not in configuration."
        ngraph = np.fromstring(config['graph'], dtype=np.int32)
        ngraph = ngraph.reshape(int(np.sqrt(ngraph.size)), int(np.sqrt(ngraph.size)))
        #logger.debug('Graph adjacency matrix: %s', str(ngraph))
        graph = nx.from_numpy_matrix(ngraph, create_using=nx.DiGraph())
        random.seed(config['seed'])
        np.random.seed(config['seed'])
        _,_,joint = generateData(graph, 10, noise=noise, method=method) # 1000 because we need
        # to generate the CPDs deterministically
        random.seed()
        np.random.seed()
        data,states,_ = generateData(graph, numdata, noise=noise, joint=joint, method=method)
    else:
        graph = generateHourGlassGraph(nodes=N)
        data, states, joint = generateData(graph,numdata,noise=noise,method=method)

    if experiment == 'single':
        if numtemplate == 0:
            b,s = sample(states, data, joint, template=None, iters=iters, burn=burn, stepscale=stepscale)
        else:
            template = sampleTemplate(graph, numtemplate)
            b,s = sample(states, data, joint, template, iters, burn=burn, stepscale=stepscale)

        mean = s.estimate_func_mean(truncate)
        r.lpush(os.environ['WORKHASH'], mean)
        print('Mean estimation: %f' % mean)

    if experiment == 'difference':
        b,s = sample(states, data, joint, template=None, iters=iters, burn=burn)
        mean1 = s.estimate_func_mean(truncate)
        template = sampleTemplate(graph, numtemplate)
        b,s = sample(states, data, joint, template=template, iters=iters, burn=burn)
        mean2 = s.estimate_func_mean(truncate)

        # Send back func_mean to store
        r.lpush(os.environ['WORKHASH'], mean1 - mean2)
        print('Function difference: %f' % (mean1 - mean2))

elif __name__ == '__main__':
    from utils import *
    if False: #WBCD Data
        iters = 3e5
        data = np.loadtxt('data/WBCD2.dat', np.int)
        data[:,:-1] -= 1
        states = np.array([10]*9 + [2],dtype=np.int)

        b,s = sample(states, data, iters=iters)
        plotHist(s)

    if True:

        #import pstats, cProfile, statprof
        #logging.critical('test %d', np.random.randint(1000))
        #x = 1/0
        #sys.exit()
        N = 5
        iters = 1e4
        numdata = 50
        priorweight = 5
        numtemplate = 5

        random.seed(123456)
        np.random.seed(123456)

        g1 = generateHourGlassGraph(nodes=N)

        #d1, states1, joint1 = generateData(g1,numdata,method='noisylogic')
        d1, states1, joint1 = generateData(g1,numdata,method='dirichlet')
        print "Ground joint: \n", joint1
        print "Effective sample size: %d" % len(set(map(tuple,d1)))

        t1 = sampleTemplate(g1, numtemplate)

        #drawGraphs(g1)

        ###############################################
        #cProfile.runctx("sample(states1, d1, joint1, template=t1,iters=iters, priorweight=priorweight, burn=0, stepscale=40000)", globals(),locals(), "prof.prof")

        #s = pstats.Stats('prof.prof')
        #s.strip_dirs().sort_stats("time").print_stats()

        #with statprof.profile():
            #b,s = sample(states1, d1, g1, template=t1,
                    #iters=iters, priorweight=priorweight, burn=0, stepscale=40000)

        ###############################################
        b,s = sample(states1, d1, joint1, template=t1,
                iters=iters, priorweight=priorweight, burn=0, stepscale=30000, temperature=10)

        #es = []
        #newes = []
        #for i in range(20):
            #es.append(s.estimate_func_mean())
            #newes.append(s.estimate_func_mean(trunc=True))
            #s.sample(1e4)
        ###############################################

        #print "##### wotemp - wtemp = %f - %f = %f #######" % (mean1, mean2, mean1-mean2)

        #b.update_graph(s.mapvalue)
        #b2.update_graph(s2.mapvalue)
        #drawGraphs(g1, t1)

        #val1 = b.global_edge_presence()
        #val2 = b2.global_edge_presence()

        #print('Global edge error: %f' % mean1)
        #print('Map edge error: %f' % val1)

<commit_msg>Major changes to experiment for jobmon<commit_after>import sys, os, io
import sha
import numpy as np
import scipy as sp
import networkx as nx
import traceback
import functools
try:
    import simplejson as js
except:
    import json as js

from time import time, sleep
import random
import logging
import logging.handlers
from utils import getHost

sys.path.append('build') # Yuck!
sys.path.append('.')
sys.path.append('lib')

def sample(states,
        data,
        ground,
        template=None,
        iters=1e4,
        priorweight=1.0,
        burn=100000,
        stepscale=10000,
        temperature = 1.0):

    nodes = np.arange(data.shape[1])

    b = BayesNetCPD(states, data, template, priorweight)
    ground = BayesNetCPD(states, np.array([]))
    s = SAMCRun(b,ground,burn,stepscale)

    t1 = time()
    detail = s.sample(iters, temperature)
    t2 = time()
    #logger.info("SAMC run took %f seconds." , (t2-t1))
    return b,s

if __name__ == '__main__':
    try:
        syslog_server = os.environ['SYSLOG']
    except:
        print "ERROR in worker: Need SYSLOG environment variable defined."
        sys.exit(1)
    try:
        if 'WORKHASH' in os.environ:
            redis_server = os.environ['REDIS']
    except:
        print "ERROR in worker: Need REDIS environment variable defined."
        sys.exit(1)

    formatter = logging.Formatter('%(name)s: samc %(levelname)s %(message)s')

    h = logging.handlers.SysLogHandler((syslog_server,10514))
    h.setLevel(logging.INFO)
    h.setFormatter(formatter)

    logger = logging.getLogger(getHost() + '-worker-' + str(os.getpid()))

    hstream = logging.StreamHandler()
    hstream.setLevel(logging.INFO)
    hstream.setFormatter(formatter)

    logger.addHandler(hstream)
    logger.addHandler(h)
    logger.setLevel(logging.DEBUG)

    def log_uncaught_exceptions(ex_cls, ex, tb):
        logger.critical(''.join(traceback.format_tb(tb)))
        logger.critical('{0}: {1}'.format(ex_cls, ex))

    sys.excepthook = log_uncaught_exceptions
    logger.info("Beginning Job")

    try:
        from samc import SAMCRun
        from bayesnet import BayesNet
        from bayesnetcpd import BayesNetCPD
        from generator import *
    except ImportError as e:
        logger.critical(e)
        logger.critical(sys.path)
        logger.critical("Make sure LD_LIBRARY_PATH is set correctly and that the build"+\
                " directory is populated by waf.")
        sys.exit()

    if 'WORKHASH' in os.environ:
        import redis
        r = redis.StrictRedis(redis_server)

    if False: #WBCD Data
        iters = 3e5
        data = np.loadtxt('data/WBCD2.dat', np.int)
        data[:,:-1] -= 1
        states = np.array([10]*9 + [2],dtype=np.int)

        b,s = sample(states, data, iters=iters)
        plotHist(s)

    if True:

        N = 5
        iters = 1e4
        numdata = 50
        priorweight = 5
        numtemplate = 5

        random.seed(123456)
        np.random.seed(123456)

        g1 = generateHourGlassGraph(nodes=N)
        #print sp.version.version

        #d1, states1, joint1 = generateData(g1,numdata,method='noisylogic')
        d1, states1, joint1 = generateData(g1,numdata,method='dirichlet')
        #print "Ground joint: \n", joint1
        #print "Effective sample size: %d" % len(set(map(tuple,d1)))

        t1 = sampleTemplate(g1, numtemplate)

        #drawGraphs(g1)

        ###############################################
        #cProfile.runctx("sample(states1, d1, joint1, template=t1,iters=iters, priorweight=priorweight, burn=0, stepscale=40000)", globals(),locals(), "prof.prof")

        #s = pstats.Stats('prof.prof')
        #s.strip_dirs().sort_stats("time").print_stats()

        #with statprof.profile():
            #b,s = sample(states1, d1, g1, template=t1,
                    #iters=iters, priorweight=priorweight, burn=0, stepscale=40000)

        ###############################################
        b,s = sample(states1, d1, joint1, template=t1,
                iters=iters, priorweight=priorweight, burn=0, stepscale=30000, temperature=10)
    if 'WORKHASH' in os.environ:
        mean = s.estimate_func_mean()
        r.lpush('jobs:done:'+os.environ['WORKHASH'], mean)

        #es = []
        #newes = []
        #for i in range(20):
            #es.append(s.estimate_func_mean())
            #newes.append(s.estimate_func_mean(trunc=True))
            #s.sample(1e4)
        ###############################################

        #print "##### wotemp - wtemp = %f - %f = %f #######" % (mean1, mean2, mean1-mean2)

        #b.update_graph(s.mapvalue)
        #b2.update_graph(s2.mapvalue)
        #drawGraphs(g1, t1)

        #val1 = b.global_edge_presence()
        #val2 = b2.global_edge_presence()

        #print('Global edge error: %f' % mean1)
        #print('Map edge error: %f' % val1)

<|endoftext|>


<commit_before>import requests
from .util import *
import json

API_ENDPOINT = "https://api.pipedrive.com/v1"

class BaseResource(object):
    RESOURCE = "resources"
    RESOURCE_SEGMENT = RESOURCE + "s"
    FIELD_SEGMENT = RESOURCE + "Fields"
    HAS_CUSTOM_FIELDS = False

    def __init__(self, client, id):
        self._client = client
        self._resource = self.RESOURCE
        self.id = id
        self._data_cache = {}
        self.url = None
        self._fields = None
        self._dirty_fields = set()

        if self._resource not in self._client.fields:
            self._client.load_fields_for_resource(self._resource)
        self._init_done = True

    def __getattr__(self, name):

        attr = self._name_to_attr(name)

        if attr in self._data:
            value = self._data[attr]
            ftype = self.field_config[attr]["type"]
            if ftype in self._client.FIELD_TO_CLASS:
                print("working on %s" % value)
                # if type is mappable, map it
                resource_class = self._client.FIELD_TO_CLASS[ftype]
                print("type is mappable: %s to %s" % (ftype, resource_class))
                rid = value["value"]
                res = resource_class(self._client, rid)
                return res
            else:
                return value
        else:
            raise AttributeError

    def __setattr__(self, name, value):
        # print("Setting %s to %s" % (name, value))
        if "_init_done" not in self.__dict__ or name in self.__dict__:
            # use default setattr
            object.__setattr__(self, name, value)
        elif name in self.field_names:
            # custom field, let's set it
            attr = self._name_to_attr(name)
            if not self._data_cache[attr] == value:
                self._data_cache[attr] = value
                self._dirty_fields.add(name)
        else:
            raise AttributeError("Can't set propery: attribute %s \
                                          not found." % (name))

    def _name_to_attr(self, name):
        "Convert property name in pipedrive's object internal key."
        if self.HAS_CUSTOM_FIELDS and name in self.field_names:
            attr = self.field_names[name]
        else:
            attr = name
        return attr
        # Check if test on HAS_CUSTOM_FIELDS is required

    def save(self):
        "Save data back to pipedrive"
        if not self._dirty_fields:
            debug("No dirty fields for object %s" % self.id)
            return
        debug("Saving dirty fields: %s" % self._dirty_fields)
        data_for_update = {}
        for f in self._dirty_fields:
            attr = self._name_to_attr(f)
            data_for_update[attr] = self._data[attr]
        self._client.update_resource(self.RESOURCE_SEGMENT, self.id,
                                     data_for_update)
        self._dirty_fields.clear()
        self._data.clear()

    @property
    def _data(self):
        if not self._data_cache:
            (r, data) = self._client._fetch_resource_by_id(
                self.RESOURCE_SEGMENT, self.id)
            self.url = r.url
            self._data_cache = data

        return self._data_cache

    @property
    def field_keys(self):
        if not self.HAS_CUSTOM_FIELDS:
            return []
        return self._client.fields[self._resource]["key_to_attr"]

    @property
    def field_names(self):
        if not self.HAS_CUSTOM_FIELDS:
            return []
        return self._client.fields[self._resource]["attr_to_key"]

    @property
    def field_config(self):
        if not self.HAS_CUSTOM_FIELDS:
            return []
        return self._client.fields[self._resource]["config"]

class Person(BaseResource):
    RESOURCE = "person"
    RESOURCE_SEGMENT = RESOURCE + "s"
    FIELD_SEGMENT = RESOURCE + "Fields"
    HAS_CUSTOM_FIELDS = True

class Deal(BaseResource):
    RESOURCE = "deal"
    RESOURCE_SEGMENT = RESOURCE + "s"
    FIELD_SEGMENT = RESOURCE + "Fields"
    HAS_CUSTOM_FIELDS = True

class Organization(BaseResource):
    RESOURCE = "organization"
    RESOURCE_SEGMENT = RESOURCE + "s"
    FIELD_SEGMENT = RESOURCE + "Fields"
    HAS_CUSTOM_FIELDS = True

class Product(BaseResource):
    RESOURCE = "product"
    RESOURCE_SEGMENT = RESOURCE + "s"
    HAS_CUSTOM_FIELDS = True
    FIELD_SEGMENT = RESOURCE + "Fields"

class User(BaseResource):
    RESOURCE = "user"
    RESOURCE_SEGMENT = RESOURCE + "s"
    HAS_CUSTOM_FIELDS = False

class Acivity(BaseResource):
    RESOURCE = "activity"
    RESOURCE_SEGMENT = "activities"
    HAS_CUSTOM_FIELDS = False

class Stage(BaseResource):
    RESOURCE = "stage"
    RESOURCE_SEGMENT = RESOURCE + "s"
    HAS_CUSTOM_FIELDS = False

class PipedriveClient():

    FIELD_TO_CLASS = {
        "people": Person,
        "org": Organization,
        "stage": Stage,
        "product": Product
    }

    def __init__(self, api_token):
        self.api_token = api_token
        self.endpoint = API_ENDPOINT
        self.fields = {}

    def get_contact(self, id):
        res = "persons"
        (r, contact) = self._fetch_resource_by_id(res, id)
        return contact

    def get_resource(self, id):
        res = "persons"
        (r, resource) = self._fetch_resource_by_id(res, id)
        return (r, resource)

    def _fetch_resource_by_id(self, resource, id):

        if id is None:
            raise Exception("Can't fetch, ID given is None.")

        return self._fetch_resource(resource, id)

    def person_search(self, text, email=False):
        pass

    def _fetch_resource(self, resource, rid=None):
        url = self._build_url(resource, rid)
        debug("Fetching resource %s (%s) at %s" % (rid, resource, url))
        r = requests.get(url)
        data = r.json()

        if "success" not in data:
            raise Exception("Bad Response - can't find success key.")

        if not data["success"]:
            msg = data.get("error", "no error message given")
            raise Exception("Pipedrive error - %s at %s" % (msg, r.url))

        if "data" not in data:
            raise Exception("Can't find data object in response: %s" % data)

        return (r, data["data"])

    def load_fields_for_resource(self, resource):

        debug("Loading fields for resource %s" % resource)
        fields = {}
        key_to_attr = {}
        attr_to_key = {}
        field_definition = {}

        (r, fdata) = self._fetch_resource(resource + "Fields")

        for f in fdata:
            key = f["key"]
            if len(key) == 40:
                # custom field, use a nicer name
                attr = to_snake_case(f["name"])
            else:
                # not a custom field, use key name
                attr = key
            key_to_attr[key] = attr
            attr_to_key[attr] = key
            field_definition[key] = {"type": f["field_type"],
                                     "editable": f["edit_flag"]}
            if "options" in f:
                field_definition["options"] = f["options"]

        fields["key_to_attr"] = key_to_attr
        fields["attr_to_key"] = attr_to_key
        fields["config"] = field_definition
        self.fields[resource] = fields

    def update_resource(self, resource, rid, data):
        url = self._build_url(resource, rid)
        # XXX todo Validate resource/id/data types
        debug("Updating resource %s (%s) at %s" % (rid, resource, url))
        headers = {"Content-Type": "application/json"}
        r = requests.put(url, data=json.dumps(data), headers=headers)
        debug("Update status code: %s" % r.status_code)
        r.raise_for_status()
        return

    def _build_url(self, resource, rid=None):
        url = self.endpoint + "/" + resource
        if rid is not None:
            url = url + "/" + str(rid)
        url = url + "?api_token=%s" % (self.api_token)
        return url
<commit_msg>use make sure to load data before setting attribute, do string comparison to verify value changed<commit_after>import requests
from .util import *
import json

API_ENDPOINT = "https://api.pipedrive.com/v1"

class BaseResource(object):
    RESOURCE = "resources"
    RESOURCE_SEGMENT = RESOURCE + "s"
    FIELD_SEGMENT = RESOURCE + "Fields"
    HAS_CUSTOM_FIELDS = False

    def __init__(self, client, id):
        self._client = client
        self._resource = self.RESOURCE
        self.id = id
        self._data_cache = {}
        self.url = None
        self._fields = None
        self._dirty_fields = set()

        if self._resource not in self._client.fields:
            self._client.load_fields_for_resource(self._resource)
        self._init_done = True

    def __getattr__(self, name):

        attr = self._name_to_attr(name)

        if attr in self._data:
            value = self._data[attr]
            ftype = self.field_config[attr]["type"]
            if ftype in self._client.FIELD_TO_CLASS:
                print("working on %s" % value)
                # if type is mappable, map it
                resource_class = self._client.FIELD_TO_CLASS[ftype]
                print("type is mappable: %s to %s" % (ftype, resource_class))
                rid = value["value"]
                res = resource_class(self._client, rid)
                return res
            else:
                return value
        else:
            raise AttributeError

    def __setattr__(self, name, value):
        # print("Setting %s to %s" % (name, value))
        if "_init_done" not in self.__dict__ or name in self.__dict__:
            # use default setattr
            object.__setattr__(self, name, value)
        elif name in self.field_names:
            # custom field, let's set it
            attr = self._name_to_attr(name)

            # Set value only if value has actually changed
            if not str(self._data[attr]) == str(value):
                self._data_cache[attr] = value
                self._dirty_fields.add(name)
        else:
            raise AttributeError("Can't set propery: attribute %s \
                                          not found." % (name))

    def _name_to_attr(self, name):
        "Convert property name in pipedrive's object internal key."
        if self.HAS_CUSTOM_FIELDS and name in self.field_names:
            attr = self.field_names[name]
        else:
            attr = name
        return attr
        # Check if test on HAS_CUSTOM_FIELDS is required

    def save(self):
        "Save data back to pipedrive"
        if not self._dirty_fields:
            debug("No dirty fields for object %s" % self.id)
            return
        debug("Saving dirty fields: %s" % self._dirty_fields)
        data_for_update = {}
        for f in self._dirty_fields:
            attr = self._name_to_attr(f)
            data_for_update[attr] = self._data[attr]
        self._client.update_resource(self.RESOURCE_SEGMENT, self.id,
                                     data_for_update)
        self._dirty_fields.clear()
        self._data.clear()

    @property
    def _data(self):
        if not self._data_cache:
            (r, data) = self._client._fetch_resource_by_id(
                self.RESOURCE_SEGMENT, self.id)
            self.url = r.url
            self._data_cache = data

        return self._data_cache

    @property
    def field_keys(self):
        if not self.HAS_CUSTOM_FIELDS:
            return []
        return self._client.fields[self._resource]["key_to_attr"]

    @property
    def field_names(self):
        if not self.HAS_CUSTOM_FIELDS:
            return []
        return self._client.fields[self._resource]["attr_to_key"]

    @property
    def field_config(self):
        if not self.HAS_CUSTOM_FIELDS:
            return []
        return self._client.fields[self._resource]["config"]

class Person(BaseResource):
    RESOURCE = "person"
    RESOURCE_SEGMENT = RESOURCE + "s"
    FIELD_SEGMENT = RESOURCE + "Fields"
    HAS_CUSTOM_FIELDS = True

class Deal(BaseResource):
    RESOURCE = "deal"
    RESOURCE_SEGMENT = RESOURCE + "s"
    FIELD_SEGMENT = RESOURCE + "Fields"
    HAS_CUSTOM_FIELDS = True

class Organization(BaseResource):
    RESOURCE = "organization"
    RESOURCE_SEGMENT = RESOURCE + "s"
    FIELD_SEGMENT = RESOURCE + "Fields"
    HAS_CUSTOM_FIELDS = True

class Product(BaseResource):
    RESOURCE = "product"
    RESOURCE_SEGMENT = RESOURCE + "s"
    HAS_CUSTOM_FIELDS = True
    FIELD_SEGMENT = RESOURCE + "Fields"

class User(BaseResource):
    RESOURCE = "user"
    RESOURCE_SEGMENT = RESOURCE + "s"
    HAS_CUSTOM_FIELDS = False

class Acivity(BaseResource):
    RESOURCE = "activity"
    RESOURCE_SEGMENT = "activities"
    HAS_CUSTOM_FIELDS = False

class Stage(BaseResource):
    RESOURCE = "stage"
    RESOURCE_SEGMENT = RESOURCE + "s"
    HAS_CUSTOM_FIELDS = False

class PipedriveClient():

    FIELD_TO_CLASS = {
        "people": Person,
        "org": Organization,
        "stage": Stage,
        "product": Product
    }

    def __init__(self, api_token):
        self.api_token = api_token
        self.endpoint = API_ENDPOINT
        self.fields = {}

    def get_contact(self, id):
        res = "persons"
        (r, contact) = self._fetch_resource_by_id(res, id)
        return contact

    def get_resource(self, id):
        res = "persons"
        (r, resource) = self._fetch_resource_by_id(res, id)
        return (r, resource)

    def _fetch_resource_by_id(self, resource, id):

        if id is None:
            raise Exception("Can't fetch, ID given is None.")

        return self._fetch_resource(resource, id)

    def person_search(self, text, email=False):
        pass

    def _fetch_resource(self, resource, rid=None):
        url = self._build_url(resource, rid)
        debug("Fetching resource %s (%s) at %s" % (rid, resource, url))
        r = requests.get(url)
        data = r.json()

        if "success" not in data:
            raise Exception("Bad Response - can't find success key.")

        if not data["success"]:
            msg = data.get("error", "no error message given")
            raise Exception("Pipedrive error - %s at %s" % (msg, r.url))

        if "data" not in data:
            raise Exception("Can't find data object in response: %s" % data)

        return (r, data["data"])

    def load_fields_for_resource(self, resource):

        debug("Loading fields for resource %s" % resource)
        fields = {}
        key_to_attr = {}
        attr_to_key = {}
        field_definition = {}

        (r, fdata) = self._fetch_resource(resource + "Fields")

        for f in fdata:
            key = f["key"]
            if len(key) == 40:
                # custom field, use a nicer name
                attr = to_snake_case(f["name"])
            else:
                # not a custom field, use key name
                attr = key
            key_to_attr[key] = attr
            attr_to_key[attr] = key
            field_definition[key] = {"type": f["field_type"],
                                     "editable": f["edit_flag"]}
            if "options" in f:
                field_definition["options"] = f["options"]

        fields["key_to_attr"] = key_to_attr
        fields["attr_to_key"] = attr_to_key
        fields["config"] = field_definition
        self.fields[resource] = fields

    def update_resource(self, resource, rid, data):
        url = self._build_url(resource, rid)
        # XXX todo Validate resource/id/data types
        debug("Updating resource %s (%s) at %s" % (rid, resource, url))
        headers = {"Content-Type": "application/json"}
        r = requests.put(url, data=json.dumps(data), headers=headers)
        debug("Update status code: %s" % r.status_code)
        r.raise_for_status()
        return

    def _build_url(self, resource, rid=None):
        url = self.endpoint + "/" + resource
        if rid is not None:
            url = url + "/" + str(rid)
        url = url + "?api_token=%s" % (self.api_token)
        return url
<|endoftext|>


<commit_before>

class MalformedFileError(Exception):
    """Error raised when the file is not well formatted"""
<commit_msg>Add new exception for malformed captions<commit_after>

class MalformedFileError(Exception):
    """Error raised when the file is not well formatted"""

class MalformedCaptionError(Exception):
    """Error raised when a caption is not well formatted"""
<|endoftext|>


<commit_before># coding: utf-8

""" Description... """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys

# Third-party
import numpy as np
import astropy.units as u
from astropy.constants import G

from ..potential import *

__all__ = ["minimum_distance_matrix",
           "relative_normalized_coordinates",
           "generalized_variance"]

def relative_normalized_coordinates(potential, particle_orbits, satellite_orbit):
    """ Compute the coordinates of particles relative to the satellite,
        with positions normalized by the tidal radius and velocities
        normalized by the escape velocity.

        Note::
            Assumes the particle and satellite position/velocity
            units are the same!

        Parameters
        ----------
        potential : streams.CartesianPotential
        particle_orbits : OrbitCollection
        satellite_orbit : OrbitCollection
    """

    # need to add a new axis to normalize each coordinate component
    r_tide = potential._tidal_radius(m=satellite_orbit._m,
                                     r=satellite_orbit._r)[:,np.newaxis,:]
    v_esc = potential._escape_velocity(m=satellite_orbit._m,
                                       r_tide=r_tide)

    return (particle_orbits._r - satellite_orbit._r) / (5.*r_tide), \
           (particle_orbits._v - satellite_orbit._v) / v_esc

def minimum_distance_matrix(potential, particle_orbits, satellite_orbit):
    """ Compute the Nx6 matrix of minimum phase-space distance vectors.

        Parameters
        ----------
        potential : Potential
        particle_orbits : OrbitCollection
        satellite_orbit : OrbitCollection
    """

    Nparticles = particle_orbits._r.shape[1]

    R,V = relative_normalized_coordinates(potential, particle_orbits, satellite_orbit)
    D_ps = np.sqrt(np.sum(R**2, axis=-1) + np.sum(V**2, axis=-1))

    # Find the index of the time of the minimum D_ps for each particle
    min_time_idx = D_ps.argmin(axis=0)

    min_ps = np.zeros((Nparticles,6))
    xx = zip(min_time_idx, range(Nparticles))
    for kk in range(Nparticles):
        jj,ii = xx[kk]
        min_ps[ii] = np.append(R[jj,ii], V[jj,ii])

    return min_ps

def generalized_variance(potential, particle_orbits, satellite_orbit):
    """ Compute the variance scalar that we will minimize.

        Parameters
        ----------
        potential : Potential
            The full Milky Way potential object.
        particle_orbits : OrbitCollection
            An object containing orbit information for a collection of
            particles.
        satellite_orbit : OrbitCollection
            Data for the Sgr satellite center, interpolated onto the
            time grid for our particles.
    """

    min_ps = minimum_distance_matrix(potential, particle_orbits, satellite_orbit)

    cov_matrix = np.cov(min_ps.T)

    # cov_matrix -> (6 x 6) covariance matrix for particles
    w,v = np.linalg.eig(cov_matrix)
    #return np.sum(w)
    return np.prod(w)

    sign,logdet = np.linalg.slogdet(cov_matrix)
    return logdet<commit_msg>log of prod<commit_after># coding: utf-8

""" Description... """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys

# Third-party
import numpy as np
import astropy.units as u
from astropy.constants import G

from ..potential import *

__all__ = ["minimum_distance_matrix",
           "relative_normalized_coordinates",
           "generalized_variance"]

def relative_normalized_coordinates(potential, particle_orbits, satellite_orbit):
    """ Compute the coordinates of particles relative to the satellite,
        with positions normalized by the tidal radius and velocities
        normalized by the escape velocity.

        Note::
            Assumes the particle and satellite position/velocity
            units are the same!

        Parameters
        ----------
        potential : streams.CartesianPotential
        particle_orbits : OrbitCollection
        satellite_orbit : OrbitCollection
    """

    # need to add a new axis to normalize each coordinate component
    r_tide = potential._tidal_radius(m=satellite_orbit._m,
                                     r=satellite_orbit._r)[:,:,np.newaxis]
    v_esc = potential._escape_velocity(m=satellite_orbit._m,
                                       r_tide=r_tide)

    return (particle_orbits._r - satellite_orbit._r) / (5.*r_tide), \
           (particle_orbits._v - satellite_orbit._v) / v_esc

def minimum_distance_matrix(potential, particle_orbits, satellite_orbit):
    """ Compute the Nx6 matrix of minimum phase-space distance vectors.

        Parameters
        ----------
        potential : Potential
        particle_orbits : OrbitCollection
        satellite_orbit : OrbitCollection
    """

    Nparticles = particle_orbits._r.shape[1]

    R,V = relative_normalized_coordinates(potential, particle_orbits, satellite_orbit)
    D_ps = np.sqrt(np.sum(R**2, axis=-1) + np.sum(V**2, axis=-1))

    # Find the index of the time of the minimum D_ps for each particle
    min_time_idx = D_ps.argmin(axis=0)

    min_ps = np.zeros((Nparticles,6))
    xx = zip(min_time_idx, range(Nparticles))
    for kk in range(Nparticles):
        jj,ii = xx[kk]
        min_ps[ii] = np.append(R[jj,ii], V[jj,ii])

    return min_ps

def generalized_variance(potential, particle_orbits, satellite_orbit):
    """ Compute the variance scalar that we will minimize.

        Parameters
        ----------
        potential : Potential
            The full Milky Way potential object.
        particle_orbits : OrbitCollection
            An object containing orbit information for a collection of
            particles.
        satellite_orbit : OrbitCollection
            Data for the Sgr satellite center, interpolated onto the
            time grid for our particles.
    """

    min_ps = minimum_distance_matrix(potential, particle_orbits, satellite_orbit)

    cov_matrix = np.cov(min_ps.T)

    # cov_matrix -> (6 x 6) covariance matrix for particles
    w,v = np.linalg.eig(cov_matrix)
    #return np.sum(w)
    return np.log(np.prod(w))

    sign,logdet = np.linalg.slogdet(cov_matrix)
    return logdet<|endoftext|>


<commit_before>"""
Functions described here are for python 2/3 compability and other tasks.
"""

from .compat import urlparse, urlencode, urljoin, parse_qsl

import io
import json
import sys

def read_config(config_file, parser):
    with io.open(config_file, 'r', encoding='utf-8') as f:
        values = json.load(f)

    return values['options'], values['arguments']

def write_json(obj, filename):
    if sys.version_info >= (3, 0):
        with open(filename, 'w+', encoding='utf-8') as f:
            json.dump(obj, f)
    else:
        # In Python 2.x, json.dump expects a bytestream
        with open(filename, 'wb') as f:
            json.dump(obj, f)

def write_config(filename, options, arguments):
    write_json({"options": options.__dict__, "arguments": arguments}, filename)

def modify_parameter(parameter, target_name, value):
    parameter[target_name] = value
    return parameter

def change_parameter(url, parameter, new_value):
    """ Returns a new url where the parameter is changed. """
    url_query = urlparse(url).query
    query = dict(parse_qsl(url_query))

    if query:
        for name, value in query.items():
            if name == parameter:
                query[name] = new_value

        encoded = "?" + urlencode(query)
        return urljoin(url, encoded)
    else:
        return url

def get_url_host(url):
    """ Returns the server of a name."""
    return urlparse(url).netloc

def get_page_text(page):
    if page.document.text:
        yield page.document.text

    for element in page.document.findall('.//*'):
        if element.text:
            yield element.text
<commit_msg>Write configuration file with indentation<commit_after>"""
Functions described here are for python 2/3 compability and other tasks.
"""

from .compat import urlparse, urlencode, urljoin, parse_qsl

import io
import json
import sys

def read_config(config_file, parser):
    with io.open(config_file, 'r', encoding='utf-8') as f:
        values = json.load(f)

    return values['options'], values['arguments']

def write_json(obj, filename, **kwargs):
    if sys.version_info >= (3, 0):
        with open(filename, 'w+', encoding='utf-8') as f:
            json.dump(obj, f, **kwargs)
    else:
        # In Python 2.x, json.dump expects a bytestream
        with open(filename, 'wb') as f:
            json.dump(obj, f, **kwargs)

def write_config(filename, options, arguments):
    write_json({"options": options.__dict__, "arguments": arguments}, filename,
               indent=4)

def modify_parameter(parameter, target_name, value):
    parameter[target_name] = value
    return parameter

def change_parameter(url, parameter, new_value):
    """ Returns a new url where the parameter is changed. """
    url_query = urlparse(url).query
    query = dict(parse_qsl(url_query))

    if query:
        for name, value in query.items():
            if name == parameter:
                query[name] = new_value

        encoded = "?" + urlencode(query)
        return urljoin(url, encoded)
    else:
        return url

def get_url_host(url):
    """ Returns the server of a name."""
    return urlparse(url).netloc

def get_page_text(page):
    if page.document.text:
        yield page.document.text

    for element in page.document.findall('.//*'):
        if element.text:
            yield element.text
<|endoftext|>