<commit_before>#!/usr/bin/env python3
#
# igcollect - Switch stats:
#             - port traffic
#             - port errors
#             - cpu usage
#
# Copyright (c) 2017, InnoGames GmbH
#

from argparse import ArgumentParser
from time import time
from pysnmp import proto, error
from pysnmp.entity.rfc3413.oneliner import cmdgen
from pysnmp.entity.rfc3413.oneliner.cmdgen import (
        CommunityData,
        UsmUserData,
        usmHMACSHAAuthProtocol,
        usmAesCfb128Protocol,
        usmDESPrivProtocol,
)
import re
import sys

# Predefine some variables, it makes this program run a bit faster.
cmd_gen = cmdgen.CommandGenerator()

OIDS = {
    'switch_model': '.1.3.6.1.2.1.1.1.0',
    'port_name': '.1.3.6.1.2.1.31.1.1.1.1',
    'port_state': '1.3.6.1.2.1.2.2.1.8',
}

LAGG_OIDS = {
    'procurve': '.1.3.6.1.4.1.11.2.14.11.5.1.7.1.3.1.1.8',
    'powerconnect': '.1.2.840.10006.300.43.1.2.1.1.12',
}

CPU_OIDS = {
    'procurve': '.1.3.6.1.4.1.11.2.14.11.5.1.9.6.1.0',
    'powerconnect': '.1.3.6.1.4.1.674.10895.5000.2.6132.1.1.1.1.4.9.0',
    'extreme': '.1.3.6.1.4.1.1916.1.32.1.2.0',
    # 1-minute average for the 1st cpu of stack because we don't stack them.
    'force10_mxl': '1.3.6.1.4.1.6027.3.26.1.4.4.1.4.2.1.1',
}

COUNTERS = {
    'bytesIn': '.1.3.6.1.2.1.31.1.1.1.6',
    'bytesOut': '.1.3.6.1.2.1.31.1.1.1.10',
    'pktsIn': '.1.3.6.1.2.1.31.1.1.1.7',
    'pktsOut': '.1.3.6.1.2.1.31.1.1.1.11',
    'brdPktsIn': '.1.3.6.1.2.1.31.1.1.1.9',
    'brdPktsOut': '.1.3.6.1.2.1.31.1.1.1.13',
    'ifInErrors': '.1.3.6.1.2.1.2.2.1.14',
    'ifOutErrors': '.1.3.6.1.2.1.2.2.1.20',
    'ifInDiscards': '.1.3.6.1.2.1.2.2.1.13',
    'ifOutDiscards': '.1.3.6.1.2.1.2.2.1.19',
}

class SwitchException(Exception):
    pass

def main():
    args = parse_args()
    if not args.prefix:
        args.prefix = 'switches.{}'.format(args.switch)
    try:
        snmp = get_snmp_connection(args)
    except error.PySnmpError as e:
        print(e, file=sys.stderr)
        return -1

    try:
        model = get_switch_model(snmp)
    except SwitchException as e:
        print(e, file=sys.stderr)
        return -1

    if not model:
        return -1

    cpu_stats(args.prefix, snmp, model)
    monitored_ports = get_monitored_ports(snmp, model)
    ports_stats(args.prefix, snmp, monitored_ports)

def parse_args():
    parser = ArgumentParser()
    parser.add_argument('switch', type=str, help='Hostname of a switch')
    parser.add_argument('--prefix', help='Graphite prefix')

    snmp_mode = parser.add_mutually_exclusive_group(required=True)
    snmp_mode.add_argument('--community', help='SNMP community')
    snmp_mode.add_argument('--user', help='SNMPv3 user')

    parser.add_argument('--auth', help='SNMPv3 authentication key')
    parser.add_argument('--priv', help='SNMPv3 privacy key')
    parser.add_argument(
            '--priv_proto',
            help='SNMPv3 privacy protocol: aes (default) or des',
            default='aes'
    )
    return parser.parse_args()

def get_snmp_connection(args):
    """ Prepare SNMP transport agent.

        Connection over SNMP v2c and v3 is supported.
        The choice of authentication and privacy algorithms for v3 is
        arbitrary, matching what our switches can do.
    """

    if args.community:
        auth_data = CommunityData(args.community, mpModel=1)
    else:
        if args.priv_proto == 'des':
            priv_proto = usmDESPrivProtocol
        if args.priv_proto == 'aes':
            priv_proto = usmAesCfb128Protocol

        auth_data = UsmUserData(
            args.user, args.auth, args.priv,
            authProtocol=usmHMACSHAAuthProtocol,
            privProtocol=priv_proto,
        )

    transport_target = cmdgen.UdpTransportTarget((args.switch, 161))

    return {
        'auth_data': auth_data,
        'transport_target': transport_target,
    }

def get_snmp_value(snmp, OID):
    """ Get a single value from SNMP """

    errorIndication, errorStatus, errorIndex, varBinds = cmd_gen.getCmd(
        snmp['auth_data'],
        snmp['transport_target'],
        OID,
    )
    if errorIndication:
        raise SwitchException('Unable to get SNMP value: {}'.
                              format(errorIndication))

    return convert_snmp_type(varBinds)

def get_snmp_table(snmp, OID):
    """ Fetch a table from SNMP.

        Returned is a dictionary mapping the last number of OID (converted to
        Python integer) to value (converted to int or str).
    """

    ret = {}
    errorIndication, errorStatus, errorIndex, varBindTable = cmd_gen.bulkCmd(
        snmp['auth_data'],
        snmp['transport_target'],
        0,
        25,
        OID,
    )
    for varBind in varBindTable:
        if errorIndication:
            raise SwitchException('Unable to get SNMP value: {}'.
                                  format(errorIndication))
        index = int(str(varBind[0][0][-1:]))
        ret[index] = convert_snmp_type(varBind)

    return ret

def convert_snmp_type(varBinds):
    """ Convert SNMP data types to something more convenient: int or str """

    val = varBinds[0][1]
    if type(val) in [
        proto.rfc1902.Integer,
        proto.rfc1902.Counter32,
        proto.rfc1902.Counter64,
    ]:
        return int(val)
    return str(val)

def get_switch_model(snmp):
    """ Recognize model of switch from SNMP MIB-2 sysDescr """

    model = get_snmp_value(snmp, OIDS['switch_model'])

    if 'PowerConnect' in model:
        return 'powerconnect'
    elif 'ProCurve' in model:
        return 'procurve'
    elif 'ExtremeXOS' in model:
        return'extreme'
    elif 'Dell Networking OS' in model:
        return 'force10_mxl'

    print('Unknown switch model {}'.format(model), file=sys.stderr)
    return None

def get_monitored_ports(snmp, model):
    """ Get ports which meet the following conditions:
        - are configured to be no shutdown
        - and don't belong to a LAGG
        - or are a configured LAGG

        SNMP works in misterious ways: it is faster to fetch a whole table
        and then operate in this program than to fetch just a few entries.
    """

    ret = {}

    port_names = get_snmp_table(snmp, OIDS['port_name'])
    laggs = get_laggs(snmp, model)

    # Get only those ports which are up.
    port_states = {
        x: y for x, y
        in get_snmp_table(snmp, OIDS['port_state']).items()
        if y == 1
    }
    for port_idx, port_state in port_states.items():
        if not laggs or port_idx not in laggs.keys():
            port_name = standarize_portname(port_names[port_idx])
            if port_name:
                ret[port_idx] = port_name

    return ret

def get_laggs(snmp, model):
    """ Get only those LAGGs which have members """

    if model in ['powerconnect', 'procurve']:
        return {
            x: y for x, y
            in get_snmp_table(snmp, LAGG_OIDS[model]).items()
            if y != 0
        }

    return None

def standarize_portname(port_name):
    """ Return a Graphite-compatible port name or None if name
        can't be translated
    """

    if re.match('\A(Po|Trk)[0-9]+\Z', port_name):
        # Dell or HP LAGG
        return port_name
    if re.match('\A[0-9]+\Z', port_name):
        # HP normal port
        return port_name
    if re.match('\A[0-9]:[0-9]+\Z', port_name):
        # Extreme normal port
        return port_name.replace(':', '_')
    if re.match('\A(Gi|Te)[0-9]/[0-9]/[0-9]+\Z', port_name):
        # Dell normal port
        return port_name.replace('/', '_')
    g = re.match(
        '\A(TenGigabitEthernet|fortyGigE) ([0-9]+/[0-9]+)\Z',
        port_name
    )
    if g:
        # Force 10 MXL port
        return g.group(2).replace('/', '_')

    return None

def ports_stats(prefix, snmp, ports):
    """ Print graphite-compatible stats for each port of switch """

    for counter, oid in COUNTERS.items():
        # SNMP is slow that we want need to get current time for each counter.
        template = prefix + '.ports.{}.{} {} ' + str(int(time()))
        table = get_snmp_table(snmp, oid)
        for port_idx, port_name in ports.items():
            if port_idx in table:
                print(template.format(port_name, counter, table[port_idx]))

def cpu_stats(prefix, snmp, model):
    """ Print graphite-compatible stats of switch CPU

        We use single OID which should percentage of CPU time used.
    """

    cpu_usage = get_snmp_value(snmp, CPU_OIDS[model])

    if model == 'powerconnect':
        # SNMP returns such ugly string
        #     5 Secs ( 18.74%)    60 Secs ( 17.84%)   300 Secs ( 18.12%)
        m = re.search('60 Secs \( *([0-9]+)[0-9\.]*%\)', cpu_usage)
        cpu_usage = int(m.group(1))

    template = prefix + '.cpu {} ' + str(int(time()))
    print(template.format(cpu_usage))

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Support Cisco routers<commit_after>#!/usr/bin/env python3
#
# igcollect - Switch stats:
#             - port traffic
#             - port errors
#             - cpu usage
#
# Copyright (c) 2017, InnoGames GmbH
#

from argparse import ArgumentParser
from time import time
from pysnmp import proto, error
from pysnmp.entity.rfc3413.oneliner import cmdgen
from pysnmp.entity.rfc3413.oneliner.cmdgen import (
        CommunityData,
        UsmUserData,
        usmHMACSHAAuthProtocol,
        usmAesCfb128Protocol,
        usmDESPrivProtocol,
)
import re
import sys

# Predefine some variables, it makes this program run a bit faster.
cmd_gen = cmdgen.CommandGenerator()

OIDS = {
    'switch_model': '.1.3.6.1.2.1.1.1.0',
    'port_name': '.1.3.6.1.2.1.31.1.1.1.1',
    'port_state': '1.3.6.1.2.1.2.2.1.8',
}

LAGG_OIDS = {
    'procurve': '.1.3.6.1.4.1.11.2.14.11.5.1.7.1.3.1.1.8',
    'powerconnect': '.1.2.840.10006.300.43.1.2.1.1.12',
}

CPU_OIDS = {
    'procurve': '.1.3.6.1.4.1.11.2.14.11.5.1.9.6.1.0',
    'powerconnect': '.1.3.6.1.4.1.674.10895.5000.2.6132.1.1.1.1.4.9.0',
    'extreme': '.1.3.6.1.4.1.1916.1.32.1.2.0',
    # 1-minute average for the 1st cpu of stack because we don't stack them.
    'force10_mxl': '1.3.6.1.4.1.6027.3.26.1.4.4.1.4.2.1.1',
    'cisco_ios': '1.3.6.1.4.1.9.9.109.1.1.1.1.4.1',
}

COUNTERS = {
    'bytesIn': '.1.3.6.1.2.1.31.1.1.1.6',
    'bytesOut': '.1.3.6.1.2.1.31.1.1.1.10',
    'pktsIn': '.1.3.6.1.2.1.31.1.1.1.7',
    'pktsOut': '.1.3.6.1.2.1.31.1.1.1.11',
    'brdPktsIn': '.1.3.6.1.2.1.31.1.1.1.9',
    'brdPktsOut': '.1.3.6.1.2.1.31.1.1.1.13',
    'ifInErrors': '.1.3.6.1.2.1.2.2.1.14',
    'ifOutErrors': '.1.3.6.1.2.1.2.2.1.20',
    'ifInDiscards': '.1.3.6.1.2.1.2.2.1.13',
    'ifOutDiscards': '.1.3.6.1.2.1.2.2.1.19',
}

class SwitchException(Exception):
    pass

def main():
    args = parse_args()
    if not args.prefix:
        args.prefix = 'switches.{}'.format(args.switch)
    try:
        snmp = get_snmp_connection(args)
    except error.PySnmpError as e:
        print(e, file=sys.stderr)
        return -1

    try:
        model = get_switch_model(snmp)
    except SwitchException as e:
        print(e, file=sys.stderr)
        return -1

    if not model:
        return -1

    cpu_stats(args.prefix, snmp, model)
    monitored_ports = get_monitored_ports(snmp, model)
    ports_stats(args.prefix, snmp, monitored_ports)

def parse_args():
    parser = ArgumentParser()
    parser.add_argument('switch', type=str, help='Hostname of a switch')
    parser.add_argument('--prefix', help='Graphite prefix')

    snmp_mode = parser.add_mutually_exclusive_group(required=True)
    snmp_mode.add_argument('--community', help='SNMP community')
    snmp_mode.add_argument('--user', help='SNMPv3 user')

    parser.add_argument('--auth', help='SNMPv3 authentication key')
    parser.add_argument('--priv', help='SNMPv3 privacy key')
    parser.add_argument(
            '--priv_proto',
            help='SNMPv3 privacy protocol: aes (default) or des',
            default='aes'
    )
    return parser.parse_args()

def get_snmp_connection(args):
    """ Prepare SNMP transport agent.

        Connection over SNMP v2c and v3 is supported.
        The choice of authentication and privacy algorithms for v3 is
        arbitrary, matching what our switches can do.
    """

    if args.community:
        auth_data = CommunityData(args.community, mpModel=1)
    else:
        if args.priv_proto == 'des':
            priv_proto = usmDESPrivProtocol
        if args.priv_proto == 'aes':
            priv_proto = usmAesCfb128Protocol

        auth_data = UsmUserData(
            args.user, args.auth, args.priv,
            authProtocol=usmHMACSHAAuthProtocol,
            privProtocol=priv_proto,
        )

    transport_target = cmdgen.UdpTransportTarget((args.switch, 161))

    return {
        'auth_data': auth_data,
        'transport_target': transport_target,
    }

def get_snmp_value(snmp, OID):
    """ Get a single value from SNMP """

    errorIndication, errorStatus, errorIndex, varBinds = cmd_gen.getCmd(
        snmp['auth_data'],
        snmp['transport_target'],
        OID,
    )
    if errorIndication:
        raise SwitchException('Unable to get SNMP value: {}'.
                              format(errorIndication))

    return convert_snmp_type(varBinds)

def get_snmp_table(snmp, OID):
    """ Fetch a table from SNMP.

        Returned is a dictionary mapping the last number of OID (converted to
        Python integer) to value (converted to int or str).
    """

    ret = {}
    errorIndication, errorStatus, errorIndex, varBindTable = cmd_gen.bulkCmd(
        snmp['auth_data'],
        snmp['transport_target'],
        1,  # nonRepeaters must be 1 to work on Cisco!
        25,
        OID,
    )
    for varBind in varBindTable:
        if errorIndication:
            raise SwitchException('Unable to get SNMP value: {}'.
                                  format(errorIndication))
        index = int(str(varBind[0][0][-1:]))
        ret[index] = convert_snmp_type(varBind)

    return ret

def convert_snmp_type(varBinds):
    """ Convert SNMP data types to something more convenient: int or str """

    val = varBinds[0][1]
    if type(val) in [
        proto.rfc1902.Integer,
        proto.rfc1902.Counter32,
        proto.rfc1902.Counter64,
    ]:
        return int(val)
    return str(val)

def get_switch_model(snmp):
    """ Recognize model of switch from SNMP MIB-2 sysDescr """

    model = get_snmp_value(snmp, OIDS['switch_model'])

    if 'PowerConnect' in model:
        return 'powerconnect'
    elif 'ProCurve' in model:
        return 'procurve'
    elif 'ExtremeXOS' in model:
        return'extreme'
    elif 'Dell Networking OS' in model:
        return 'force10_mxl'
    elif 'Cisco IOS Software' in model:
        return 'cisco_ios'

    print('Unknown switch model {}'.format(model), file=sys.stderr)
    return None

def get_monitored_ports(snmp, model):
    """ Get ports which meet the following conditions:
        - are configured to be no shutdown
        - and don't belong to a LAGG
        - or are a configured LAGG

        SNMP works in misterious ways: it is faster to fetch a whole table
        and then operate in this program than to fetch just a few entries.
    """

    ret = {}

    port_names = get_snmp_table(snmp, OIDS['port_name'])
    laggs = get_laggs(snmp, model)

    # Get only those ports which are up.
    port_states = {
        x: y for x, y
        in get_snmp_table(snmp, OIDS['port_state']).items()
        if y == 1
    }
    for port_idx, port_state in port_states.items():
        if not laggs or port_idx not in laggs.keys():
            port_name = standarize_portname(port_names[port_idx])
            if port_name:
                ret[port_idx] = port_name

    return ret

def get_laggs(snmp, model):
    """ Get only those LAGGs which have members """

    if model in ['powerconnect', 'procurve']:
        return {
            x: y for x, y
            in get_snmp_table(snmp, LAGG_OIDS[model]).items()
            if y != 0
        }

    return None

def standarize_portname(port_name):
    """ Return a Graphite-compatible port name or None if name
        can't be translated
    """

    if re.match('\A(Po|Trk)[0-9]+\Z', port_name):
        # Dell or HP LAGG
        return port_name
    if re.match('\A[0-9]+\Z', port_name):
        # HP normal port
        return port_name
    if re.match('\A[0-9]:[0-9]+\Z', port_name):
        # Extreme normal port
        return port_name.replace(':', '_')
    if re.match('\A(Gi|Te)[0-9]/[0-9]/[0-9]+\Z', port_name):
        # Dell normal port
        return port_name.replace('/', '_')
    if re.match('\A(Fa|Gi|Tu)[0-9]/[0-9]+\Z', port_name):
        # Cisco ports
        return port_name.replace('/', '_')
    if re.match('\A(Fa|Gi|Tu)[0-9]+\Z', port_name):
        # Cisco tunnels
        return port_name
    g = re.match(
        '\A(TenGigabitEthernet|fortyGigE) ([0-9]+/[0-9]+)\Z',
        port_name
    )
    if g:
        # Force 10 MXL port
        return g.group(2).replace('/', '_')

    return None

def ports_stats(prefix, snmp, ports):
    """ Print graphite-compatible stats for each port of switch """

    for counter, oid in COUNTERS.items():
        # SNMP is slow that we want need to get current time for each counter.
        template = prefix + '.ports.{}.{} {} ' + str(int(time()))
        table = get_snmp_table(snmp, oid)
        for port_idx, port_name in ports.items():
            if port_idx in table:
                print(template.format(port_name, counter, table[port_idx]))

def cpu_stats(prefix, snmp, model):
    """ Print graphite-compatible stats of switch CPU

        We use single OID which should percentage of CPU time used.
    """

    cpu_usage = get_snmp_value(snmp, CPU_OIDS[model])

    if model == 'powerconnect':
        # SNMP returns such ugly string
        #     5 Secs ( 18.74%)    60 Secs ( 17.84%)   300 Secs ( 18.12%)
        m = re.search('60 Secs \( *([0-9]+)[0-9\.]*%\)', cpu_usage)
        cpu_usage = int(m.group(1))

    template = prefix + '.cpu {} ' + str(int(time()))
    print(template.format(cpu_usage))

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>version = '1.0.0'
<commit_msg>Bump version<commit_after>version = '1.0.1'
<|endoftext|>


<commit_before>from .base import PostProcessorBase
from .pdf import PDFPostProcessor
from .serve import ServePostProcessor
<commit_msg>protect against unavailable tornado in post_processors.init<commit_after>from .base import PostProcessorBase
from .pdf import PDFPostProcessor

# protect against unavailable tornado
try:
    from .serve import ServePostProcessor
except ImportError:
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
import cStringIO as StringIO
import flask
import random
from ibeis import ibsfuncs
from ibeis.constants import TAU
from flask import request, current_app
from os.path import join, dirname, abspath  # NOQA
from datetime import date
import base64
import jinja2
import utool as ut

DEFAULT_WEB_API_PORT = ut.get_argval('--port', type_=int, default=5000)
TARGET_WIDTH = 1200.0
PAGE_SIZE = 500
VALID_TURK_MODES = [
    ('turk_viewpoint', 'Viewpoint'),
    ('turk_quality', 'Quality'),
]

class NavbarClass(object):
    def __init__(nav):
        nav.item_list = [
            ('root', 'Home'),
            ('view', 'View'),
            ('turk', 'Turk'),
            ('api_root',  'API'),
            ('group_review',  'Group Review'),
        ]

    def __iter__(nav):
        _link = flask.request.path.strip('/').split('/')
        for link, nice in nav.item_list:
            yield link == _link[0], link, nice

def resize_via_web_parameters(image):
    w_pix = request.args.get('resize_pix_w',      request.form.get('resize_pix_w',      None ))
    h_pix = request.args.get('resize_pix_h',      request.form.get('resize_pix_h',      None ))
    w_per = request.args.get('resize_per_w',      request.form.get('resize_per_w',      None ))
    h_per = request.args.get('resize_per_h',      request.form.get('resize_per_h',      None ))
    _pix  = request.args.get('resize_prefer_pix', request.form.get('resize_prefer_pix', False))
    _per  = request.args.get('resize_prefer_per', request.form.get('resize_prefer_per', False))
    args = (w_pix, h_pix, w_per, h_per, _pix, _per, )
    print('CHECKING RESIZING WITH %r pix, %r pix, %r %%, %r %% [%r, %r]' % args)
    # Check for nothing
    if not (w_pix or h_pix or w_per or h_per):
        return image
    # Check for both pixels and images
    if (w_pix or h_pix) and (w_per or h_per):
        if _pix:
            w_per, h_per = None
        elif _per:
            w_pix, h_pix = None
        else:
            raise ValueError('Cannot resize using pixels and percentages, pick one')
    # Resize using percentages, transform to pixels
    if w_per:
        w_pix = float(w_per) * image.shape[1]
    if h_per:
        h_pix = float(h_per) * image.shape[0]
    # Perform resize
    return ibsfuncs._resize(image, t_width=w_pix, t_height=h_pix)

def embed_image_html(imgBGR, target_width=TARGET_WIDTH):
    """ Creates an image embedded in HTML base64 format. """
    import cv2
    from PIL import Image
    imgBGR = ibsfuncs._resize(imgBGR, target_width)
    imgRGB = cv2.cvtColor(imgBGR, cv2.COLOR_BGR2RGB)
    pil_img = Image.fromarray(imgRGB)
    string_buf = StringIO.StringIO()
    pil_img.save(string_buf, format='jpeg')
    data = string_buf.getvalue().encode('base64').replace('\n', '')
    return 'data:image/jpeg;base64,' + data

def check_valid_function_name(string):
    return all([ char.isalpha() or char == '_' or char.isalnum() for char in string])

def encode_refer_url(url):
    return base64.urlsafe_b64encode(str(url))

def decode_refer_url(encode):
    return base64.urlsafe_b64decode(str(encode))

def template(template_directory=None, template_filename=None, **kwargs):
    global_args = {
        'NAVBAR': NavbarClass(),
        'YEAR':   date.today().year,
        'URL':    flask.request.url,
        'REFER_SRC_STR':  flask.request.url.replace(flask.request.url_root, ''),
        '__wrapper__' : True,
    }
    global_args['REFER_SRC_ENCODED'] = encode_refer_url(global_args['REFER_SRC_STR'])
    if 'refer' in flask.request.args.keys():
        refer = flask.request.args['refer']
        print('[web] REFER: %r' % (refer, ))
        global_args['REFER_DST_ENCODED'] = refer
        global_args['REFER_DST_STR'] = decode_refer_url(refer)
    if template_directory is None:
        template_directory = ''
        #template_directory = abspath(join(dirname(__file__), 'templates'))
        #template_directory = join(dirname(dirname(__file__)))
    if template_filename is None:
        template_filename = 'index'
    template_ = join(template_directory, template_filename + '.html')
    # Update global args with the template's args
    _global_args = dict(global_args)
    _global_args.update(kwargs)
    print('[appfuncs] template()')
    from ibeis.control import controller_inject
    app = controller_inject.get_flask_app()
    # flask hates windows apparently
    template_ = template_.replace('\\', '/')
    print('[appfuncs.template] * app.template_folder = %r' % (app.template_folder,))
    print('[appfuncs.template] * template_directory = %r' % (template_directory,))
    print('[appfuncs.template] * template_filename = %r' % (template_filename,))
    print('[appfuncs.template] * template_ = %r' % (template_,))
    try:
        ret = flask.render_template(template_, **_global_args)
        #ret = flask.render_template(full_template_fpath, **_global_args)
    except jinja2.exceptions.TemplateNotFound as ex:
        print('Error template not found')
        full_template_fpath = join(app.template_folder, template_)
        print('[appfuncs.template] * full_template_fpath = %r' % (full_template_fpath,))
        ut.checkpath(full_template_fpath, verbose=True)
        ut.printex(ex, 'Template error in appfuncs', tb=True)
        raise
    except Exception as ex:
        ut.printex(ex, 'Error in appfuncs', tb=True)
        raise
    return ret

def send_csv_file(string, filename):
    response = flask.make_response(str(string))
    response.headers['Content-Description'] = 'File Transfer'
    response.headers['Cache-Control'] = 'no-cache'
    response.headers['Content-Type'] = 'text/csv'
    response.headers['Content-Disposition'] = 'attachment; filename=%s' % filename
    response.headers['Content-Length'] = len(string)
    return response

def get_turk_annot_args(is_reviewed_func):
    """
    Helper to return aids in an imageset or a group review
    """
    ibs = current_app.ibs
    def _ensureid(_id):
        return None if _id == 'None' or _id == '' else int(_id)

    imgsetid = request.args.get('imgsetid', '')
    src_ag = request.args.get('src_ag', '')
    dst_ag = request.args.get('dst_ag', '')

    imgsetid = _ensureid(imgsetid)
    src_ag = _ensureid(src_ag)
    dst_ag = _ensureid(dst_ag)

    group_review_flag = src_ag is not None and dst_ag is not None
    if not group_review_flag:
        gid_list = ibs.get_valid_gids(imgsetid=imgsetid)
        aid_list = ut.flatten(ibs.get_image_aids(gid_list))
        reviewed_list = is_reviewed_func(ibs, aid_list)
    else:
        src_gar_rowid_list = ibs.get_annotgroup_gar_rowids(src_ag)
        dst_gar_rowid_list = ibs.get_annotgroup_gar_rowids(dst_ag)
        src_aid_list = ibs.get_gar_aid(src_gar_rowid_list)
        dst_aid_list = ibs.get_gar_aid(dst_gar_rowid_list)
        aid_list = src_aid_list
        reviewed_list = [ src_aid in dst_aid_list for src_aid in src_aid_list ]

    try:
        progress = '%0.2f' % (100.0 * reviewed_list.count(True) / len(aid_list), )
    except ZeroDivisionError:
        progress = '0.00'
    aid = request.args.get('aid', '')
    if len(aid) > 0:
        aid = int(aid)
    else:
        aid_list_ = ut.filterfalse_items(aid_list, reviewed_list)
        if len(aid_list_) == 0:
            aid = None
        else:
            if group_review_flag:
                aid = aid_list_[0]
            else:
                aid = random.choice(aid_list_)

    previous = request.args.get('previous', None)

    print('aid = %r' % (aid,))
    #print(ut.dict_str(ibs.get_annot_info(aid)))
    print(ut.obj_str(ibs.get_annot_info(aid, default=True, nl=True)))
    return aid_list, reviewed_list, imgsetid, src_ag, dst_ag, progress, aid, previous

def movegroup_aid(ibs, aid, src_ag, dst_ag):
    gar_rowid_list = ibs.get_annot_gar_rowids(aid)
    annotgroup_rowid_list = ibs.get_gar_annotgroup_rowid(gar_rowid_list)
    src_index = annotgroup_rowid_list.index(src_ag)
    src_gar_rowid = gar_rowid_list[src_index]
    vals = (aid, src_ag, src_gar_rowid, dst_ag)
    print('Moving aid: %s from src_ag: %s (%s) to dst_ag: %s' % vals)
    # ibs.delete_gar([src_gar_rowid])
    ibs.add_gar([dst_ag], [aid])

def default_species(ibs):
    # hack function
    dbname = ibs.get_dbname()
    if dbname == 'CHTA_Master':
        default_species = 'cheetah'
    elif dbname == 'ELPH_Master':
        default_species = 'elephant_savanna'
    elif dbname == 'GIR_Master':
        default_species = 'giraffe_reticulated'
    elif dbname == 'GZ_Master':
        default_species = 'zebra_grevys'
    elif dbname == 'LION_Master':
        default_species = 'lion'
    elif dbname == 'PZ_Master':
        default_species = 'zebra_plains'
    elif dbname == 'WD_Master':
        default_species = 'wild_dog'
    elif dbname == 'NNP_MasterGIRM':
        default_species = 'giraffe_masai'
    elif 'NNP_' in dbname:
        default_species = 'zebra_plains'
    elif 'GZC' in dbname:
        default_species = 'zebra_plains'
    else:
        default_species = None
    print('[web] DEFAULT SPECIES: %r' % (default_species))
    return default_species

def imageset_image_processed(ibs, gid_list):
    images_reviewed = [ reviewed == 1 for reviewed in ibs.get_image_reviewed(gid_list) ]
    return images_reviewed

def imageset_annot_viewpoint_processed(ibs, aid_list):
    annots_reviewed = [ reviewed is not None for reviewed in ibs.get_annot_yaws(aid_list) ]
    return annots_reviewed

def imageset_annot_quality_processed(ibs, aid_list):
    annots_reviewed = [ reviewed is not None and reviewed is not -1 for reviewed in ibs.get_annot_qualities(aid_list) ]
    return annots_reviewed

def imageset_annot_additional_processed(ibs, aid_list, nid_list):
    sex_list = ibs.get_annot_sex(aid_list)
    age_list = ibs.get_annot_age_months_est(aid_list)
    annots_reviewed = [
        (nid < 0) or (nid > 0 and sex >= 0 and -1 not in list(age) and list(age).count(None) < 2)
        for nid, sex, age in zip(nid_list, sex_list, age_list)
    ]
    return annots_reviewed

def convert_old_viewpoint_to_yaw(view_angle):
    """ we initially had viewpoint coordinates inverted

    Example:
        >>> import math
        >>> TAU = 2 * math.pi
        >>> old_viewpoint_labels = [
        >>>     ('left'       ,   0, 0.000 * TAU,),
        >>>     ('frontleft'  ,  45, 0.125 * TAU,),
        >>>     ('front'      ,  90, 0.250 * TAU,),
        >>>     ('frontright' , 135, 0.375 * TAU,),
        >>>     ('right'      , 180, 0.500 * TAU,),
        >>>     ('backright'  , 225, 0.625 * TAU,),
        >>>     ('back'       , 270, 0.750 * TAU,),
        >>>     ('backleft'   , 315, 0.875 * TAU,),
        >>> ]
        >>> fmtstr = 'old %15r %.2f -> new %15r %.2f'
        >>> for lbl, angle, radians in old_viewpoint_labels:
        >>>     print(fmtstr % (lbl, angle, lbl, convert_old_viewpoint_to_yaw(angle)))
    """
    if view_angle is None:
        return None
    view_angle = ut.deg_to_rad(view_angle)
    yaw = (-view_angle + (TAU / 2)) % TAU
    return yaw

def convert_yaw_to_old_viewpoint(yaw):
    """ we initially had viewpoint coordinates inverted

    Example:
        >>> import math
        >>> TAU = 2 * math.pi
        >>> old_viewpoint_labels = [
        >>>     ('left'       ,   0, 0.000 * TAU,),
        >>>     ('frontleft'  ,  45, 0.125 * TAU,),
        >>>     ('front'      ,  90, 0.250 * TAU,),
        >>>     ('frontright' , 135, 0.375 * TAU,),
        >>>     ('right'      , 180, 0.500 * TAU,),
        >>>     ('backright'  , 225, 0.625 * TAU,),
        >>>     ('back'       , 270, 0.750 * TAU,),
        >>>     ('backleft'   , 315, 0.875 * TAU,),
        >>> ]
        >>> fmtstr = 'original_angle %15r %.2f -> yaw %15r %.2f -> reconstructed_angle %15r %.2f'
        >>> for lbl, angle, radians in old_viewpoint_labels:
        >>>     yaw = convert_old_viewpoint_to_yaw(angle)
        >>>     reconstructed_angle = convert_yaw_to_old_viewpoint(yaw)
        >>>     print(fmtstr % (lbl, angle, lbl, yaw, lbl, reconstructed_angle))
    """
    if yaw is None:
        return None
    view_angle = ((TAU / 2) - yaw) % TAU
    view_angle = ut.rad_to_deg(view_angle)
    return view_angle
<commit_msg>Processing for NMEA files<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
import cStringIO as StringIO
import flask
import random
from ibeis import ibsfuncs
from ibeis.constants import TAU
from flask import request, current_app
from os.path import join, dirname, abspath  # NOQA
from datetime import datetime
from datetime import date
import base64
import jinja2
import utool as ut
import pynmea2
import simplejson as json

DEFAULT_WEB_API_PORT = ut.get_argval('--port', type_=int, default=5000)
TARGET_WIDTH = 1200.0
PAGE_SIZE = 500
VALID_TURK_MODES = [
    ('turk_viewpoint', 'Viewpoint'),
    ('turk_quality', 'Quality'),
]

class NavbarClass(object):
    def __init__(nav):
        nav.item_list = [
            ('root', 'Home'),
            ('view', 'View'),
            ('turk', 'Turk'),
            ('api_root',  'API'),
            ('group_review',  'Group Review'),
        ]

    def __iter__(nav):
        _link = flask.request.path.strip('/').split('/')
        for link, nice in nav.item_list:
            yield link == _link[0], link, nice

def resize_via_web_parameters(image):
    w_pix = request.args.get('resize_pix_w',      request.form.get('resize_pix_w',      None ))
    h_pix = request.args.get('resize_pix_h',      request.form.get('resize_pix_h',      None ))
    w_per = request.args.get('resize_per_w',      request.form.get('resize_per_w',      None ))
    h_per = request.args.get('resize_per_h',      request.form.get('resize_per_h',      None ))
    _pix  = request.args.get('resize_prefer_pix', request.form.get('resize_prefer_pix', False))
    _per  = request.args.get('resize_prefer_per', request.form.get('resize_prefer_per', False))
    args = (w_pix, h_pix, w_per, h_per, _pix, _per, )
    print('CHECKING RESIZING WITH %r pix, %r pix, %r %%, %r %% [%r, %r]' % args)
    # Check for nothing
    if not (w_pix or h_pix or w_per or h_per):
        return image
    # Check for both pixels and images
    if (w_pix or h_pix) and (w_per or h_per):
        if _pix:
            w_per, h_per = None
        elif _per:
            w_pix, h_pix = None
        else:
            raise ValueError('Cannot resize using pixels and percentages, pick one')
    # Resize using percentages, transform to pixels
    if w_per:
        w_pix = float(w_per) * image.shape[1]
    if h_per:
        h_pix = float(h_per) * image.shape[0]
    # Perform resize
    return ibsfuncs._resize(image, t_width=w_pix, t_height=h_pix)

def embed_image_html(imgBGR, target_width=TARGET_WIDTH):
    """ Creates an image embedded in HTML base64 format. """
    import cv2
    from PIL import Image
    imgBGR = ibsfuncs._resize(imgBGR, target_width)
    imgRGB = cv2.cvtColor(imgBGR, cv2.COLOR_BGR2RGB)
    pil_img = Image.fromarray(imgRGB)
    string_buf = StringIO.StringIO()
    pil_img.save(string_buf, format='jpeg')
    data = string_buf.getvalue().encode('base64').replace('\n', '')
    return 'data:image/jpeg;base64,' + data

def check_valid_function_name(string):
    return all([ char.isalpha() or char == '_' or char.isalnum() for char in string])

def encode_refer_url(url):
    return base64.urlsafe_b64encode(str(url))

def decode_refer_url(encode):
    return base64.urlsafe_b64decode(str(encode))

def template(template_directory=None, template_filename=None, **kwargs):
    global_args = {
        'NAVBAR': NavbarClass(),
        'YEAR':   date.today().year,
        'URL':    flask.request.url,
        'REFER_SRC_STR':  flask.request.url.replace(flask.request.url_root, ''),
        '__wrapper__' : True,
    }
    global_args['REFER_SRC_ENCODED'] = encode_refer_url(global_args['REFER_SRC_STR'])
    if 'refer' in flask.request.args.keys():
        refer = flask.request.args['refer']
        print('[web] REFER: %r' % (refer, ))
        global_args['REFER_DST_ENCODED'] = refer
        global_args['REFER_DST_STR'] = decode_refer_url(refer)
    if template_directory is None:
        template_directory = ''
        #template_directory = abspath(join(dirname(__file__), 'templates'))
        #template_directory = join(dirname(dirname(__file__)))
    if template_filename is None:
        template_filename = 'index'
    template_ = join(template_directory, template_filename + '.html')
    # Update global args with the template's args
    _global_args = dict(global_args)
    _global_args.update(kwargs)
    print('[appfuncs] template()')
    from ibeis.control import controller_inject
    app = controller_inject.get_flask_app()
    # flask hates windows apparently
    template_ = template_.replace('\\', '/')
    print('[appfuncs.template] * app.template_folder = %r' % (app.template_folder,))
    print('[appfuncs.template] * template_directory = %r' % (template_directory,))
    print('[appfuncs.template] * template_filename = %r' % (template_filename,))
    print('[appfuncs.template] * template_ = %r' % (template_,))
    try:
        ret = flask.render_template(template_, **_global_args)
        #ret = flask.render_template(full_template_fpath, **_global_args)
    except jinja2.exceptions.TemplateNotFound as ex:
        print('Error template not found')
        full_template_fpath = join(app.template_folder, template_)
        print('[appfuncs.template] * full_template_fpath = %r' % (full_template_fpath,))
        ut.checkpath(full_template_fpath, verbose=True)
        ut.printex(ex, 'Template error in appfuncs', tb=True)
        raise
    except Exception as ex:
        ut.printex(ex, 'Error in appfuncs', tb=True)
        raise
    return ret

def send_csv_file(string, filename):
    response = flask.make_response(str(string))
    response.headers['Content-Description'] = 'File Transfer'
    response.headers['Cache-Control'] = 'no-cache'
    response.headers['Content-Type'] = 'text/csv'
    response.headers['Content-Disposition'] = 'attachment; filename=%s' % filename
    response.headers['Content-Length'] = len(string)
    return response

def get_turk_annot_args(is_reviewed_func):
    """
    Helper to return aids in an imageset or a group review
    """
    ibs = current_app.ibs
    def _ensureid(_id):
        return None if _id == 'None' or _id == '' else int(_id)

    imgsetid = request.args.get('imgsetid', '')
    src_ag = request.args.get('src_ag', '')
    dst_ag = request.args.get('dst_ag', '')

    imgsetid = _ensureid(imgsetid)
    src_ag = _ensureid(src_ag)
    dst_ag = _ensureid(dst_ag)

    group_review_flag = src_ag is not None and dst_ag is not None
    if not group_review_flag:
        gid_list = ibs.get_valid_gids(imgsetid=imgsetid)
        aid_list = ut.flatten(ibs.get_image_aids(gid_list))
        reviewed_list = is_reviewed_func(ibs, aid_list)
    else:
        src_gar_rowid_list = ibs.get_annotgroup_gar_rowids(src_ag)
        dst_gar_rowid_list = ibs.get_annotgroup_gar_rowids(dst_ag)
        src_aid_list = ibs.get_gar_aid(src_gar_rowid_list)
        dst_aid_list = ibs.get_gar_aid(dst_gar_rowid_list)
        aid_list = src_aid_list
        reviewed_list = [ src_aid in dst_aid_list for src_aid in src_aid_list ]

    try:
        progress = '%0.2f' % (100.0 * reviewed_list.count(True) / len(aid_list), )
    except ZeroDivisionError:
        progress = '0.00'
    aid = request.args.get('aid', '')
    if len(aid) > 0:
        aid = int(aid)
    else:
        aid_list_ = ut.filterfalse_items(aid_list, reviewed_list)
        if len(aid_list_) == 0:
            aid = None
        else:
            if group_review_flag:
                aid = aid_list_[0]
            else:
                aid = random.choice(aid_list_)

    previous = request.args.get('previous', None)

    print('aid = %r' % (aid,))
    #print(ut.dict_str(ibs.get_annot_info(aid)))
    print(ut.obj_str(ibs.get_annot_info(aid, default=True, nl=True)))
    return aid_list, reviewed_list, imgsetid, src_ag, dst_ag, progress, aid, previous

def movegroup_aid(ibs, aid, src_ag, dst_ag):
    gar_rowid_list = ibs.get_annot_gar_rowids(aid)
    annotgroup_rowid_list = ibs.get_gar_annotgroup_rowid(gar_rowid_list)
    src_index = annotgroup_rowid_list.index(src_ag)
    src_gar_rowid = gar_rowid_list[src_index]
    vals = (aid, src_ag, src_gar_rowid, dst_ag)
    print('Moving aid: %s from src_ag: %s (%s) to dst_ag: %s' % vals)
    # ibs.delete_gar([src_gar_rowid])
    ibs.add_gar([dst_ag], [aid])

def default_species(ibs):
    # hack function
    dbname = ibs.get_dbname()
    if dbname == 'CHTA_Master':
        default_species = 'cheetah'
    elif dbname == 'ELPH_Master':
        default_species = 'elephant_savanna'
    elif dbname == 'GIR_Master':
        default_species = 'giraffe_reticulated'
    elif dbname == 'GZ_Master':
        default_species = 'zebra_grevys'
    elif dbname == 'LION_Master':
        default_species = 'lion'
    elif dbname == 'PZ_Master':
        default_species = 'zebra_plains'
    elif dbname == 'WD_Master':
        default_species = 'wild_dog'
    elif dbname == 'NNP_MasterGIRM':
        default_species = 'giraffe_masai'
    elif 'NNP_' in dbname:
        default_species = 'zebra_plains'
    elif 'GZC' in dbname:
        default_species = 'zebra_plains'
    else:
        default_species = None
    print('[web] DEFAULT SPECIES: %r' % (default_species))
    return default_species

def imageset_image_processed(ibs, gid_list):
    images_reviewed = [ reviewed == 1 for reviewed in ibs.get_image_reviewed(gid_list) ]
    return images_reviewed

def imageset_annot_viewpoint_processed(ibs, aid_list):
    annots_reviewed = [ reviewed is not None for reviewed in ibs.get_annot_yaws(aid_list) ]
    return annots_reviewed

def imageset_annot_quality_processed(ibs, aid_list):
    annots_reviewed = [ reviewed is not None and reviewed is not -1 for reviewed in ibs.get_annot_qualities(aid_list) ]
    return annots_reviewed

def imageset_annot_additional_processed(ibs, aid_list, nid_list):
    sex_list = ibs.get_annot_sex(aid_list)
    age_list = ibs.get_annot_age_months_est(aid_list)
    annots_reviewed = [
        (nid < 0) or (nid > 0 and sex >= 0 and -1 not in list(age) and list(age).count(None) < 2)
        for nid, sex, age in zip(nid_list, sex_list, age_list)
    ]
    return annots_reviewed

def convert_old_viewpoint_to_yaw(view_angle):
    """ we initially had viewpoint coordinates inverted

    Example:
        >>> import math
        >>> TAU = 2 * math.pi
        >>> old_viewpoint_labels = [
        >>>     ('left'       ,   0, 0.000 * TAU,),
        >>>     ('frontleft'  ,  45, 0.125 * TAU,),
        >>>     ('front'      ,  90, 0.250 * TAU,),
        >>>     ('frontright' , 135, 0.375 * TAU,),
        >>>     ('right'      , 180, 0.500 * TAU,),
        >>>     ('backright'  , 225, 0.625 * TAU,),
        >>>     ('back'       , 270, 0.750 * TAU,),
        >>>     ('backleft'   , 315, 0.875 * TAU,),
        >>> ]
        >>> fmtstr = 'old %15r %.2f -> new %15r %.2f'
        >>> for lbl, angle, radians in old_viewpoint_labels:
        >>>     print(fmtstr % (lbl, angle, lbl, convert_old_viewpoint_to_yaw(angle)))
    """
    if view_angle is None:
        return None
    view_angle = ut.deg_to_rad(view_angle)
    yaw = (-view_angle + (TAU / 2)) % TAU
    return yaw

def convert_yaw_to_old_viewpoint(yaw):
    """ we initially had viewpoint coordinates inverted

    Example:
        >>> import math
        >>> TAU = 2 * math.pi
        >>> old_viewpoint_labels = [
        >>>     ('left'       ,   0, 0.000 * TAU,),
        >>>     ('frontleft'  ,  45, 0.125 * TAU,),
        >>>     ('front'      ,  90, 0.250 * TAU,),
        >>>     ('frontright' , 135, 0.375 * TAU,),
        >>>     ('right'      , 180, 0.500 * TAU,),
        >>>     ('backright'  , 225, 0.625 * TAU,),
        >>>     ('back'       , 270, 0.750 * TAU,),
        >>>     ('backleft'   , 315, 0.875 * TAU,),
        >>> ]
        >>> fmtstr = 'original_angle %15r %.2f -> yaw %15r %.2f -> reconstructed_angle %15r %.2f'
        >>> for lbl, angle, radians in old_viewpoint_labels:
        >>>     yaw = convert_old_viewpoint_to_yaw(angle)
        >>>     reconstructed_angle = convert_yaw_to_old_viewpoint(yaw)
        >>>     print(fmtstr % (lbl, angle, lbl, yaw, lbl, reconstructed_angle))
    """
    if yaw is None:
        return None
    view_angle = ((TAU / 2) - yaw) % TAU
    view_angle = ut.rad_to_deg(view_angle)
    return view_angle

def convert_nmea_to_json(nmea_str, filename, GMT_OFFSET=0):
    json_list = []
    filename = filename.strip('.LOG').strip('N')
    year = 2000 + int(filename[0:2])
    month = int(filename[2:4])
    day = int(filename[4:6])
    print(year, month, day)
    for line in nmea_str.split('\n'):
        line = line.strip()
        if '@' in line or 'GPRMC' in line or len(line) == 0:
            continue
        record = pynmea2.parse(line)
        dt = record.timestamp
        dt = datetime(year, month, day, dt.hour, dt.minute, dt.second)
        # Gather values
        posix = int(dt.strftime("%s"))
        posix += (60 * 60 * GMT_OFFSET)
        lat   = float(record.latitude)
        lon   = float(record.longitude)
        json_list.append({
            'time': posix,
            'lat':  lat,
            'lon':  lon,
        })
    return json.dumps({ "track": json_list })
<|endoftext|>


<commit_before><commit_msg>made executable<commit_after><|endoftext|>


<commit_before><commit_msg>5927: NPE while adding messages.<commit_after><|endoftext|>


<commit_before># import json
# import os
import pytest
import requests

from dcicutils.env_utils import get_bucket_env, is_stg_or_prd_env, get_standard_mirror_env
from dcicutils.misc_utils import find_association
from dcicutils.s3_utils import s3Utils

pytestmark = [pytest.mark.working, pytest.mark.integrated]

_S3_UTILS_BUCKET_VAR_DATA = [
    {
        'attribute': 'sys_bucket',
        'health_key': 'system_bucket',
        'description': "The 'xxx-system' bucket",
        'template': 'SYS_BUCKET_TEMPLATE',
        'recent': False,
    },
    {
        'attribute': 'outfile_bucket',
        'health_key': 'processed_file_bucket',
        'description': "The 'xxx-wfoutput' bucket",
        'template': 'OUTFILE_BUCKET_TEMPLATE',
        'recent': False,
    },
    {
        'attribute': 'raw_file_bucket',
        'health_key': 'file_upload_bucket',
        'description': "The 'xxx-files' bucket",
        'template': 'RAW_BUCKET_TEMPLATE',
        'recent': False,
    },
    {
        'attribute': 'blob_bucket',
        'health_key': 'blob_bucket',
        'description': "The 'xxx-blobs' bucket",
        'template': 'BLOB_BUCKET_TEMPLATE',
        'recent': False,
    },
    {
        'attribute': 'metadata_bucket',
        'health_key': 'metadata_bucket',
        'description': "The 'xxx-metadata-bundles' bucket",
        'template': 'METADATA_BUCKET_TEMPLATE',
        'recent': True,
    },
    {
        'attribute': 'tibanna_output_bucket',
        'health_key': 'tibanna_output_bucket',
        'description': "The 'tibanna-output' bucket",
        'template': 'TIBANNA_OUTPUT_BUCKET_TEMPLATE',
        'recent': True,
    },
    {
        'attribute': 'tibanna_cwls_bucket',
        'health_key': 'tibanna_cwls_bucket',
        'description': "The 'tibanna-cwls' bucket",
        'template': 'TIBANNA_CWLS_BUCKET_TEMPLATE',
        'recent': True,
    },
]

def _health_page(*, url):
    return requests.get(url+"/health?format=json").json()

def _apply_s3_bucket_name_template(template, arg):
    if '%' in template:
        return template % arg
    else:
        return template

_PRD_URL = "https://data.4dnucleome.org"
_STG_URL = "http://staging.4dnucleome.org"

def _test_stg_or_prd(*, me, my_twin, env_from_beanstalk, s3utils):
    assert s3utils.url == me
    mirror_env = get_standard_mirror_env(env_from_beanstalk)
    mirror_s = s3Utils(env=mirror_env)
    assert mirror_s.url == my_twin
    mirror_health = _health_page(url=mirror_s.url)
    assert mirror_health['beanstalk_env'] == mirror_env
    assert get_standard_mirror_env(mirror_env) == env_from_beanstalk

def _test_data(*, env_from_beanstalk, s3utils):
    _test_stg_or_prd(me=_PRD_URL, my_twin=_STG_URL, env_from_beanstalk=env_from_beanstalk, s3utils=s3utils)

def _test_staging(*, env_from_beanstalk, s3utils):
    _test_stg_or_prd(me=_STG_URL, my_twin=_PRD_URL, env_from_beanstalk=env_from_beanstalk, s3utils=s3utils)

@pytest.mark.parametrize('env', [None, 'fourfront-mastertest', 'fourfront-green', 'fourfront-blue', 'data', 'staging'])
def test_s3_utils_bare(env):

    # Calling without an env argument or explicit bucket names is only expected to work
    # in orchestrated environments with a single environment.

    s = s3Utils(env=env)

    # No matter where invoked, we should at least get an AWS s3 object
    assert s.s3

    # This is probably the same everywhere. It doesn't need to vary.
    assert s.ACCESS_KEYS_S3_KEY == 'access_key_admin'

    for datum in _S3_UTILS_BUCKET_VAR_DATA:

        attr_name = datum['attribute']
        template_name = datum['template']

        # This is behavior we don't want, but it's the normal behavior, so test stability.
        # e.g., for env=None, assert s.sys_bucket == 'elasticbeanstalk-None-system'
        #   but for env='fourfront-mastertest', assert s.sys_bucket == 'elasticbeanstalk-fourfront-mastertest-system'
        if hasattr(s, attr_name) and hasattr(s, template_name):
            assert getattr(s, attr_name) == _apply_s3_bucket_name_template(getattr(s, template_name),
                                                                           get_bucket_env(env))
        else:
            assert datum['recent'], f"Problem with: {datum}"

    # As of dcicutils 2.3.0 or 2.3.1 (there was a bug fix in the initial patch),
    # the .url is expected to always be set for beanstalk environments, even ones
    # that are not stg/prd. But it's still the case that you can call s3Utils(env=None)
    # and get back an object that has some things filled even though the bucket names
    # are nonsensical and the env is None.

    if env:

        assert s.url is not ''

        health = _health_page(url=s.url)

        for k, v in health.items():
            if k.endswith("bucket"):
                print(f"Considering health page key {k}...")
                entry = find_association(_S3_UTILS_BUCKET_VAR_DATA, health_key=k)
                assert entry, f"No entry for health key {k}."
                if v:
                    assert getattr(s, entry['attribute']) == v
                    print("Attribute matches.")
                else:
                    print("No health page value.")

        env_from_beanstalk = health['beanstalk_env']

        if is_stg_or_prd_env(env):

            if env == 'data':
                _test_data(env_from_beanstalk=env_from_beanstalk, s3utils=s)
            elif env == 'staging':
                _test_staging(env_from_beanstalk=env_from_beanstalk, s3utils=s)
            elif is_stg_or_prd_env(env):
                assert env_from_beanstalk == env
                if s.url == _PRD_URL:
                    _test_data(env_from_beanstalk=env_from_beanstalk, s3utils=s)
                else:
                    _test_staging(env_from_beanstalk=env_from_beanstalk, s3utils=s)
<commit_msg>disable acceptence tests<commit_after># import json
# import os
import pytest
import requests

from dcicutils.env_utils import get_bucket_env, is_stg_or_prd_env, get_standard_mirror_env
from dcicutils.misc_utils import find_association
from dcicutils.s3_utils import s3Utils

pytestmark = [pytest.mark.working, pytest.mark.integrated]

_S3_UTILS_BUCKET_VAR_DATA = [
    {
        'attribute': 'sys_bucket',
        'health_key': 'system_bucket',
        'description': "The 'xxx-system' bucket",
        'template': 'SYS_BUCKET_TEMPLATE',
        'recent': False,
    },
    {
        'attribute': 'outfile_bucket',
        'health_key': 'processed_file_bucket',
        'description': "The 'xxx-wfoutput' bucket",
        'template': 'OUTFILE_BUCKET_TEMPLATE',
        'recent': False,
    },
    {
        'attribute': 'raw_file_bucket',
        'health_key': 'file_upload_bucket',
        'description': "The 'xxx-files' bucket",
        'template': 'RAW_BUCKET_TEMPLATE',
        'recent': False,
    },
    {
        'attribute': 'blob_bucket',
        'health_key': 'blob_bucket',
        'description': "The 'xxx-blobs' bucket",
        'template': 'BLOB_BUCKET_TEMPLATE',
        'recent': False,
    },
    {
        'attribute': 'metadata_bucket',
        'health_key': 'metadata_bucket',
        'description': "The 'xxx-metadata-bundles' bucket",
        'template': 'METADATA_BUCKET_TEMPLATE',
        'recent': True,
    },
    {
        'attribute': 'tibanna_output_bucket',
        'health_key': 'tibanna_output_bucket',
        'description': "The 'tibanna-output' bucket",
        'template': 'TIBANNA_OUTPUT_BUCKET_TEMPLATE',
        'recent': True,
    },
    {
        'attribute': 'tibanna_cwls_bucket',
        'health_key': 'tibanna_cwls_bucket',
        'description': "The 'tibanna-cwls' bucket",
        'template': 'TIBANNA_CWLS_BUCKET_TEMPLATE',
        'recent': True,
    },
]

def _health_page(*, url):
    return requests.get(url+"/health?format=json").json()

def _apply_s3_bucket_name_template(template, arg):
    if '%' in template:
        return template % arg
    else:
        return template

_PRD_URL = "https://data.4dnucleome.org"
_STG_URL = "http://staging.4dnucleome.org"

def _test_stg_or_prd(*, me, my_twin, env_from_beanstalk, s3utils):
    assert s3utils.url == me
    mirror_env = get_standard_mirror_env(env_from_beanstalk)
    mirror_s = s3Utils(env=mirror_env)
    assert mirror_s.url == my_twin
    mirror_health = _health_page(url=mirror_s.url)
    assert mirror_health['beanstalk_env'] == mirror_env
    assert get_standard_mirror_env(mirror_env) == env_from_beanstalk

def _test_data(*, env_from_beanstalk, s3utils):
    _test_stg_or_prd(me=_PRD_URL, my_twin=_STG_URL, env_from_beanstalk=env_from_beanstalk, s3utils=s3utils)

def _test_staging(*, env_from_beanstalk, s3utils):
    _test_stg_or_prd(me=_STG_URL, my_twin=_PRD_URL, env_from_beanstalk=env_from_beanstalk, s3utils=s3utils)

@pytest.mark.skip  # this test relies on string building the buckets and must be updated
@pytest.mark.parametrize('env', [None, 'fourfront-mastertest', 'fourfront-webdev', 'fourfront-hotseat', 'data', 'staging'])
def test_s3_utils_bare(env):

    # Calling without an env argument or explicit bucket names is only expected to work
    # in orchestrated environments with a single environment.

    s = s3Utils(env=env)

    # No matter where invoked, we should at least get an AWS s3 object
    assert s.s3

    # This is probably the same everywhere. It doesn't need to vary.
    assert s.ACCESS_KEYS_S3_KEY == 'access_key_admin'

    for datum in _S3_UTILS_BUCKET_VAR_DATA:

        attr_name = datum['attribute']
        template_name = datum['template']

        # This is behavior we don't want, but it's the normal behavior, so test stability.
        # e.g., for env=None, assert s.sys_bucket == 'elasticbeanstalk-None-system'
        #   but for env='fourfront-mastertest', assert s.sys_bucket == 'elasticbeanstalk-fourfront-mastertest-system'
        if hasattr(s, attr_name) and hasattr(s, template_name):
            assert getattr(s, attr_name) == _apply_s3_bucket_name_template(getattr(s, template_name),
                                                                           get_bucket_env(env))
        else:
            assert datum['recent'], f"Problem with: {datum}"

    # As of dcicutils 2.3.0 or 2.3.1 (there was a bug fix in the initial patch),
    # the .url is expected to always be set for beanstalk environments, even ones
    # that are not stg/prd. But it's still the case that you can call s3Utils(env=None)
    # and get back an object that has some things filled even though the bucket names
    # are nonsensical and the env is None.

    if env:

        assert s.url is not ''

        health = _health_page(url=s.url)

        for k, v in health.items():
            if k.endswith("bucket"):
                print(f"Considering health page key {k}...")
                entry = find_association(_S3_UTILS_BUCKET_VAR_DATA, health_key=k)
                assert entry, f"No entry for health key {k}."
                if v:
                    assert getattr(s, entry['attribute']) == v
                    print("Attribute matches.")
                else:
                    print("No health page value.")

        env_from_beanstalk = health['beanstalk_env']

        if is_stg_or_prd_env(env):

            if env == 'data':
                _test_data(env_from_beanstalk=env_from_beanstalk, s3utils=s)
            elif env == 'staging':
                _test_staging(env_from_beanstalk=env_from_beanstalk, s3utils=s)
            elif is_stg_or_prd_env(env):
                assert env_from_beanstalk == env
                if s.url == _PRD_URL:
                    _test_data(env_from_beanstalk=env_from_beanstalk, s3utils=s)
                else:
                    _test_staging(env_from_beanstalk=env_from_beanstalk, s3utils=s)
<|endoftext|>


<commit_before>import os
import subprocess
import shutil
import json
import uuid
import stat

from whack import downloads
from whack.hashes import Hasher
from whack.tempdir import create_temporary_dir

__all__ = ["PackageInstaller"]

_WHACK_ROOT = "/usr/local/whack"

class RelocatableTemplate(object):
    def install_to_cache(self, run, build_dir, working_dir):
        build_script = os.path.join(working_dir, "whack/build")
        run([build_script])

    def install_from_cache(self, build_dir, working_dir, install_dir):
        subprocess.check_call(
            [os.path.join(working_dir, "whack/install"), install_dir],
            cwd=working_dir
        )

class FixedRootTemplate(object):
    def install_to_cache(self, run, build_dir, working_dir):
        install_script = os.path.join(working_dir, "whack/install")
        install_dir = self._cached_install_dir(build_dir)
        os.mkdir(install_dir)
        run([
            "whack-run-with-whack-root",
            install_dir,
            install_script,
            _WHACK_ROOT
        ])

        with open(os.path.join(install_dir, "run"), "w") as run_file:
            run_file.write('#!/usr/bin/env sh\nexec whack-run-with-whack-root "$(dirname $0)" "$@"')
        subprocess.check_call(["chmod", "+x", os.path.join(install_dir, "run")])

        self._create_bin_dir(install_dir, "bin")
        self._create_bin_dir(install_dir, "sbin")

    def _create_bin_dir(self, install_dir, bin_dir_name):
        def install_path(path):
            return os.path.join(install_dir, path)

        dot_bin_dir = install_path(".{0}".format(bin_dir_name))
        bin_dir = install_path(bin_dir_name)
        if os.path.exists(dot_bin_dir) and not os.path.exists(bin_dir):
            os.mkdir(bin_dir)
            for bin_filename in self._list_executable_files(dot_bin_dir):
                bin_file_path = os.path.join(bin_dir, bin_filename)
                with open(bin_file_path, "w") as bin_file:
                    bin_file.write(
                        '#!/usr/bin/env sh\n\n' +
                        'TARGET="$(dirname $0)/../.{0}/{1}"\n'.format(bin_dir_name, bin_filename) +
                        'ACTIVE_ROOT_ID_FILE=\'{0}\'\n'.format(_WHACK_ROOT) +
                        'MY_ROOT_ID=`cat $(dirname $0)/../.whack-root-id`\n' +
                        'ACTIVE_ROOT_ID=`cat "$ACTIVE_ROOT_ID_FILE" 2>/dev/null`\n' +
                        'if [ -f "$ACTIVE_ROOT_ID_FILE" ] && [ "$MY_ROOT_ID" = "$ACTIVE_ROOT_ID" ]; then\n' +
                        'exec "$TARGET" "$@"\n' +
                        'else\n' +
                        'exec "$(dirname $0)/../run" "$TARGET" "$@"\n' +
                        'fi\n'
                    )
                os.chmod(bin_file_path, 0755)

    def _list_executable_files(self, dir_path):
        def is_executable_file(filename):
            path = os.path.join(dir_path, filename)
            is_executable = stat.S_IXUSR & os.stat(path)[stat.ST_MODE]
            return not os.path.isdir(path) and is_executable
        return filter(is_executable_file, os.listdir(dir_path))

    def install_from_cache(self, build_dir, working_dir, install_dir):
        cached_install_dir = self._cached_install_dir(build_dir)
        # TODO: should be pure Python, but there isn't a stdlib function
        # that allows the destination to already exist
        subprocess.check_call(["cp", "-rT", cached_install_dir, install_dir])
        with open(os.path.join(install_dir, ".whack-root-id"), "w") as root_id_file:
            root_id_file.write(str(uuid.uuid4()))

    def _cached_install_dir(self, build_dir):
        return os.path.join(build_dir, "install")

_templates = {
    "fixed-root": FixedRootTemplate(),
    "relocatable": RelocatableTemplate()
}

_default_template_name = "relocatable"

class PackageInstaller(object):
    def __init__(self, package_dir, cacher):
        self._package_dir = package_dir
        self._cacher = cacher

    def install(self, install_dir, params={}):
        install_id = _generate_install_id(self._package_dir, params)

        with create_temporary_dir() as temp_dir:
            build_dir = os.path.join(temp_dir, "build")
            working_dir = os.path.join(build_dir, "workspace")

            result = self._cacher.fetch(install_id, working_dir)
            if not result.cache_hit:
                self._build(build_dir, working_dir, params)
                self._cacher.put(install_id, working_dir)

            self._template().install_from_cache(build_dir, working_dir, install_dir)

    def _build(self, build_dir, working_dir, params):
        ignore = shutil.ignore_patterns(".svn", ".hg", ".hgignore", ".git", ".gitignore")
        shutil.copytree(self._package_dir, working_dir, ignore=ignore)
        build_env = params_to_build_env(params)
        self._fetch_downloads(working_dir, build_env)

        def run(command):
            subprocess.check_call(
                command,
                cwd=working_dir,
                env=build_env
            )

        self._template().install_to_cache(run, build_dir, working_dir)

    def _template(self):
        return _templates[self._template_name()]

    def _fetch_downloads(self, build_dir, build_env):
        downloads_file_path = os.path.join(build_dir, "whack/downloads")
        downloads_file = self._read_downloads_file(downloads_file_path, build_env)
        for download_line in downloads_file:
            downloads.download(download_line.url, os.path.join(build_dir, download_line.filename))

    def _read_downloads_file(self, path, build_env):
        if os.path.exists(path):
            first_line = open(path).readline()
            if first_line.startswith("#!"):
                downloads_string = subprocess.check_output(
                    [path],
                    env=build_env
                )
            else:
                downloads_string = open(path).read()

            return downloads.read_downloads_string(downloads_string)
        else:
            return []

    def _template_name(self):
        return _read_package_description(self._package_dir).template_name

def params_to_build_env(params):
    build_env = os.environ.copy()
    for name, value in (params or {}).iteritems():
        build_env[name.upper()] = str(value)
    return build_env

def _generate_install_id(package_dir, params):
    hasher = Hasher()
    hasher.update(_uname("--kernel-name"))
    hasher.update(_uname("--machine"))
    hasher.update_with_dir(package_dir)
    hasher.update(json.dumps(params, sort_keys=True))
    return hasher.hexdigest()

def _uname(arg):
    return subprocess.check_output(["uname", arg])

def _read_package_description(package_dir):
    whack_json_path = os.path.join(package_dir, "whack/whack.json")
    if os.path.exists(whack_json_path):
        with open(whack_json_path, "r") as whack_json_file:
            whack_json = json.load(whack_json_file)
        return DictBackedPackageDescription(whack_json)
    else:
        return DefaultPackageDescription()

class DefaultPackageDescription(object):
    template_name = _default_template_name

class DictBackedPackageDescription(object):
    def __init__(self, values):
        self._values = values

    @property
    def template_name(self):
        return self._values.get("template", _default_template_name)
<commit_msg>Fix caching<commit_after>import os
import subprocess
import shutil
import json
import uuid
import stat

from whack import downloads
from whack.hashes import Hasher
from whack.tempdir import create_temporary_dir

__all__ = ["PackageInstaller"]

_WHACK_ROOT = "/usr/local/whack"

class RelocatableTemplate(object):
    def install_to_cache(self, run, build_dir, working_dir):
        build_script = os.path.join(working_dir, "whack/build")
        run([build_script])

    def install_from_cache(self, build_dir, working_dir, install_dir):
        subprocess.check_call(
            [os.path.join(working_dir, "whack/install"), install_dir],
            cwd=working_dir
        )

class FixedRootTemplate(object):
    def install_to_cache(self, run, build_dir, working_dir):
        install_script = os.path.join(working_dir, "whack/install")
        install_dir = self._cached_install_dir(build_dir)
        os.mkdir(install_dir)
        run([
            "whack-run-with-whack-root",
            install_dir,
            install_script,
            _WHACK_ROOT
        ])

        with open(os.path.join(install_dir, "run"), "w") as run_file:
            run_file.write('#!/usr/bin/env sh\nexec whack-run-with-whack-root "$(dirname $0)" "$@"')
        subprocess.check_call(["chmod", "+x", os.path.join(install_dir, "run")])

        self._create_bin_dir(install_dir, "bin")
        self._create_bin_dir(install_dir, "sbin")

    def _create_bin_dir(self, install_dir, bin_dir_name):
        def install_path(path):
            return os.path.join(install_dir, path)

        dot_bin_dir = install_path(".{0}".format(bin_dir_name))
        bin_dir = install_path(bin_dir_name)
        if os.path.exists(dot_bin_dir) and not os.path.exists(bin_dir):
            os.mkdir(bin_dir)
            for bin_filename in self._list_executable_files(dot_bin_dir):
                bin_file_path = os.path.join(bin_dir, bin_filename)
                with open(bin_file_path, "w") as bin_file:
                    bin_file.write(
                        '#!/usr/bin/env sh\n\n' +
                        'TARGET="$(dirname $0)/../.{0}/{1}"\n'.format(bin_dir_name, bin_filename) +
                        'ACTIVE_ROOT_ID_FILE=\'{0}\'\n'.format(_WHACK_ROOT) +
                        'MY_ROOT_ID=`cat $(dirname $0)/../.whack-root-id`\n' +
                        'ACTIVE_ROOT_ID=`cat "$ACTIVE_ROOT_ID_FILE" 2>/dev/null`\n' +
                        'if [ -f "$ACTIVE_ROOT_ID_FILE" ] && [ "$MY_ROOT_ID" = "$ACTIVE_ROOT_ID" ]; then\n' +
                        'exec "$TARGET" "$@"\n' +
                        'else\n' +
                        'exec "$(dirname $0)/../run" "$TARGET" "$@"\n' +
                        'fi\n'
                    )
                os.chmod(bin_file_path, 0755)

    def _list_executable_files(self, dir_path):
        def is_executable_file(filename):
            path = os.path.join(dir_path, filename)
            is_executable = stat.S_IXUSR & os.stat(path)[stat.ST_MODE]
            return not os.path.isdir(path) and is_executable
        return filter(is_executable_file, os.listdir(dir_path))

    def install_from_cache(self, build_dir, working_dir, install_dir):
        cached_install_dir = self._cached_install_dir(build_dir)
        # TODO: should be pure Python, but there isn't a stdlib function
        # that allows the destination to already exist
        subprocess.check_call(["cp", "-rT", cached_install_dir, install_dir])
        with open(os.path.join(install_dir, ".whack-root-id"), "w") as root_id_file:
            root_id_file.write(str(uuid.uuid4()))

    def _cached_install_dir(self, build_dir):
        return os.path.join(build_dir, "install")

_templates = {
    "fixed-root": FixedRootTemplate(),
    "relocatable": RelocatableTemplate()
}

_default_template_name = "relocatable"

class PackageInstaller(object):
    def __init__(self, package_dir, cacher):
        self._package_dir = package_dir
        self._cacher = cacher

    def install(self, install_dir, params={}):
        install_id = _generate_install_id(self._package_dir, params)

        with create_temporary_dir() as temp_dir:
            build_dir = os.path.join(temp_dir, "build")
            working_dir = os.path.join(build_dir, "workspace")

            result = self._cacher.fetch(install_id, build_dir)
            if not result.cache_hit:
                self._build(build_dir, working_dir, params)
                self._cacher.put(install_id, build_dir)

            self._template().install_from_cache(build_dir, working_dir, install_dir)

    def _build(self, build_dir, working_dir, params):
        ignore = shutil.ignore_patterns(".svn", ".hg", ".hgignore", ".git", ".gitignore")
        shutil.copytree(self._package_dir, working_dir, ignore=ignore)
        build_env = params_to_build_env(params)
        self._fetch_downloads(working_dir, build_env)

        def run(command):
            subprocess.check_call(
                command,
                cwd=working_dir,
                env=build_env
            )

        self._template().install_to_cache(run, build_dir, working_dir)

    def _template(self):
        return _templates[self._template_name()]

    def _fetch_downloads(self, build_dir, build_env):
        downloads_file_path = os.path.join(build_dir, "whack/downloads")
        downloads_file = self._read_downloads_file(downloads_file_path, build_env)
        for download_line in downloads_file:
            downloads.download(download_line.url, os.path.join(build_dir, download_line.filename))

    def _read_downloads_file(self, path, build_env):
        if os.path.exists(path):
            first_line = open(path).readline()
            if first_line.startswith("#!"):
                downloads_string = subprocess.check_output(
                    [path],
                    env=build_env
                )
            else:
                downloads_string = open(path).read()

            return downloads.read_downloads_string(downloads_string)
        else:
            return []

    def _template_name(self):
        return _read_package_description(self._package_dir).template_name

def params_to_build_env(params):
    build_env = os.environ.copy()
    for name, value in (params or {}).iteritems():
        build_env[name.upper()] = str(value)
    return build_env

def _generate_install_id(package_dir, params):
    hasher = Hasher()
    hasher.update(_uname("--kernel-name"))
    hasher.update(_uname("--machine"))
    hasher.update_with_dir(package_dir)
    hasher.update(json.dumps(params, sort_keys=True))
    return hasher.hexdigest()

def _uname(arg):
    return subprocess.check_output(["uname", arg])

def _read_package_description(package_dir):
    whack_json_path = os.path.join(package_dir, "whack/whack.json")
    if os.path.exists(whack_json_path):
        with open(whack_json_path, "r") as whack_json_file:
            whack_json = json.load(whack_json_file)
        return DictBackedPackageDescription(whack_json)
    else:
        return DefaultPackageDescription()

class DefaultPackageDescription(object):
    template_name = _default_template_name

class DictBackedPackageDescription(object):
    def __init__(self, values):
        self._values = values

    @property
    def template_name(self):
        return self._values.get("template", _default_template_name)
<|endoftext|>