<commit_before>import client
import csv
import queries
import itertools
from operator import itemgetter

hostname = "dbserver02.cs.washington.edu"
port = "10032"

client.init_connection(hostname, port)

def experiment(filename, exp_queries):
    with open(filename, "wb") as f:
        writer = csv.writer(f)
        writer.writerow(
            ["name", "qid", "time", "algebra",
             "profilingMode", "output", "success"])
        for query in exp_queries:
            # submit queries
            lang, algebra, profile, query_str, name = query
            if query_str.find("store") == -1:
                output_name = "{}_{}".format(name, algebra)
                query_str += "store(query, {});\n".format(output_name)
            else:
                raise ValueError("stored relation cannot be pre-defined.")
            actual_query = (lang, algebra, profile, query_str, name)
            result, status = client.execute_query(actual_query)
            # log experiment result
            if result == 'success':
                print "success"
                time = float(status["elapsedNanos"]) / client.NANO_IN_ONE_SEC
                writer.writerow(
                    [name, status["queryId"], time,
                     algebra, profile, output_name, result])
            else:
                print "error"
                writer.writerow(
                    [name, "N.A.", "N.A.", algebra,
                     profile, output_name, result])

exp_raw_queries = [
    (queries.triangle, 'triangle'),
    (queries.fb_q1, 'fb_q1'),
    (queries.rectangle, 'rectangle'),
    (queries.fb_q2, 'fb_q2'),
    (queries.two_rings, 'two_rings'),
    (queries.fb_q3, 'fb_q3'),
    (queries.clique, 'clique'),
    (queries.fb_q4, 'fb_q4')
]

small_set_queries = [
    (queries.triangle, 'triangle'),
    (queries.fb_q1, 'fb_q1'),
    (queries.clique, 'clique'),
    (queries.fb_q1, 'fb_q1')
]

phys_algebras = [
    ('RS_HJ',),
    ('HC_HJ',),
    ('BR_HJ',),
    ('RS_LFJ',),
    ('HC_LFJ',),
    ('BR_LFJ',)
]

languages = [('myrial',)]

# experiment 1:  resouce usage
def resource_exp():
    profilingModes = [('RESOURCE',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, small_set_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment("resource_exp_3_aug.csv", exp_queries)

# experiment 2: profile query execution only
def profile_exp():
    profilingModes = [('QUERY',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, small_set_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment("profile_exp_3_aug.csv", exp_queries)

# experiment 3: cold cache experiment
def cold_cache_exp(filename):
    profilingModes = [('NONE',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, small_set_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 4: regular shuffle with leapfrog join
def cold_cache_rslfj_exp(filename):
    profilingModes = [('NONE',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 5: rs lfj profile
def profile_rslfj(filename):
    profilingModes = [('QUERY',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 6: rs lfj resource
def resource_rslfj(filename):
    profilingModes = [('RESOURCE',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

def collect_network_data(query_id):
    """collect skew and """
    send = list(client.connection.get_sent_logs(query_id))
    # sanitiy checking
    assert len(send[0]) == 4
    # remove header
    send = send[1:]
    # convert to long
    send = [map(long, r) for r in send]
    # compute total number
    total_num_tuples = sum([r[3] for r in send])
    # group by fragmentid
    key_func = lambda x: x[1]
    send = sorted(send, key=lambda x: x[1])
    sends = itertools.groupby(send, key_func)
    groups = []
    for fid, shuffles in sends:
        groups.append([r for r in shuffles])

    # compute skew per group
    def compute_skew(group):
        # group by destination
        received = sorted(group, key=itemgetter(2))
        received = itertools.groupby(received, key=itemgetter(2))
        r_groups = []
        for r_wid, grp in received:
            r_groups.append([r[3] for r in grp])
        data = [sum(grp) for grp in r_groups]
        skew = max(data)/(sum(data)/64.0)
        # if the data is not significant, ignore it
        if sum(data) < total_num_tuples/64:
            skew = 1
        return skew
    skews = map(compute_skew, groups)
    # return number of tuples being shuffled, and max skew
    print skews
    return total_num_tuples, max(skews)

def add_network_data(query_file, output_file):
    with open(query_file, "rU") as f:
        csvreader = csv.reader(f)
        data = [r for r in csvreader]
        data[0].extend(["shuffled tuples", "max skew"])
        for i, row in enumerate(data):
            if i != 0:
                shuffled_tupels, max_skew = collect_network_data(row[1])
                data[i].extend([shuffled_tupels, max_skew])

    with open(output_file, "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerows(data)

def collect_resource_data(query_id):
    resource = list(client.connection.get_resource_log(query_id))
    # remove header
    resource = resource[1:]
    # convert numeric fields to long
    # row = [timestamp,opId,measurement,value,queryId,subqueryId,workerId]
    resource = [
        [long(row[0]),
         long(row[1]),
         row[2],
         long(row[3]),
         long(row[4]),
         long(row[5]),
         long(row[6])] for row in resource]
    # filter out row with zero value
    resource = filter(lambda row: row[3] != 0, resource)
    # group by opId, measurement and worker
    resource = sorted(resource, key=itemgetter(1, 2, 6))
    resource = itertools.groupby(resource, key=itemgetter(1, 2, 6))
    groups = []
    for key, grp in resource:
        groups.append([row for row in grp])
    resource = [max(grp, key=itemgetter(0)) for grp in groups]
    # store processed raw data as csv
    with open("query_{}_resource.csv".format(query_id), "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerow(["opId", "measure", "value", "queryId", "workerId"])
        for row in resource:
            csvwriter.writerow([row[1], row[2], row[3], row[4], row[6]])
    # group by measurement
    resource = sorted(resource, key=itemgetter(2))
    resource = itertools.groupby(resource, key=itemgetter(2))
    groups = []
    for key, grp in resource:
        groups.append([row for row in grp])
    result = []
    for grp in groups:
        values = [row[3] for row in grp]
        result.append([grp[0][2], sum(values)])
    cpu, hash_table_size = 0, 0
    for measure, value in result:
        if measure == 'cpuTotal':
            cpu = value
        elif measure == 'hashTableSize':
            hash_table_size = value
    return cpu, hash_table_size

def add_resource_data(query_file, output_file):
    with open(query_file, "rU") as f:
        csvreader = csv.reader(f)
        data = [r for r in csvreader]
        data[0].extend(["cpu time", "hash table size"])
        for i, row in enumerate(data):
            if i != 0:
                cpu, ht_size = collect_resource_data(row[1])
                data[i].extend([cpu, ht_size])

    with open(output_file, "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerows(data)

def get_output_skew(table_name):
    stored_count_table_name = "cnt_{}".format(table_name)
    query = """
    R = scan({});
    output = [from R emit count(*) as cnt, worker_id() as worker];
    store(output, {});
    """.format(table_name, stored_count_table_name)
    result, status = client.execute_query(
        ('myrial', 'RS_HJ', 'NONE', query, 'get_skew'))
    if result == 'success':
        print "success"
        rel_key = {
            'userName': 'public',
            'programName': 'adhoc',
            'relationName': stored_count_table_name
        }
        tpl_cnts = client.connection.download_dataset(rel_key)
        tpl_cnts = [long(entry['cnt']) for entry in tpl_cnts]
        skew = max(tpl_cnts) / (sum(tpl_cnts) / float(num_servers))
        return skew
    else:
        print "error"
        raise Exception(status)

if __name__ == '__main__':
    cold_cache_exp("cold_cache_small_1.csv")
    cold_cache_exp("cold_cache_small_2.csv")
    cold_cache_exp("cold_cache_small_3.csv")
    cold_cache_exp("cold_cache_small_4.csv")
    cold_cache_exp("cold_cache_small_5.csv")
    #resource_exp()
    #profile_exp()
    #print get_output_skew('rect_e5_skew')
    # add_network_data(
    #    "/Users/chushumo/Project/papers/2014-multiwayjoin/profile_rslfj.csv",
    #    "profile_rslfj_extend_30_july.csv")
    #add_network_data(
    #    "/Users/chushumo/Project/papers/2014-multiwayjoin/profile_exp.csv",
    #    "profile_extend_30_july.csv")
<commit_msg>adhoc setting changes<commit_after>import client
import csv
import queries
import itertools
from operator import itemgetter

hostname = "dbserver02.cs.washington.edu"
port = "10032"
num_servers = 64

client.init_connection(hostname, port)

def experiment(filename, exp_queries):
    with open(filename, "wb") as f:
        writer = csv.writer(f)
        writer.writerow(
            ["name", "qid", "time", "algebra",
             "profilingMode", "output", "success"])
        for query in exp_queries:
            # submit queries
            lang, algebra, profile, query_str, name = query
            if query_str.find("store") == -1:
                output_name = "{}_{}".format(name, algebra)
                query_str += "store(query, {});\n".format(output_name)
            else:
                raise ValueError("stored relation cannot be pre-defined.")
            actual_query = (lang, algebra, profile, query_str, name)
            result, status = client.execute_query(actual_query)
            # log experiment result
            if result == 'success':
                print "success"
                time = float(status["elapsedNanos"]) / client.NANO_IN_ONE_SEC
                writer.writerow(
                    [name, status["queryId"], time,
                     algebra, profile, output_name, result])
            else:
                print "error"
                writer.writerow(
                    [name, "N.A.", "N.A.", algebra,
                     profile, output_name, result])

exp_raw_queries = [
    (queries.triangle, 'triangle'),
    (queries.fb_q1, 'fb_q1'),
    (queries.rectangle, 'rectangle'),
    (queries.fb_q2, 'fb_q2'),
    (queries.two_rings, 'two_rings'),
    (queries.fb_q3, 'fb_q3'),
    (queries.clique, 'clique'),
    (queries.fb_q4, 'fb_q4')
]

small_set_queries = [
    (queries.triangle, 'triangle'),
    (queries.fb_q1, 'fb_q1'),
    (queries.clique, 'clique'),
    (queries.fb_q1, 'fb_q1')
]

only_q5 = [
    (queries.fb_q5, 'fb_q5'),
]

rect_query = [
    (queries.rectangle, 'rectangle'),
]

phys_algebras = [
    ('RS_HJ',),
    ('HC_HJ',),
    ('BR_HJ',),
    ('RS_LFJ',),
    ('HC_LFJ',),
    ('BR_LFJ',)
]

phys_algebra_without_rs_lfj = [
    ('RS_HJ',),
    ('HC_HJ',),
    ('BR_HJ',),
    ('HC_LFJ',),
    ('BR_LFJ',)
]

languages = [('myrial',)]

# experiment 1:  resouce usage
def resource_exp():
    profilingModes = [('RESOURCE',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, small_set_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment("resource_exp_3_aug.csv", exp_queries)

# experiment 2: profile query execution only
def profile_exp():
    profilingModes = [('QUERY',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, small_set_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment("profile_exp_3_aug.csv", exp_queries)

# experiment 3: cold cache experiment
def cold_cache_exp(filename):
    profilingModes = [('NONE',)]
    exp_queries = itertools.product(
        languages, phys_algebra_without_rs_lfj, profilingModes, rect_query)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 4: regular shuffle with leapfrog join
def cold_cache_rslfj_exp(filename):
    profilingModes = [('NONE',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 5: rs lfj profile
def profile_rslfj(filename):
    profilingModes = [('QUERY',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 6: rs lfj resource
def resource_rslfj(filename):
    profilingModes = [('RESOURCE',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

def collect_network_data(query_id):
    """collect skew and """
    send = list(client.connection.get_sent_logs(query_id))
    # sanitiy checking
    assert len(send[0]) == 4
    # remove header
    send = send[1:]
    # convert to long
    send = [map(long, r) for r in send]
    # compute total number
    total_num_tuples = sum([r[3] for r in send])
    # group by fragmentid
    key_func = lambda x: x[1]
    send = sorted(send, key=lambda x: x[1])
    sends = itertools.groupby(send, key_func)
    groups = []
    for fid, shuffles in sends:
        groups.append([r for r in shuffles])

    # compute skew per group
    def compute_skew(group):
        # group by destination
        received = sorted(group, key=itemgetter(2))
        received = itertools.groupby(received, key=itemgetter(2))
        r_groups = []
        for r_wid, grp in received:
            r_groups.append([r[3] for r in grp])
        data = [sum(grp) for grp in r_groups]
        skew = max(data)/(sum(data)/64.0)
        # if the data is not significant, ignore it
        if sum(data) < total_num_tuples/64:
            skew = 1
        return skew
    skews = map(compute_skew, groups)
    # return number of tuples being shuffled, and max skew
    print skews
    return total_num_tuples, max(skews)

def add_network_data(query_file, output_file):
    with open(query_file, "rU") as f:
        csvreader = csv.reader(f)
        data = [r for r in csvreader]
        data[0].extend(["shuffled tuples", "max skew"])
        for i, row in enumerate(data):
            if i != 0:
                shuffled_tupels, max_skew = collect_network_data(row[1])
                data[i].extend([shuffled_tupels, max_skew])

    with open(output_file, "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerows(data)

def collect_resource_data(query_id):
    resource = list(client.connection.get_resource_log(query_id))
    # remove header
    resource = resource[1:]
    # convert numeric fields to long
    # row = [timestamp,opId,measurement,value,queryId,subqueryId,workerId]
    resource = [
        [long(row[0]),
         long(row[1]),
         row[2],
         long(row[3]),
         long(row[4]),
         long(row[5]),
         long(row[6])] for row in resource]
    # filter out row with zero value
    resource = filter(lambda row: row[3] != 0, resource)
    # group by opId, measurement and worker
    resource = sorted(resource, key=itemgetter(1, 2, 6))
    resource = itertools.groupby(resource, key=itemgetter(1, 2, 6))
    groups = []
    for key, grp in resource:
        groups.append([row for row in grp])
    resource = [max(grp, key=itemgetter(0)) for grp in groups]
    # store processed raw data as csv
    with open("query_{}_resource.csv".format(query_id), "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerow(["opId", "measure", "value", "queryId", "workerId"])
        for row in resource:
            csvwriter.writerow([row[1], row[2], row[3], row[4], row[6]])
    # group by measurement
    resource = sorted(resource, key=itemgetter(2))
    resource = itertools.groupby(resource, key=itemgetter(2))
    groups = []
    for key, grp in resource:
        groups.append([row for row in grp])
    result = []
    for grp in groups:
        values = [row[3] for row in grp]
        result.append([grp[0][2], sum(values)])
    cpu, hash_table_size = 0, 0
    for measure, value in result:
        if measure == 'cpuTotal':
            cpu = value
        elif measure == 'hashTableSize':
            hash_table_size = value
    return cpu, hash_table_size

def add_resource_data(query_file, output_file):
    with open(query_file, "rU") as f:
        csvreader = csv.reader(f)
        data = [r for r in csvreader]
        data[0].extend(["cpu time", "hash table size"])
        for i, row in enumerate(data):
            if i != 0:
                cpu, ht_size = collect_resource_data(row[1])
                data[i].extend([cpu, ht_size])

    with open(output_file, "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerows(data)

def get_output_skew(table_name):
    stored_count_table_name = "cnt_{}".format(table_name)
    query = """
    R = scan({});
    output = [from R emit count(*) as cnt, worker_id() as worker];
    store(output, {});
    """.format(table_name, stored_count_table_name)
    result, status = client.execute_query(
        ('myrial', 'RS_HJ', 'NONE', query, 'get_skew'))
    if result == 'success':
        print "success"
        rel_key = {
            'userName': 'public',
            'programName': 'adhoc',
            'relationName': stored_count_table_name
        }
        tpl_cnts = client.connection.download_dataset(rel_key)
        tpl_cnts = [long(entry['cnt']) for entry in tpl_cnts]
        skew = max(tpl_cnts) / (sum(tpl_cnts) / float(num_servers))
        return skew
    else:
        print "error"
        raise Exception(status)

if __name__ == '__main__':
    cold_cache_exp("cold_cache_small_1.csv")
    cold_cache_exp("cold_cache_small_2.csv")
    cold_cache_exp("cold_cache_small_3.csv")
    cold_cache_exp("cold_cache_small_4.csv")
    cold_cache_exp("cold_cache_small_5.csv")
    #resource_exp()
    #profile_exp()
    #print get_output_skew('rect_e5_skew')
    # add_network_data(
    #    "/Users/chushumo/Project/papers/2014-multiwayjoin/profile_rslfj.csv",
    #    "profile_rslfj_extend_30_july.csv")
    #add_network_data(
    #    "/Users/chushumo/Project/papers/2014-multiwayjoin/profile_exp.csv",
    #    "profile_extend_30_july.csv")
<|endoftext|>


<commit_before>import re
from checks.log_parser import LogParserCheck, LogParser
from util import namedtuple, get_hostname

class NagiosParsingError(Exception): pass
class InvalidDataTemplate(Exception): pass

class Nagios(LogParserCheck):

    key = "Nagios"

    def __init__(self, name, init_config, agentConfig):
        LogParserCheck.__init__(self, name, init_config, agentConfig)

    def create_event(self, event_data):
        timestamp, event_type, fields = event_data

        # FIXME Oli: kind of ugly to have to go through a named dict for this, and inefficient too
        # but couldn't think of anything smarter
        d = fields._asdict()
        d.update({'timestamp': timestamp, 'event_type': event_type, 'api_key': self.agentConfig.get('api_key', '')})
        # if host is localhost, turn that into the internal host name
        host = d.get('host', None)
        if host == "localhost":
            d["host"] = get_hostname(self.agentConfig)

        self.log.debug("Nagios event: %s" % (d))
        self.event(d)

    def check(self, instance):

        # Check arguments
        cfg_path = instance.get('cfg_file', None)
        log_path = instance.get('log_file', None)
        event_log = instance.get('event_log', True)
        perf_data = instance.get('perf_data', False)
        if cfg_path is None and log_path is None:
            raise Exception("Not checking nagios because 'cfg_file' is not set in nagios config")

        tags = instance.get('tags', None)

        instance_key = self._instance_key(instance)

        parsers = []
        if instance_key in self.parsers:
            parsers = self.parsers[instance_key]

        # Build our tail -f
        if len(parsers) == 0:
            nagios_config = {}
            if cfg_path:
                nagios_config = self.parse_nagios_config(cfg_path)

                if perf_data:
                    host_parser = NagiosHostPerfData(self.log, nagios_config)
                    service_parser = NagiosServicePerfData(self.log, nagios_config)

                    if not host_parser.tail and not service_parser.tail:
                        raise Exception("The Nagios check is configured to check performance data, but failed to parse any of the perfdata files. Please ensure that the file and file_template fields for at least one of host or service perfdata is filled in correctly in the nagios config file.")

                    if host_parser.tail:
                        parsers.append(host_parser)
                    if service_parser.tail:
                        parsers.append(service_parser)

            if event_log:
                log_file = nagios_config.get('log_file', log_path)
                if not log_file:
                    raise Exception("Nagios check is configured to parse the event log, but failed to do so. Please ensure the value for log_file is correct.")

                nagios_log_parser = NagiosLogParser(self.log, log_file)
                parsers.append(nagios_log_parser)

            self.parsers[instance_key] = parsers

        # read until the end of file
        for parser in parsers:
            parser.parse_file()
            events = parser._get_events()
            for event in events:
                self.create_event(event)

            metrics = parser._get_metrics()
            for metric in metrics:
                self.gauge(metric['metric'], float(metric['value']), tags=tags, hostname=metric['hostname'], device_name=metric['device_name'])

    @staticmethod
    def parse_agent_config(agentConfig):
        cfg_file = agentConfig.get('nagios_perf_cfg', None)
        log_file = agentConfig.get('nagios_log', None)

        if not log_file and not cfg_file:
            return False

        instance = {}

        if cfg_file:
            instance['cfg_file'] = cfg_file
            instance['perf_data'] = True

        if log_file:
            instance['log_file'] = log_file
            instance['event_log'] = True

        return {
            'instances': [instance]
        }

    @classmethod
    def parse_nagios_config(cls, filename):
        output = {}
        keys = [
            'log_file',
            'host_perfdata_file_template',
            'service_perfdata_file_template',
            'host_perfdata_file',
            'service_perfdata_file',
        ]

        f = None
        try:
            f = open(filename)
            for line in f:
                line = line.strip()
                if not line:
                    continue
                for key in keys:
                    if line.startswith(key + '='):
                        eq_pos = line.find('=')
                        if eq_pos:
                            output[key] = line[eq_pos + 1:]
                            break
        finally:
            if f is not None:
                f.close()
        return output

class NagiosLogParser(LogParser):

    # fields order for each event type, as named tuples
    EVENT_FIELDS = {
        'CURRENT HOST STATE':       namedtuple('E_CurrentHostState', 'host, event_state, event_soft_hard, return_code, payload'),
        'CURRENT SERVICE STATE':    namedtuple('E_CurrentServiceState', 'host, check_name, event_state, event_soft_hard, return_code, payload'),
        'SERVICE ALERT':            namedtuple('E_ServiceAlert', 'host, check_name, event_state, event_soft_hard, return_code, payload'),
        'PASSIVE SERVICE CHECK':    namedtuple('E_PassiveServiceCheck', 'host, check_name, return_code, payload'),
        'HOST ALERT':               namedtuple('E_HostAlert', 'host, event_state, event_soft_hard, return_code, payload'),

        # [1305744274] SERVICE NOTIFICATION: ops;ip-10-114-237-165;Metric ETL;ACKNOWLEDGEMENT (CRITICAL);notify-service-by-email;HTTP CRITICAL: HTTP/1.1 503 Service Unavailable - 394 bytes in 0.010 second response time;datadog;alq
        'SERVICE NOTIFICATION':     namedtuple('E_ServiceNotification', 'contact, host, check_name, event_state, notification_type, payload'),

        # [1296509331] SERVICE FLAPPING ALERT: ip-10-114-97-27;cassandra JVM Heap;STARTED; Service appears to have started flapping (23.4% change >= 20.0% threshold)
        # [1296662511] SERVICE FLAPPING ALERT: ip-10-114-97-27;cassandra JVM Heap;STOPPED; Service appears to have stopped flapping (3.8% change < 5.0% threshold)
        'SERVICE FLAPPING ALERT':   namedtuple('E_FlappingAlert', 'host, check_name, flap_start_stop, payload'),

        # Reference for external commands: http://old.nagios.org/developerinfo/externalcommands/commandlist.php
        # Command Format:
        # ACKNOWLEDGE_SVC_PROBLEM;<host_name>;<service_description>;<sticky>;<notify>;<persistent>;<author>;<comment>
        # [1305832665] EXTERNAL COMMAND: ACKNOWLEDGE_SVC_PROBLEM;ip-10-202-161-236;Resources ETL;2;1;0;datadog;alq checking
        'ACKNOWLEDGE_SVC_PROBLEM': namedtuple('E_ServiceAck', 'host, check_name, sticky_ack, notify_ack, persistent_ack, ack_author, payload'),

        # Command Format:
        # ACKNOWLEDGE_HOST_PROBLEM;<host_name>;<sticky>;<notify>;<persistent>;<author>;<comment>
        'ACKNOWLEDGE_HOST_PROBLEM': namedtuple('E_HostAck', 'host, sticky_ack, notify_ack, persistent_ack, ack_author, payload'),

        # Host Downtime
        # [1297894825] HOST DOWNTIME ALERT: ip-10-114-89-59;STARTED; Host has entered a period of scheduled downtime
        # [1297894825] SERVICE DOWNTIME ALERT: ip-10-114-237-165;intake;STARTED; Service has entered a period of scheduled downtime

        'HOST DOWNTIME ALERT': namedtuple('E_HostDowntime', 'host, downtime_start_stop, payload'),
        'SERVICE DOWNTIME ALERT': namedtuple('E_ServiceDowntime', 'host, check_name, downtime_start_stop, payload'),
    }

    # Regex alternation ends up being tricker than expected, and much less readable
    RE_LINE_REG = re.compile('^\[(\d+)\] EXTERNAL COMMAND: (\w+);(.*)$')
    RE_LINE_EXT = re.compile('^\[(\d+)\] ([^:]+): (.*)$')

    def __init__(self, logger, log_path):
        LogParser.__init__(self, logger, log_path)

    def _parse_line(self, line):
        """Actual nagios parsing
        Return True if we found an event, False Otherwise
        """

        # We need to use try/catch here because a specific return value
        # of True or False is expected, and one line failing shouldn't
        # cause the entire thing to fail
        try:
            self._line_parsed = self._line_parsed + 1

            m = self.RE_LINE_REG.match(line)
            if m is None:
                m = self.RE_LINE_EXT.match(line)
            if m is None:
                return False

            (tstamp, event_type, remainder) = m.groups()
            tstamp = int(tstamp)

            # then retrieve the event format for each specific event type
            fields = self.EVENT_FIELDS.get(event_type, None)
            if fields is None:
                self.log.warn("Ignoring unkown nagios event for line: %s" % (line[:-1]))
                return False

            # and parse the rest of the line
            parts = map(lambda p: p.strip(), remainder.split(';'))
            # Chop parts we don't recognize
            parts = parts[:len(fields._fields)]

            self.event_data.append((tstamp, event_type, fields._make(parts)))

            return True
        except Exception, e:
            self.log.exception(e)
            return False

class NagiosPerfData(LogParser):
    perfdata_field = ''  # Should be overriden by subclasses
    metric_prefix = 'nagios'
    pair_pattern = re.compile(r"".join([
            r"'?(?P<label>[^=']+)'?=",
            r"(?P<value>[-0-9.]+)",
            r"(?P<unit>s|us|ms|%|B|KB|MB|GB|TB|c)?",
            r"(;(?P<warn>@?[-0-9.~]*:?[-0-9.~]*))?",
            r"(;(?P<crit>@?[-0-9.~]*:?[-0-9.~]*))?",
            r"(;(?P<min>[-0-9.]*))?",
            r"(;(?P<max>[-0-9.]*))?",
        ]))

    def __init__(self, logger, line_pattern, log_path):
        if isinstance(line_pattern, (str, unicode)):
            self.line_pattern = re.compile(line_pattern)
        else:
            self.line_pattern = line_pattern

        LogParser.__init__(self, logger, log_path)

    @staticmethod
    def template_regex(file_template):
        try:
            # Escape characters that will be interpreted as regex bits
            # e.g. [ and ] in "[SERVICEPERFDATA]"
            regex = re.sub(r'[[\]*]', r'.', file_template)
            regex = re.sub(r'\$([^\$]*)\$', r'(?P<\1>[^\$]*)', regex)
            return re.compile(regex)
        except Exception, e:
            raise InvalidDataTemplate("%s (%s)" % (file_template, e))

    @staticmethod
    def underscorize(s):
        return s.replace(' ', '_').lower()

    def _get_metric_prefix(self, data):
        raise NotImplementedError()

    def _parse_line(self, line):
        matched = self.line_pattern.match(line)
        if matched:
            data = matched.groupdict()
            metric_prefix = self._get_metric_prefix(data)

            # Parse the prefdata values, which are a space-delimited list of:
            #   'label'=value[UOM];[warn];[crit];[min];[max]
            perf_data = data.get(self.perfdata_field, '').split(' ')
            for pair in perf_data:
                pair_match = self.pair_pattern.match(pair)
                if not pair_match:
                    continue
                else:
                    pair_data = pair_match.groupdict()

                label = pair_data['label']
                timestamp = data.get('TIMET', '')
                value = pair_data['value']
                device_name = None

                if '/' in label:
                    # Special case: if the label begins
                    # with a /, treat the label as the device
                    # and use the metric prefix as the metric name
                    metric = '.'.join(metric_prefix)
                    device_name = label

                else:
                    # Otherwise, append the label to the metric prefix
                    # and use that as the metric name
                    metric = '.'.join(metric_prefix + [label])

                host_name = data.get('HOSTNAME', None)

                self.metric_data.append({
                    'metric': metric,
                    'value': value,
                    'hostname': host_name,
                    'device_name': device_name,
                    'timestamp': timestamp
                })
            # Matches have been processed
            return True

        # No matches
        return False

class NagiosHostPerfData(NagiosPerfData):
    perfdata_field = 'HOSTPERFDATA'

    def __init__(self, logger, nagios_config):
        host_perfdata_file_template = nagios_config.get('host_perfdata_file_template', None)
        host_perfdata_file = nagios_config.get('host_perfdata_file', None)

        self.tail = None
        if host_perfdata_file_template and host_perfdata_file:
            host_pattern = NagiosPerfData.template_regex(host_perfdata_file_template)
            NagiosPerfData.__init__(self, logger, host_pattern, host_perfdata_file)

    def _get_metric_prefix(self, line_data):
        return [self.metric_prefix, 'host']

class NagiosServicePerfData(NagiosPerfData):
    perfdata_field = 'SERVICEPERFDATA'

    def __init__(self, logger, nagios_config):
        service_perfdata_file_template = nagios_config.get('service_perfdata_file_template', None)
        service_perfdata_file = nagios_config.get('service_perfdata_file', None)

        self.tail = None
        if service_perfdata_file_template and service_perfdata_file:
            service_pattern = NagiosPerfData.template_regex(service_perfdata_file_template)
            return NagiosPerfData.__init__(self, logger, service_pattern, service_perfdata_file)

    def _get_metric_prefix(self, line_data):
        metric = [self.metric_prefix]
        middle_name = line_data.get('SERVICEDESC', None)
        if middle_name:
            metric.append(self.underscorize(middle_name))
        return metric
<commit_msg>Removed unneeded constructor calls<commit_after>import re
from checks.log_parser import LogParserCheck, LogParser
from util import namedtuple, get_hostname

class NagiosParsingError(Exception): pass
class InvalidDataTemplate(Exception): pass

class Nagios(LogParserCheck):

    key = "Nagios"

    def create_event(self, event_data):
        timestamp, event_type, fields = event_data

        # FIXME Oli: kind of ugly to have to go through a named dict for this, and inefficient too
        # but couldn't think of anything smarter
        d = fields._asdict()
        d.update({'timestamp': timestamp, 'event_type': event_type, 'api_key': self.agentConfig.get('api_key', '')})
        # if host is localhost, turn that into the internal host name
        host = d.get('host', None)
        if host == "localhost":
            d["host"] = get_hostname(self.agentConfig)

        self.log.debug("Nagios event: %s" % (d))
        self.event(d)

    def check(self, instance):

        # Check arguments
        cfg_path = instance.get('cfg_file', None)
        log_path = instance.get('log_file', None)
        event_log = instance.get('event_log', True)
        perf_data = instance.get('perf_data', False)
        if cfg_path is None and log_path is None:
            raise Exception("Not checking nagios because 'cfg_file' is not set in nagios config")

        tags = instance.get('tags', None)

        instance_key = self._instance_key(instance)

        parsers = []
        if instance_key in self.parsers:
            parsers = self.parsers[instance_key]

        # Build our tail -f
        if len(parsers) == 0:
            nagios_config = {}
            if cfg_path:
                nagios_config = self.parse_nagios_config(cfg_path)

                if perf_data:
                    host_parser = NagiosHostPerfData(self.log, nagios_config)
                    service_parser = NagiosServicePerfData(self.log, nagios_config)

                    if not host_parser.tail and not service_parser.tail:
                        raise Exception("The Nagios check is configured to check performance data, but failed to parse any of the perfdata files. Please ensure that the file and file_template fields for at least one of host or service perfdata is filled in correctly in the nagios config file.")

                    if host_parser.tail:
                        parsers.append(host_parser)
                    if service_parser.tail:
                        parsers.append(service_parser)

            if event_log:
                log_file = nagios_config.get('log_file', log_path)
                if not log_file:
                    raise Exception("Nagios check is configured to parse the event log, but failed to do so. Please ensure the value for log_file is correct.")

                nagios_log_parser = NagiosLogParser(self.log, log_file)
                parsers.append(nagios_log_parser)

            self.parsers[instance_key] = parsers

        # read until the end of file
        for parser in parsers:
            parser.parse_file()
            events = parser._get_events()
            for event in events:
                self.create_event(event)

            metrics = parser._get_metrics()
            for metric in metrics:
                self.gauge(metric['metric'], float(metric['value']), tags=tags, hostname=metric['hostname'], device_name=metric['device_name'])

    @staticmethod
    def parse_agent_config(agentConfig):
        cfg_file = agentConfig.get('nagios_perf_cfg', None)
        log_file = agentConfig.get('nagios_log', None)

        if not log_file and not cfg_file:
            return False

        instance = {}

        if cfg_file:
            instance['cfg_file'] = cfg_file
            instance['perf_data'] = True

        if log_file:
            instance['log_file'] = log_file
            instance['event_log'] = True

        return {
            'instances': [instance]
        }

    @classmethod
    def parse_nagios_config(cls, filename):
        output = {}
        keys = [
            'log_file',
            'host_perfdata_file_template',
            'service_perfdata_file_template',
            'host_perfdata_file',
            'service_perfdata_file',
        ]

        f = None
        try:
            f = open(filename)
            for line in f:
                line = line.strip()
                if not line:
                    continue
                for key in keys:
                    if line.startswith(key + '='):
                        eq_pos = line.find('=')
                        if eq_pos:
                            output[key] = line[eq_pos + 1:]
                            break
        finally:
            if f is not None:
                f.close()
        return output

class NagiosLogParser(LogParser):

    # fields order for each event type, as named tuples
    EVENT_FIELDS = {
        'CURRENT HOST STATE':       namedtuple('E_CurrentHostState', 'host, event_state, event_soft_hard, return_code, payload'),
        'CURRENT SERVICE STATE':    namedtuple('E_CurrentServiceState', 'host, check_name, event_state, event_soft_hard, return_code, payload'),
        'SERVICE ALERT':            namedtuple('E_ServiceAlert', 'host, check_name, event_state, event_soft_hard, return_code, payload'),
        'PASSIVE SERVICE CHECK':    namedtuple('E_PassiveServiceCheck', 'host, check_name, return_code, payload'),
        'HOST ALERT':               namedtuple('E_HostAlert', 'host, event_state, event_soft_hard, return_code, payload'),

        # [1305744274] SERVICE NOTIFICATION: ops;ip-10-114-237-165;Metric ETL;ACKNOWLEDGEMENT (CRITICAL);notify-service-by-email;HTTP CRITICAL: HTTP/1.1 503 Service Unavailable - 394 bytes in 0.010 second response time;datadog;alq
        'SERVICE NOTIFICATION':     namedtuple('E_ServiceNotification', 'contact, host, check_name, event_state, notification_type, payload'),

        # [1296509331] SERVICE FLAPPING ALERT: ip-10-114-97-27;cassandra JVM Heap;STARTED; Service appears to have started flapping (23.4% change >= 20.0% threshold)
        # [1296662511] SERVICE FLAPPING ALERT: ip-10-114-97-27;cassandra JVM Heap;STOPPED; Service appears to have stopped flapping (3.8% change < 5.0% threshold)
        'SERVICE FLAPPING ALERT':   namedtuple('E_FlappingAlert', 'host, check_name, flap_start_stop, payload'),

        # Reference for external commands: http://old.nagios.org/developerinfo/externalcommands/commandlist.php
        # Command Format:
        # ACKNOWLEDGE_SVC_PROBLEM;<host_name>;<service_description>;<sticky>;<notify>;<persistent>;<author>;<comment>
        # [1305832665] EXTERNAL COMMAND: ACKNOWLEDGE_SVC_PROBLEM;ip-10-202-161-236;Resources ETL;2;1;0;datadog;alq checking
        'ACKNOWLEDGE_SVC_PROBLEM': namedtuple('E_ServiceAck', 'host, check_name, sticky_ack, notify_ack, persistent_ack, ack_author, payload'),

        # Command Format:
        # ACKNOWLEDGE_HOST_PROBLEM;<host_name>;<sticky>;<notify>;<persistent>;<author>;<comment>
        'ACKNOWLEDGE_HOST_PROBLEM': namedtuple('E_HostAck', 'host, sticky_ack, notify_ack, persistent_ack, ack_author, payload'),

        # Host Downtime
        # [1297894825] HOST DOWNTIME ALERT: ip-10-114-89-59;STARTED; Host has entered a period of scheduled downtime
        # [1297894825] SERVICE DOWNTIME ALERT: ip-10-114-237-165;intake;STARTED; Service has entered a period of scheduled downtime

        'HOST DOWNTIME ALERT': namedtuple('E_HostDowntime', 'host, downtime_start_stop, payload'),
        'SERVICE DOWNTIME ALERT': namedtuple('E_ServiceDowntime', 'host, check_name, downtime_start_stop, payload'),
    }

    # Regex alternation ends up being tricker than expected, and much less readable
    RE_LINE_REG = re.compile('^\[(\d+)\] EXTERNAL COMMAND: (\w+);(.*)$')
    RE_LINE_EXT = re.compile('^\[(\d+)\] ([^:]+): (.*)$')

    def _parse_line(self, line):
        """Actual nagios parsing
        Return True if we found an event, False Otherwise
        """

        # We need to use try/catch here because a specific return value
        # of True or False is expected, and one line failing shouldn't
        # cause the entire thing to fail
        try:
            self._line_parsed = self._line_parsed + 1

            m = self.RE_LINE_REG.match(line)
            if m is None:
                m = self.RE_LINE_EXT.match(line)
            if m is None:
                return False

            (tstamp, event_type, remainder) = m.groups()
            tstamp = int(tstamp)

            # then retrieve the event format for each specific event type
            fields = self.EVENT_FIELDS.get(event_type, None)
            if fields is None:
                self.log.warn("Ignoring unkown nagios event for line: %s" % (line[:-1]))
                return False

            # and parse the rest of the line
            parts = map(lambda p: p.strip(), remainder.split(';'))
            # Chop parts we don't recognize
            parts = parts[:len(fields._fields)]

            self.event_data.append((tstamp, event_type, fields._make(parts)))

            return True
        except Exception, e:
            self.log.exception(e)
            return False

class NagiosPerfData(LogParser):
    perfdata_field = ''  # Should be overriden by subclasses
    metric_prefix = 'nagios'
    pair_pattern = re.compile(r"".join([
            r"'?(?P<label>[^=']+)'?=",
            r"(?P<value>[-0-9.]+)",
            r"(?P<unit>s|us|ms|%|B|KB|MB|GB|TB|c)?",
            r"(;(?P<warn>@?[-0-9.~]*:?[-0-9.~]*))?",
            r"(;(?P<crit>@?[-0-9.~]*:?[-0-9.~]*))?",
            r"(;(?P<min>[-0-9.]*))?",
            r"(;(?P<max>[-0-9.]*))?",
        ]))

    def __init__(self, logger, line_pattern, log_path):
        if isinstance(line_pattern, (str, unicode)):
            self.line_pattern = re.compile(line_pattern)
        else:
            self.line_pattern = line_pattern

        LogParser.__init__(self, logger, log_path)

    @staticmethod
    def template_regex(file_template):
        try:
            # Escape characters that will be interpreted as regex bits
            # e.g. [ and ] in "[SERVICEPERFDATA]"
            regex = re.sub(r'[[\]*]', r'.', file_template)
            regex = re.sub(r'\$([^\$]*)\$', r'(?P<\1>[^\$]*)', regex)
            return re.compile(regex)
        except Exception, e:
            raise InvalidDataTemplate("%s (%s)" % (file_template, e))

    @staticmethod
    def underscorize(s):
        return s.replace(' ', '_').lower()

    def _get_metric_prefix(self, data):
        raise NotImplementedError()

    def _parse_line(self, line):
        matched = self.line_pattern.match(line)
        if matched:
            data = matched.groupdict()
            metric_prefix = self._get_metric_prefix(data)

            # Parse the prefdata values, which are a space-delimited list of:
            #   'label'=value[UOM];[warn];[crit];[min];[max]
            perf_data = data.get(self.perfdata_field, '').split(' ')
            for pair in perf_data:
                pair_match = self.pair_pattern.match(pair)
                if not pair_match:
                    continue
                else:
                    pair_data = pair_match.groupdict()

                label = pair_data['label']
                timestamp = data.get('TIMET', '')
                value = pair_data['value']
                device_name = None

                if '/' in label:
                    # Special case: if the label begins
                    # with a /, treat the label as the device
                    # and use the metric prefix as the metric name
                    metric = '.'.join(metric_prefix)
                    device_name = label

                else:
                    # Otherwise, append the label to the metric prefix
                    # and use that as the metric name
                    metric = '.'.join(metric_prefix + [label])

                host_name = data.get('HOSTNAME', None)

                self.metric_data.append({
                    'metric': metric,
                    'value': value,
                    'hostname': host_name,
                    'device_name': device_name,
                    'timestamp': timestamp
                })
            # Matches have been processed
            return True

        # No matches
        return False

class NagiosHostPerfData(NagiosPerfData):
    perfdata_field = 'HOSTPERFDATA'

    def __init__(self, logger, nagios_config):
        host_perfdata_file_template = nagios_config.get('host_perfdata_file_template', None)
        host_perfdata_file = nagios_config.get('host_perfdata_file', None)

        self.tail = None
        if host_perfdata_file_template and host_perfdata_file:
            host_pattern = NagiosPerfData.template_regex(host_perfdata_file_template)
            NagiosPerfData.__init__(self, logger, host_pattern, host_perfdata_file)

    def _get_metric_prefix(self, line_data):
        return [self.metric_prefix, 'host']

class NagiosServicePerfData(NagiosPerfData):
    perfdata_field = 'SERVICEPERFDATA'

    def __init__(self, logger, nagios_config):
        service_perfdata_file_template = nagios_config.get('service_perfdata_file_template', None)
        service_perfdata_file = nagios_config.get('service_perfdata_file', None)

        self.tail = None
        if service_perfdata_file_template and service_perfdata_file:
            service_pattern = NagiosPerfData.template_regex(service_perfdata_file_template)
            return NagiosPerfData.__init__(self, logger, service_pattern, service_perfdata_file)

    def _get_metric_prefix(self, line_data):
        metric = [self.metric_prefix]
        middle_name = line_data.get('SERVICEDESC', None)
        if middle_name:
            metric.append(self.underscorize(middle_name))
        return metric
<|endoftext|>


<commit_before>import copy
import datetime

from botocore.parsers import ResponseParserError
from botocore.exceptions import ClientError, EndpointConnectionError

from classes.tail_ec2_instance import TailEC2Instance

class TailEBEnvironment(object):

    def __init__(self, eb_env_alias, eb_client, ec2_client, config, environment_dict, logger):

        # EB & EC2 clients
        self.eb_client = eb_client
        self.ec2_client = ec2_client

        # Config
        self.eb_env_alias = eb_env_alias
        self.environment_id = ''
        self.environment_name = ''

        if 'id' in config:
            self.environment_id = config['id']
        if 'name' in config:
            self.environment_name = config['name']

        self.key_pem = config['key_pem']
        self.files = config['files']
        self.keep_results_on_disk = config['keep_results_on_disk'] if 'keep_results_on_disk' in config else True
        self.user = config['user'] if 'user' in config else 'ec2-user'
        self.use_private_ip = config['use_private_ip'] if 'use_private_ip' in config else False
        self.api_endpoint = config['api_endpoint'] if 'api_endpoint' in config else None

        # EC2 host to tail logs
        self.hosts = {}

        # Dictionary of one EB environment to keep track of previously retrieved logs
        self.environment_dict = environment_dict

        self.logger = logger

    def find_instances(self):
        """
        Retrieves EC2 instance identifiers for the given EB environment
        :return:
        """
        eb_env_args = {}
        if len(self.environment_name) >= 4:
            eb_env_args["EnvironmentName"] = self.environment_name
        if self.environment_id != '':
            eb_env_args["EnvironmentId"] = self.environment_id

        try:
            responses = self.eb_client.describe_environment_resources(**eb_env_args)
            resources = responses['EnvironmentResources']
            for instance in resources['Instances']:
                self.hosts[instance['Id']] = {}
                self.logger.info("{eb_env}: Found instance {instance}".format(eb_env= self.eb_env_alias, instance=instance['Id']))
        except EndpointConnectionError as e:
            raise e

    def find_ec2_instance_hosts(self):
        """
        Looks for EC2 instance hosts based on the EC2 identifiers retrieved
        :return:
        """
        try:
            responses = self.ec2_client.describe_instances(InstanceIds=list(self.hosts.keys()))
            for reservation in responses['Reservations']:
                for instance in reservation['Instances']:
                    ip = instance['PrivateIpAddress'] if self.use_private_ip else instance['PublicIpAddress']
                    self.logger.info("{eb_env}: Found host {ip} for instance {instance}".format(eb_env=self.eb_env_alias, ip=ip, instance=instance['InstanceId']))
                    self.hosts[instance['InstanceId']] = ip
        except ClientError as e:
            raise e
        except EndpointConnectionError as e:
            raise e

    def tail_ec2_hosts(self):
        """
        Tails logs of each EC2 host listed for this EN environment
        :return:
        """
        for instance_id in self.hosts.keys():

            if instance_id not in self.environment_dict.keys():
                self.environment_dict[instance_id] = {}

            self.environment_dict[instance_id] = TailEC2Instance(self.eb_env_alias, instance_id, self.hosts[instance_id],
                                                                 self.user, self.files, self.key_pem,
                                                                 self.environment_dict[instance_id], self.api_endpoint,
                                                                 self.keep_results_on_disk, self.logger).run()

    def run(self):
        """
        Orchestrates the tailing logs process for one EB environment
        :return:
        """
        try:
            self.logger.info("{eb_env}: Finding EC2 instances ...".format(eb_env=self.eb_env_alias))
            self.find_instances()

            self.logger.info("{eb_env}: Retrieving EC2 instance hosts ...".format(eb_env=self.eb_env_alias))
            self.find_ec2_instance_hosts()

            self.logger.info("{eb_env}: Tailing logs from EC2 hosts ...".format(eb_env=self.eb_env_alias))
            self.tail_ec2_hosts()

            self.logger.info("{eb_env}: Tailing logs completed".format(eb_env=self.eb_env_alias))

            # Add last date time updated
            self.environment_dict['last_time_updated'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

            # Updating dictionary of this EB environment to remove EC2 instances keys that don't exist anymore
            eb_dict_copy = copy.deepcopy(self.environment_dict)
            for ec2_instance_key in list(eb_dict_copy):
                if ec2_instance_key not in self.hosts.keys():
                    self.environment_dict.pop(ec2_instance_key, None)

        except ResponseParserError as e:
            self.logger.error("{eb_env}: {error}".format(eb_env=self.eb_env_alias, error=str(e)))
            self.logger.info("{eb_env}: Expected to be fixed in botocore 1.4.53".format(eb_env=self.eb_env_alias))
        except ClientError as e:
            message = str(e)
            if 'Error' in e.response and 'Message' in e.response['Error']:
                message = e.response['Error']['Message']

            self.logger.error("{eb_env}: {error}".format(eb_env=self.eb_env_alias, error=message))
        except EndpointConnectionError as e:
            self.logger.error("{eb_env}: {error}".format(eb_env=self.eb_env_alias, error=str(e)))
        except Exception as e:
            self.logger.error("{eb_env}: Type ({type}) - Error ({error})".format(eb_env=self.eb_env_alias, type=type(e), error=str(e)))
        finally:
            return self.environment_dict
<commit_msg>updated TailEBEnvironment init method<commit_after>import copy
import datetime

from botocore.parsers import ResponseParserError
from botocore.exceptions import ClientError, EndpointConnectionError

from classes.tail_ec2_instance import TailEC2Instance

class TailEBEnvironment(object):

    def __init__(self, eb_env_alias, eb_client, ec2_client, config, environment_dict, logger):

        # EB & EC2 clients
        self.eb_client = eb_client
        self.ec2_client = ec2_client

        # Config
        self.eb_env_alias = eb_env_alias
        self.environment_id = config['id'] if 'id' in config else ''
        self.environment_name = config['name'] if 'name' in config else ''

        self.key_pem = config['key_pem']
        self.files = config['files']
        self.keep_results_on_disk = config['keep_results_on_disk'] if 'keep_results_on_disk' in config else True
        self.user = config['user'] if 'user' in config else 'ec2-user'
        self.use_private_ip = config['use_private_ip'] if 'use_private_ip' in config else False
        self.api_endpoint = config['api_endpoint'] if 'api_endpoint' in config else None

        # EC2 host to tail logs
        self.hosts = {}

        # Dictionary of one EB environment to keep track of previously retrieved logs
        self.environment_dict = environment_dict

        self.logger = logger

    def find_instances(self):
        """
        Retrieves EC2 instance identifiers for the given EB environment
        :return:
        """
        eb_env_args = {}
        if len(self.environment_name) >= 4:
            eb_env_args["EnvironmentName"] = self.environment_name
        if self.environment_id != '':
            eb_env_args["EnvironmentId"] = self.environment_id

        try:
            responses = self.eb_client.describe_environment_resources(**eb_env_args)
            resources = responses['EnvironmentResources']
            for instance in resources['Instances']:
                self.hosts[instance['Id']] = {}
                self.logger.info("{eb_env}: Found instance {instance}".format(eb_env= self.eb_env_alias, instance=instance['Id']))
        except EndpointConnectionError as e:
            raise e

    def find_ec2_instance_hosts(self):
        """
        Looks for EC2 instance hosts based on the EC2 identifiers retrieved
        :return:
        """
        try:
            responses = self.ec2_client.describe_instances(InstanceIds=list(self.hosts.keys()))
            for reservation in responses['Reservations']:
                for instance in reservation['Instances']:
                    ip = instance['PrivateIpAddress'] if self.use_private_ip else instance['PublicIpAddress']
                    self.logger.info("{eb_env}: Found host {ip} for instance {instance}".format(eb_env=self.eb_env_alias, ip=ip, instance=instance['InstanceId']))
                    self.hosts[instance['InstanceId']] = ip
        except ClientError as e:
            raise e
        except EndpointConnectionError as e:
            raise e

    def tail_ec2_hosts(self):
        """
        Tails logs of each EC2 host listed for this EN environment
        :return:
        """
        for instance_id in self.hosts.keys():

            if instance_id not in self.environment_dict.keys():
                self.environment_dict[instance_id] = {}

            self.environment_dict[instance_id] = TailEC2Instance(self.eb_env_alias, instance_id, self.hosts[instance_id],
                                                                 self.user, self.files, self.key_pem,
                                                                 self.environment_dict[instance_id], self.api_endpoint,
                                                                 self.keep_results_on_disk, self.logger).run()

    def run(self):
        """
        Orchestrates the tailing logs process for one EB environment
        :return:
        """
        try:
            self.logger.info("{eb_env}: Finding EC2 instances ...".format(eb_env=self.eb_env_alias))
            self.find_instances()

            self.logger.info("{eb_env}: Retrieving EC2 instance hosts ...".format(eb_env=self.eb_env_alias))
            self.find_ec2_instance_hosts()

            self.logger.info("{eb_env}: Tailing logs from EC2 hosts ...".format(eb_env=self.eb_env_alias))
            self.tail_ec2_hosts()

            self.logger.info("{eb_env}: Tailing logs completed".format(eb_env=self.eb_env_alias))

            # Add last date time updated
            self.environment_dict['last_time_updated'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

            # Updating dictionary of this EB environment to remove EC2 instances keys that don't exist anymore
            eb_dict_copy = copy.deepcopy(self.environment_dict)
            for ec2_instance_key in list(eb_dict_copy):
                if ec2_instance_key not in self.hosts.keys():
                    self.environment_dict.pop(ec2_instance_key, None)

        except ResponseParserError as e:
            self.logger.error("{eb_env}: {error}".format(eb_env=self.eb_env_alias, error=str(e)))
            self.logger.info("{eb_env}: Expected to be fixed in botocore 1.4.53".format(eb_env=self.eb_env_alias))
        except ClientError as e:
            message = str(e)
            if 'Error' in e.response and 'Message' in e.response['Error']:
                message = e.response['Error']['Message']

            self.logger.error("{eb_env}: {error}".format(eb_env=self.eb_env_alias, error=message))
        except EndpointConnectionError as e:
            self.logger.error("{eb_env}: {error}".format(eb_env=self.eb_env_alias, error=str(e)))
        except Exception as e:
            self.logger.error("{eb_env}: Type ({type}) - Error ({error})".format(eb_env=self.eb_env_alias, type=type(e), error=str(e)))
        finally:
            return self.environment_dict
<|endoftext|>


<commit_before>from flask import Flask
from flask import jsonify
from flask_cors import CORS
import math
import pandas as pd
import os
import datetime
import numpy as np
import time

app = Flask(__name__)
cors = CORS(app)

"""
CODE SNIPPED TO PRINT COLUMNS AND COLUMN NUMBERS
f = open('../crash_data/intersections.csv', 'r')
cols = f.readline().strip().split(',')
for i in range(len(cols)): print i, cols[i]
key = {}
for i in range(len(cols)): key[cols[i]] = i
"""
key = {'NUMBER OF CYCLIST KILLED': 0, 'ZIP CODE': 5, 'FOUR_SQUARE_DRINK': 21, 'YELP_SHOW': 20, 'YELP_DRINK': 17, 'OFF STREET NAME': 15, 'NUMBER OF MOTORIST KILLED': 8, 'YELP_MUSIC': 19, 'ON STREET NAME': 1, 'CROSS STREET NAME': 9, 'YELP_FOOD': 18, 'NUMBER OF MOTORIST INJURED': 4, 'LONGITUDE': 6, 'NUMBER OF PEDESTRIANS INJURED': 11, 'NUMBER OF PEDESTRIANS KILLED': 10, 'CRASHES': 2, 'NUMBER OF PERSONS KILLED': 3, 'NUMBER OF CYCLIST INJURED': 7, 'YELP_BAR': 16, 'LATITUDE': 12, 'NUMBER OF PERSONS INJURED': 13, 'BOROUGH': 14, 'FOUR_SQUARE_FOOD': 22}

@app.route("/")
def default():
    return jsonify({"msg":"Flask API working!"})

@app.route("/explore/")
@app.route("/explore/<danger>&<n>&<loc_type>&<loc_spec>&<start>&<end>", methods=['GET'])
def explore(danger=None, n=None, loc_type=None, loc_spec=None, start=0, end=0):
    #sort the intersection data by danger
    #return jsonify({'loc_type':loc_type, 'loc_spec':loc_spec,
    #                'start':start, 'end':end})
    start_time = time.time()
    #get for a specific time slice
    if start == '0':  # no time slice -- use pre-grouped
        df = pd.read_csv('crash_data/intersections.csv')
    else:
        # get specific time frame
        global crashes
        s = datetime.datetime.strptime(start, '%b %Y')
        e = datetime.datetime.strptime(end, '%b %Y')+datetime.timedelta(days=30)
        crashes = crashes[(crashes.DATE >= s) & (crashes.DATE <= e)]
        if len(crashes) == 0:  # no crashes in that time frame
            print "Zero crashes for that time period"
            return jsonify({'markers':[], 'success':False})

        # do the grouping procedure
        df = group(crashes)

    #get for the specific location
    if loc_type == "CITYWIDE":
        intersections = df.sort(danger, ascending=False).head(n=int(n)).iterrows()
    else:
        if loc_type is not "BOROUGH":
            loc_spec = int(loc_spec)
        intersections = df[df[loc_type]==loc_spec].sort(danger, ascending=False).head(n=int(n)).iterrows()

    #get top n and make them into markers
    j = {'markers':[], 'success':True}
    rank=0
    for intersection in intersections:
        lat = intersection[1].LATITUDE
        lon = intersection[1].LONGITUDE
        rank+=1
        main_text = "{0}
{1}".format(rank, intersection[1][danger])
        popup_text = """Rank: {0}

                        {1}: {2}

                        Street 1: {3}

                        Street 2: {4}"""
        popup_text = popup_text.format(rank,
                                       danger,
                                       intersection[1][danger],
                                       intersection[1]["ON STREET NAME"],
                                       intersection[1]["CROSS STREET NAME"])

        j['markers'].append({'lon':lon,
                             'lat':lat,
                             'main_text':main_text,
                             'popup_text':popup_text,
                             'color':'#%02x%02x%02x' % (255, 0, 0)
                             })
    return jsonify(j)

def group(crashes):
    """
    Helper function for doing the groupby operation when looking
    for crashes within a specific time slice.
    """
    #add index and crash counter
    crashes['INTERSECTION'] = crashes['LATITUDE'].apply(str) + "_" + crashes['LONGITUDE'].apply(str)
    crashes['CRASHES'] = 1

    #group and aggregate
    gb = crashes.groupby(by='INTERSECTION', as_index=False)
    longest = lambda x: x.get(x.map(lambda x: len(str(x))).idxmax())  # longest string
    mirror = lambda x: x.iget(0)
    agg_fun = {u'BOROUGH':mirror,
               u'DISTRICT':mirror,
               u'PRECINCT':mirror,
               u'ZIP CODE':mirror,
               u'LATITUDE':mirror,
               u'LONGITUDE':mirror,
               u'ON STREET NAME':mirror,
               u'CROSS STREET NAME':mirror,
               u'OFF STREET NAME':mirror,
               u'NUMBER OF PERSONS INJURED':np.sum,
               u'NUMBER OF PERSONS KILLED':np.sum,
               u'NUMBER OF PEDESTRIANS INJURED':np.sum,
               u'NUMBER OF PEDESTRIANS KILLED':np.sum,
               u'NUMBER OF CYCLIST INJURED':np.sum,
               u'NUMBER OF CYCLIST KILLED':np.sum,
               u'NUMBER OF MOTORIST INJURED':np.sum,
               u'NUMBER OF MOTORIST KILLED':np.sum,
               'CRASHES':np.sum
               }

    return gb.aggregate(agg_fun)

@app.route("/intersections/")
@app.route("/intersections/<danger>&<traffic>", methods=['GET'])
def intersections(danger=None, traffic=None):
    if not danger or not traffic:
        return jsonify({'error':True, "message":"Not all parameters specified", 'danger':danger, "traffic":traffic})
    if "FOUR SQUARE" in traffic:
        transform = lambda x: 20*math.log(x)
    else:
        transform = lambda x: x
    j = {'circles':[]}

    f = open('crash_data/intersections.csv', 'r')
    f.readline()  # skip header
    for line in f:
        line = line.strip().split(',')
        #if np.random.random() > 0.5:# and line[key['BOROUGH']] != 'MANHATTAN': #
        #    continue

        #add circles
        d,t = line[key[danger]], transform(line[key[traffic]])
        red = 255 - ((float(t)/20) * 255)
        green = 255 - min((255, int(d)))
        blue = 0
        color = '#%02x%02x%02x' % (red, green, blue)

        j['circles'].append({'lon':float(line[key['LONGITUDE']]),
                             'lat':float(line[key['LATITUDE']]),
                             'color':color})
    f.close()
    return jsonify(j)

@app.route("/humps/")
def humps():
    #return "hello world"
    j = {'markers':[]}
    f = open('intervention_data/speed_humps.csv', 'r')
    f.readline()  # skip header
    #return 'hello world'
    for line in f:
        line = line.strip().split(',')
        lat = float(line[-8].replace("(","").replace('"',''))
        lon = float(line[-7].replace(")","").replace('"',''))
        pd3 = float(line[-10])
        if pd3<0: color = '#%02x%02x%02x' % (255*-max((-1,pd3)), 0, 0)
        elif pd3 > 0: color = '#%02x%02x%02x' %  (0, 255*min((1,pd3)), 0)
        else: color = '#%02x%02x%02x' %  (0, 0, 255)
        text = """ Date Built: {0}

                3 month percent decrease: {1}

                6 month percent decrease:  {2}

                =========================

                Crashes 3 month before:  {3}

                Crashes 3 month after:  {4}

                Crashes 6 month before: {5}

                Crashes 6 month after: {6} """

        text = text.format(line[11], float(line[-11]), float(line[-10]), line[21],  line[20], line[23], line[22])

        j['markers'].append({'lon':lon,
                             'lat':lat,
                             'text':text,
                             'color':color
                             })
    return jsonify(j)

@app.route("/clusters/")
def clusters():
    #define some stuff
    j = {'circles':[]}
    colors = ["#FF00FF", "#000000", "#008000", "#FFFF00",
              "#0000FF", "#FF0000", "#00FFFF", "#FFFFFF"]

    #build the json to return
    intersections = pd.read_csv('crash_data/clean_intersections.csv')
    for row in intersections.iterrows():
        j['circles'].append({'lon':float(row[1].lon),
                             'lat':float(row[1].lat),
                             'color':colors[row[1].cluster%len(colors)]})
    return jsonify(j)

@app.route("/schools/") # NOT IMPLEMENTED YET!
def schools():
    #define some stuff
    j = {'circles':[]}
    colors = ["#FF00FF", "#000000", "#008000", "#FFFF00",
              "#0000FF", "#FF0000", "#00FFFF", "#FFFFFF"]

    #build the json to return
    intersections = pd.read_csv('crash_data/clean_intersections.csv')
    for row in intersections.iterrows():
        j['circles'].append({'lon':float(row[1].lon),
                             'lat':float(row[1].lat),
                             'color':colors[row[1].cluster%len(colors)]})

port = os.getenv('VCAP_APP_PORT', '5000')
if __name__ == "__main__":
    #preload crash data
    print "loading crash data..."
    crashes = pd.read_csv('crash_data/crashes_clean.csv')
    crashes.DATE = crashes.DATE.apply(pd.to_datetime)
    print 'done.'
    app.run(debug = True)
    #app.run(host='0.0.0.0', port=int(port))
<commit_msg>remove debug mode<commit_after>from flask import Flask
from flask import jsonify
from flask_cors import CORS
import math
import pandas as pd
import os
import datetime
import numpy as np
import time

app = Flask(__name__)
cors = CORS(app)

"""
CODE SNIPPED TO PRINT COLUMNS AND COLUMN NUMBERS
f = open('../crash_data/intersections.csv', 'r')
cols = f.readline().strip().split(',')
for i in range(len(cols)): print i, cols[i]
key = {}
for i in range(len(cols)): key[cols[i]] = i
"""
key = {'NUMBER OF CYCLIST KILLED': 0, 'ZIP CODE': 5, 'FOUR_SQUARE_DRINK': 21, 'YELP_SHOW': 20, 'YELP_DRINK': 17, 'OFF STREET NAME': 15, 'NUMBER OF MOTORIST KILLED': 8, 'YELP_MUSIC': 19, 'ON STREET NAME': 1, 'CROSS STREET NAME': 9, 'YELP_FOOD': 18, 'NUMBER OF MOTORIST INJURED': 4, 'LONGITUDE': 6, 'NUMBER OF PEDESTRIANS INJURED': 11, 'NUMBER OF PEDESTRIANS KILLED': 10, 'CRASHES': 2, 'NUMBER OF PERSONS KILLED': 3, 'NUMBER OF CYCLIST INJURED': 7, 'YELP_BAR': 16, 'LATITUDE': 12, 'NUMBER OF PERSONS INJURED': 13, 'BOROUGH': 14, 'FOUR_SQUARE_FOOD': 22}

@app.route("/")
def default():
    return jsonify({"msg":"Flask API working!"})

@app.route("/explore/")
@app.route("/explore/<danger>&<n>&<loc_type>&<loc_spec>&<start>&<end>", methods=['GET'])
def explore(danger=None, n=None, loc_type=None, loc_spec=None, start=0, end=0):
    #sort the intersection data by danger
    #return jsonify({'loc_type':loc_type, 'loc_spec':loc_spec,
    #                'start':start, 'end':end})
    start_time = time.time()
    #get for a specific time slice
    if start == '0':  # no time slice -- use pre-grouped
        df = pd.read_csv('crash_data/intersections.csv')
    else:
        # get specific time frame
        global crashes
        s = datetime.datetime.strptime(start, '%b %Y')
        e = datetime.datetime.strptime(end, '%b %Y')+datetime.timedelta(days=30)
        crashes = crashes[(crashes.DATE >= s) & (crashes.DATE <= e)]
        if len(crashes) == 0:  # no crashes in that time frame
            print "Zero crashes for that time period"
            return jsonify({'markers':[], 'success':False})

        # do the grouping procedure
        df = group(crashes)

    #get for the specific location
    if loc_type == "CITYWIDE":
        intersections = df.sort(danger, ascending=False).head(n=int(n)).iterrows()
    else:
        if loc_type is not "BOROUGH":
            loc_spec = int(loc_spec)
        intersections = df[df[loc_type]==loc_spec].sort(danger, ascending=False).head(n=int(n)).iterrows()

    #get top n and make them into markers
    j = {'markers':[], 'success':True}
    rank=0
    for intersection in intersections:
        lat = intersection[1].LATITUDE
        lon = intersection[1].LONGITUDE
        rank+=1
        main_text = "{0}
{1}".format(rank, intersection[1][danger])
        popup_text = """Rank: {0}

                        {1}: {2}

                        Street 1: {3}

                        Street 2: {4}"""
        popup_text = popup_text.format(rank,
                                       danger,
                                       intersection[1][danger],
                                       intersection[1]["ON STREET NAME"],
                                       intersection[1]["CROSS STREET NAME"])

        j['markers'].append({'lon':lon,
                             'lat':lat,
                             'main_text':main_text,
                             'popup_text':popup_text,
                             'color':'#%02x%02x%02x' % (255, 0, 0)
                             })
    return jsonify(j)

def group(crashes):
    """
    Helper function for doing the groupby operation when looking
    for crashes within a specific time slice.
    """
    #add index and crash counter
    crashes['INTERSECTION'] = crashes['LATITUDE'].apply(str) + "_" + crashes['LONGITUDE'].apply(str)
    crashes['CRASHES'] = 1

    #group and aggregate
    gb = crashes.groupby(by='INTERSECTION', as_index=False)
    longest = lambda x: x.get(x.map(lambda x: len(str(x))).idxmax())  # longest string
    mirror = lambda x: x.iget(0)
    agg_fun = {u'BOROUGH':mirror,
               u'DISTRICT':mirror,
               u'PRECINCT':mirror,
               u'ZIP CODE':mirror,
               u'LATITUDE':mirror,
               u'LONGITUDE':mirror,
               u'ON STREET NAME':mirror,
               u'CROSS STREET NAME':mirror,
               u'OFF STREET NAME':mirror,
               u'NUMBER OF PERSONS INJURED':np.sum,
               u'NUMBER OF PERSONS KILLED':np.sum,
               u'NUMBER OF PEDESTRIANS INJURED':np.sum,
               u'NUMBER OF PEDESTRIANS KILLED':np.sum,
               u'NUMBER OF CYCLIST INJURED':np.sum,
               u'NUMBER OF CYCLIST KILLED':np.sum,
               u'NUMBER OF MOTORIST INJURED':np.sum,
               u'NUMBER OF MOTORIST KILLED':np.sum,
               'CRASHES':np.sum
               }

    return gb.aggregate(agg_fun)

@app.route("/intersections/")
@app.route("/intersections/<danger>&<traffic>", methods=['GET'])
def intersections(danger=None, traffic=None):
    if not danger or not traffic:
        return jsonify({'error':True, "message":"Not all parameters specified", 'danger':danger, "traffic":traffic})
    if "FOUR SQUARE" in traffic:
        transform = lambda x: 20*math.log(x)
    else:
        transform = lambda x: x
    j = {'circles':[]}

    f = open('crash_data/intersections.csv', 'r')
    f.readline()  # skip header
    for line in f:
        line = line.strip().split(',')
        #if np.random.random() > 0.5:# and line[key['BOROUGH']] != 'MANHATTAN': #
        #    continue

        #add circles
        d,t = line[key[danger]], transform(line[key[traffic]])
        red = 255 - ((float(t)/20) * 255)
        green = 255 - min((255, int(d)))
        blue = 0
        color = '#%02x%02x%02x' % (red, green, blue)

        j['circles'].append({'lon':float(line[key['LONGITUDE']]),
                             'lat':float(line[key['LATITUDE']]),
                             'color':color})
    f.close()
    return jsonify(j)

@app.route("/humps/")
def humps():
    #return "hello world"
    j = {'markers':[]}
    f = open('intervention_data/speed_humps.csv', 'r')
    f.readline()  # skip header
    #return 'hello world'
    for line in f:
        line = line.strip().split(',')
        lat = float(line[-8].replace("(","").replace('"',''))
        lon = float(line[-7].replace(")","").replace('"',''))
        pd3 = float(line[-10])
        if pd3<0: color = '#%02x%02x%02x' % (255*-max((-1,pd3)), 0, 0)
        elif pd3 > 0: color = '#%02x%02x%02x' %  (0, 255*min((1,pd3)), 0)
        else: color = '#%02x%02x%02x' %  (0, 0, 255)
        text = """ Date Built: {0}

                3 month percent decrease: {1}

                6 month percent decrease:  {2}

                =========================

                Crashes 3 month before:  {3}

                Crashes 3 month after:  {4}

                Crashes 6 month before: {5}

                Crashes 6 month after: {6} """

        text = text.format(line[11], float(line[-11]), float(line[-10]), line[21],  line[20], line[23], line[22])

        j['markers'].append({'lon':lon,
                             'lat':lat,
                             'text':text,
                             'color':color
                             })
    return jsonify(j)

@app.route("/clusters/")
def clusters():
    #define some stuff
    j = {'circles':[]}
    colors = ["#FF00FF", "#000000", "#008000", "#FFFF00",
              "#0000FF", "#FF0000", "#00FFFF", "#FFFFFF"]

    #build the json to return
    intersections = pd.read_csv('crash_data/clean_intersections.csv')
    for row in intersections.iterrows():
        j['circles'].append({'lon':float(row[1].lon),
                             'lat':float(row[1].lat),
                             'color':colors[row[1].cluster%len(colors)]})
    return jsonify(j)

@app.route("/schools/") # NOT IMPLEMENTED YET!
def schools():
    #define some stuff
    j = {'circles':[]}
    colors = ["#FF00FF", "#000000", "#008000", "#FFFF00",
              "#0000FF", "#FF0000", "#00FFFF", "#FFFFFF"]

    #build the json to return
    intersections = pd.read_csv('crash_data/clean_intersections.csv')
    for row in intersections.iterrows():
        j['circles'].append({'lon':float(row[1].lon),
                             'lat':float(row[1].lat),
                             'color':colors[row[1].cluster%len(colors)]})

port = os.getenv('VCAP_APP_PORT', '5000')
if __name__ == "__main__":
    #preload crash data
    print "loading crash data..."
    crashes = pd.read_csv('crash_data/crashes_clean.csv')
    crashes.DATE = crashes.DATE.apply(pd.to_datetime)
    print 'done.'
    #app.run(debug = True)
    app.run(host='0.0.0.0', port=int(port))
<|endoftext|>