<commit_before>#!/usr/bin/env python
# encoding: utf-8

import sys
import pandas as pd
from toolshed import nopen, reader
from Bio import trie, triefind
from collections import Counter
from itertools import islice, ifilterfalse

def read_fastq(fh):
    """FASTQ parser that yields name, seq, and qual."""
    while True:
        values = list(islice(fh, 4))
        if len(values) == 4:
            id1, seq, id2, qual = values
        elif len(values) == 0:
            raise StopIteration
        else:
            raise EOFError("unexpected end of file")
        assert id1.startswith('@')
        assert id2.startswith('+')
        assert len(seq) == len(qual)
        yield id1[1:-1], seq[:-1], qual[:-1]

def trim_seq(seq, base=5):
    """round length of sequence to nearest `base`"""
    return seq[:int(base * round(len(seq)/base))]

def process_exact_fastq(fastq, n):
    """Group identical reads using a Counter. Returns Counter."""
    c = Counter()
    with nopen(fastq) as fh:
        for name, seq, qual in read_fastq(fh):
            seq = trim_seq(seq, 4)
            if len(seq) < n: continue
            c.update([seq])
    return c

def process_exact_txt(files, cutoff):
    """returns Counter from multiple quantify runs"""
    c = Counter()
    for f in files:
        for l in reader(f, header=['seq','count']):
            if l['count'] < cutoff: continue
            c.update([l['seq']])
    return c

def process_counted(fp):
    c = Counter()
    for l in reader(fp, header=['seq','count']):
        c[l['seq']] = int(l['count'])
    return c

def get_seq_bins(fp):
    c = Counter()
    for l in nopen(fp):
        c[l.strip()] = 0
    return c, sorted(set([len(i) for i in c.keys()]))

def chunker(it, n):
    # chunker('AAAABBBC', 4) --> AAAA AAAB AABB ABBB BBBC
    return [it[i:i+n] for i in xrange(0, len(it)+1-n, 1)]

def construct_simple_trie(counter):
    t = trie.trie()
    for seq, count in counter.iteritems():
        t[seq] = count
    return t

def construct_complex_trie(counter, lengths=None):
    t = trie.trie()
    seqs = list(counter)
    seqs.sort(key=len, reverse=True)
    if lengths is None:
        lengths = sorted(set([len(k) for k in seqs]))
    for seq in seqs:
        seq_len = len(seq)
        for l in lengths:
            if l > seq_len: continue
            for subseq in chunker(seq, l):
                if t.has_key(subseq): continue
                if subseq == seq:
                    t[seq] = counter[seq]
                else:
                    t[subseq] = seq
    return t

def process_exact_substring(counter, t):
    """use triefind.find to gather identical substring matches"""
    seqs = list(counter)
    seqs.sort(key=len, reverse=True)
    for seq in seqs:
        l = len(seq)
        for (match, start, end) in triefind.find(seq, t):
            if len(match) == l: continue
            counter[seq] += counter[match]
            counter[match] = 0
    counter += Counter()
    return counter

def unique_everseen(iterable, key=None):
    seen = set()
    seen_add = seen.add
    if key is None:
        for element in ifilterfalse(seen.__contains__, iterable):
            seen_add(element)
            yield element
    else:
        for element in iterable:
            k = key(element)
            if k not in seen:
                seen_add(k)
                yield element

def process_similar(counter, t, n):
    seqs = list(counter)
    seqs.sort(key=len, reverse=True)
    lengths = sorted(set([len(k) for k in seqs]))
    progress = 100
    to_process = len(seqs)
    for i, seq in enumerate(seqs, start=1):
        if i % progress == 0:
            progress = int(progress * 1.5)
            print >>sys.stderr, "processed %d of %d" % (i, to_process)
        if counter[seq] == 0: continue
        for (k, v, dist) in unique_everseen(t.get_approximate(seq, n), lambda (m,c,d): m):
            if dist == 0 or k == seq: continue
            if type(v) is int:
                counter[seq] += counter[k]
                counter[k] = 0
            else:
                # k is a subsequence; therefore add seq to v
                counter[v] += counter[seq]
                counter[seq] = 0
    counter += Counter()
    return counter

def process_similar_matrix(bins, seqs, t, n):
    binseqs = list(bins)
    binseqs.sort(key=len, reverse=True)
    to_process = len(binseqs)
    progress = 100
    for i, seq in enumerate(binseqs, start=1):
        if i % progress == 0:
            progress = int(progress * 1.5)
            print >>sys.stderr, "    >> processed %d of %d" % (i, to_process)
        for (k, v, dist) in unique_everseen(t.get_approximate(seq, n), lambda (m,c,d): m):
            # if type(v) is int:
            if type(v) is int:
                bins[seq] += seqs[seq]
                # set to zero? avoids adding counts to multiple bins
                seqs[seq] = 0
            else:
                bins[seq] += seqs[v]
                seqs[v] = 0
    return bins

def write_table(d):
    df = pd.DataFrame(d)
    df.to_csv(sys.stdout, sep="\t")
<commit_msg>cast int<commit_after>#!/usr/bin/env python
# encoding: utf-8

import sys
import pandas as pd
from toolshed import nopen, reader
from Bio import trie, triefind
from collections import Counter
from itertools import islice, ifilterfalse

def read_fastq(fh):
    """FASTQ parser that yields name, seq, and qual."""
    while True:
        values = list(islice(fh, 4))
        if len(values) == 4:
            id1, seq, id2, qual = values
        elif len(values) == 0:
            raise StopIteration
        else:
            raise EOFError("unexpected end of file")
        assert id1.startswith('@')
        assert id2.startswith('+')
        assert len(seq) == len(qual)
        yield id1[1:-1], seq[:-1], qual[:-1]

def trim_seq(seq, base=5):
    """round length of sequence to nearest `base`"""
    return seq[:int(base * round(len(seq)/base))]

def process_exact_fastq(fastq, n):
    """Group identical reads using a Counter. Returns Counter."""
    c = Counter()
    with nopen(fastq) as fh:
        for name, seq, qual in read_fastq(fh):
            seq = trim_seq(seq, 4)
            if len(seq) < n: continue
            c.update([seq])
    return c

def process_exact_txt(files, cutoff):
    """returns Counter from multiple quantify runs"""
    c = Counter()
    for f in files:
        for l in reader(f, header=['seq','count']):
            if int(l['count']) < cutoff: continue
            c.update([l['seq']])
    return c

def process_counted(fp):
    c = Counter()
    for l in reader(fp, header=['seq','count']):
        c[l['seq']] = int(l['count'])
    return c

def get_seq_bins(fp):
    c = Counter()
    for l in nopen(fp):
        c[l.strip()] = 0
    return c, sorted(set([len(i) for i in c.keys()]))

def chunker(it, n):
    # chunker('AAAABBBC', 4) --> AAAA AAAB AABB ABBB BBBC
    return [it[i:i+n] for i in xrange(0, len(it)+1-n, 1)]

def construct_simple_trie(counter):
    t = trie.trie()
    for seq, count in counter.iteritems():
        t[seq] = count
    return t

def construct_complex_trie(counter, lengths=None):
    t = trie.trie()
    seqs = list(counter)
    seqs.sort(key=len, reverse=True)
    if lengths is None:
        lengths = sorted(set([len(k) for k in seqs]))
    for seq in seqs:
        seq_len = len(seq)
        for l in lengths:
            if l > seq_len: continue
            for subseq in chunker(seq, l):
                if t.has_key(subseq): continue
                if subseq == seq:
                    t[seq] = counter[seq]
                else:
                    t[subseq] = seq
    return t

def process_exact_substring(counter, t):
    """use triefind.find to gather identical substring matches"""
    seqs = list(counter)
    seqs.sort(key=len, reverse=True)
    for seq in seqs:
        l = len(seq)
        for (match, start, end) in triefind.find(seq, t):
            if len(match) == l: continue
            counter[seq] += counter[match]
            counter[match] = 0
    counter += Counter()
    return counter

def unique_everseen(iterable, key=None):
    seen = set()
    seen_add = seen.add
    if key is None:
        for element in ifilterfalse(seen.__contains__, iterable):
            seen_add(element)
            yield element
    else:
        for element in iterable:
            k = key(element)
            if k not in seen:
                seen_add(k)
                yield element

def process_similar(counter, t, n):
    seqs = list(counter)
    seqs.sort(key=len, reverse=True)
    lengths = sorted(set([len(k) for k in seqs]))
    progress = 100
    to_process = len(seqs)
    for i, seq in enumerate(seqs, start=1):
        if i % progress == 0:
            progress = int(progress * 1.5)
            print >>sys.stderr, "processed %d of %d" % (i, to_process)
        if counter[seq] == 0: continue
        for (k, v, dist) in unique_everseen(t.get_approximate(seq, n), lambda (m,c,d): m):
            if dist == 0 or k == seq: continue
            if type(v) is int:
                counter[seq] += counter[k]
                counter[k] = 0
            else:
                # k is a subsequence; therefore add seq to v
                counter[v] += counter[seq]
                counter[seq] = 0
    counter += Counter()
    return counter

def process_similar_matrix(bins, seqs, t, n):
    binseqs = list(bins)
    binseqs.sort(key=len, reverse=True)
    to_process = len(binseqs)
    progress = 100
    for i, seq in enumerate(binseqs, start=1):
        if i % progress == 0:
            progress = int(progress * 1.5)
            print >>sys.stderr, "    >> processed %d of %d" % (i, to_process)
        for (k, v, dist) in unique_everseen(t.get_approximate(seq, n), lambda (m,c,d): m):
            # if type(v) is int:
            if type(v) is int:
                bins[seq] += seqs[seq]
                # set to zero? avoids adding counts to multiple bins
                seqs[seq] = 0
            else:
                bins[seq] += seqs[v]
                seqs[v] = 0
    return bins

def write_table(d):
    df = pd.DataFrame(d)
    df.to_csv(sys.stdout, sep="\t")
<|endoftext|>


<commit_before># Implement the cvmfsmon API
# A URL is of the form:
#  /cvmfsmon/api/v1.0/montests&param1=value1&param2=value2
# The URL is parsed by the calling function, and the params come in a
#   dictionary of keys, each with a list of values
# Currently supported "montests" are
#  ok - always returns OK
#  all - runs all applicable tests but 'ok'
#  updated - verifies that updates are happening on a stratum 1
#  gc - verifies that repositories that have done garbage collection
#       have done it successfully recently
# Currently supported parameters are
#  format - value one of the following (default: list)
#    status - reports only one line: OK, WARNING, or CRITICAL
#    list - reports a line for each available status, followed by colon,
#      followed by a comma-separated list of repositories at that status.
#    details - detailed json output with all the tests and messages
#  server - value is either 'local' (the default, indicating localhost) or
#    an alias of a server as defined in /etc/cvmfsmon/api.conf

import os, sys, socket, urllib2, anyjson, pprint, StringIO, string
import time, threading
import cvmfsmon_updated, cvmfsmon_gc

negative_expire_secs = 60*2         # 2 minutes
positive_expire_secs = 60*2         # 2 minutes
timeout_secs = 5                    # tries twice for 5 seconds
request_max_secs = 30               # maximum cache seconds when reading
config_update_time = 60             # seconds between checking config file

conf_mod_time = 0
last_config_time = 0
aliases = {}
excludes = []
limits = {}

def error_request(start_response, response_code, response_body):
    response_body = response_body + '\n'
    start_response(response_code,
                   [('Cache-control', 'max-age=' + str(negative_expire_secs)),
                    ('Content-Length', str(len(response_body)))])
    return [response_body]

def bad_request(start_response, reason):
    response_body = 'Bad Request: ' + reason
    return error_request(start_response, '400 Bad Request', response_body)

def good_request(start_response, response_body):
    response_code = '200 OK'
    start_response(response_code,
                  [('Content-Type', 'text/plain'),
                   ('Cache-Control', 'max-age=' + str(positive_expire_secs)),
                   ('Content-Length', str(len(response_body)))])
    return [response_body]

def parse_api_conf():
    global aliases, excludes, limits
    global conf_mod_time
    conffile = '/etc/cvmfsmon/api.conf'
    try:
        modtime = os.stat(conffile).st_mtime
        if modtime == conf_mod_time:
            # no change
            return
        conf_mod_time = modtime

        aliases = { 'local' : '127.0.0.1' }
        excludes = []
        limits = {
            'updated-warning': 8,
            'updated-critical': 24,
            'gc-warning': 10,
            'gc-critical': 20
        }

        for line in open(conffile, 'r').read().split('\n'):
            words = line.split()
            if words:
                if words[0] == 'serveralias' and len(words) > 1:
                    parts = words[1].split('=')
                    aliases[parts[0]] = parts[1]
                elif words[0] == 'excluderepo':
                    excludes.append(words[1])
                elif words[0] == 'limit' and len(words) > 1:
                    parts = words[1].split('=')
                    limits[parts[0]] = int(parts[1])

        print('processed ' + conffile)
        print('aliases: ' + str(aliases))
        print('excludes: ' + str(excludes))
        print('limits: ' + str(limits))
    except Exception, e:
	print('error reading ' + conffile + ', continuing: ' + str(e))
        conf_mod_time = 0

def dispatch(version, montests, parameters, start_response, environ):
    now = time.time()
    lock = threading.Lock()
    lock.acquire()
    global last_config_time
    if now - config_update_time > last_config_time:
        last_config_time = now
        lock.release()
        parse_api_conf()
    else:
        lock.release()

    if 'server' in parameters:
        serveralias = parameters['server'][0]
    else:
        serveralias = 'local'
    if serveralias in aliases:
        server = aliases[serveralias]
    else:
        return bad_request(start_response, 'unrecognized server alias ' + serveralias)

    socket.setdefaulttimeout(timeout_secs)

    url = 'http://' + server + '/cvmfs/info/v1/repositories.json'
    replicas = []
    repos = []
    try:
        request = urllib2.Request(url,
            headers={"Cache-control" : "max-age=" + str(request_max_secs)})
        json_data = urllib2.urlopen(request).read()
        repos_info = anyjson.deserialize(json_data)
        if 'replicas' in repos_info:
            for repo_info in repos_info['replicas']:
                # the url always has the visible name
                # use "str" to convert from unicode to string
                replicas.append(str(repo_info['url'].replace('/cvmfs/','')))
        if 'repositories' in repos_info:
            for repo_info in repos_info['repositories']:
                repos.append(str(repo_info['url'].replace('/cvmfs/','')))
    except:
        return error_request(start_response, '502 Bad Gateway', url + ' error: ' + str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1]))

    allresults = []
    for repo in replicas + repos:
        if repo in excludes:
            continue
        results = []
        if montests == 'ok':
            results.append([ 'ok', repo, 'OK', '' ])
            continue
        errormsg = ""
        doupdated = False
        if (repo in replicas) and ((montests == "updated") or (montests == "all")):
            doupdated = True
        url = 'http://' + server + '/cvmfs/' + repo + '/.cvmfs_status.json'
        try:
            request = urllib2.Request(url, headers={"Cache-control" : "max-age=30"})
            status_json = urllib2.urlopen(request).read()
            repo_status = anyjson.deserialize(status_json)
        except urllib2.HTTPError, e:
            if e.code == 404:
                if doupdated:
                    # for backward compatibility, look for .cvmfs_last_snapshot
                    #   if .cvmfs_status.json was not found
                    try:
                        url2 = 'http://' + server + '/cvmfs/' + repo + '/.cvmfs_last_snapshot'
                        request = urllib2.Request(url2,
                            headers={"Cache-control" : "max-age=" + str(request_max_secs)})
                        snapshot_string = urllib2.urlopen(request).read()
                        repo_status = {"last_snapshot": snapshot_string}
                    except urllib2.HTTPError, e:
                        if e.code == 404:
                            errormsg = url + ' and .cvmfs_last_snapshot Not found'
                        else:
                            errormsg =  str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1])
                    except:
                        errormsg =  str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1])
                else:
                    errormsg = url + ' Not found'
            else:
                errormsg =  str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1])
        except:
            errormsg =  str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1])

        if doupdated:
            results.append(cvmfsmon_updated.runtest(repo, limits, repo_status, errormsg))
        if (montests == "gc") or (montests == "all"):
            results.append(cvmfsmon_gc.runtest(repo, limits, repo_status, errormsg))
        if results == []:
            return bad_request(start_response, 'unrecognized montests ' + montests)
        allresults.extend(results)

    format = 'list'
    if 'format' in parameters:
        formats = parameters['format']
        l = len(formats)
        if l > 0:
            format = formats[l - 1]

    body = ""
    if format == 'status':
        worststatus = 'OK'
        for result in allresults:
            if len(result) == 0:
                continue
            status = result[2]
            if status == 'CRITICAL':
                worststatus = 'CRITICAL'
            elif (status == 'WARNING') and (worststatus != 'CRITICAL'):
                worststatus = 'WARNING'
        body = worststatus + '\n'
    elif format == 'details':
        details = {}
        for result in allresults:
            if len(result) == 0:
                continue
            test = result[0]
            status = result[2]
            repomsg = {'repo' : result[1], 'msg': result[3]}
            if status in details:
                if test in details[status]:
                    details[status][test].append(repomsg)
                else:
                    details[status][test] = [repomsg]
            else:
                details[status] = {}
                details[status][test] = [repomsg]

        output = StringIO.StringIO()
        pprint.pprint(details, output)
        body = output.getvalue()
        output.close()
        body = string.replace(body,"'", '"')
    else:  # list format
        repostatuses = {}
        for result in allresults:
            if len(result) == 0:
                continue
            repo = result[1]
            status = result[2]
            worststatus = 'OK'
            if repo in repostatuses:
                worststatus = repostatuses[repo]
            if status == 'CRITICAL':
                worststatus = status
            elif (status == 'WARNING') and (worststatus != 'CRITICAL'):
                worststatus = status
            repostatuses[repo] = worststatus

        statusrepos = {}
        for repo in repostatuses:
            status = repostatuses[repo]
            if not status in statusrepos:
                statusrepos[status] = []
            statusrepos[status].append(repo)
        for status in ['CRITICAL', 'WARNING', 'OK']:
            if status in statusrepos:
                statusrepos[status].sort()
                body += status + ':' + ",".join(statusrepos[status]) + '\n'

    return good_request(start_response, body)

<commit_msg>set default empty value for repo_status<commit_after># Implement the cvmfsmon API
# A URL is of the form:
#  /cvmfsmon/api/v1.0/montests&param1=value1&param2=value2
# The URL is parsed by the calling function, and the params come in a
#   dictionary of keys, each with a list of values
# Currently supported "montests" are
#  ok - always returns OK
#  all - runs all applicable tests but 'ok'
#  updated - verifies that updates are happening on a stratum 1
#  gc - verifies that repositories that have done garbage collection
#       have done it successfully recently
# Currently supported parameters are
#  format - value one of the following (default: list)
#    status - reports only one line: OK, WARNING, or CRITICAL
#    list - reports a line for each available status, followed by colon,
#      followed by a comma-separated list of repositories at that status.
#    details - detailed json output with all the tests and messages
#  server - value is either 'local' (the default, indicating localhost) or
#    an alias of a server as defined in /etc/cvmfsmon/api.conf

import os, sys, socket, urllib2, anyjson, pprint, StringIO, string
import time, threading
import cvmfsmon_updated, cvmfsmon_gc

negative_expire_secs = 60*2         # 2 minutes
positive_expire_secs = 60*2         # 2 minutes
timeout_secs = 5                    # tries twice for 5 seconds
request_max_secs = 30               # maximum cache seconds when reading
config_update_time = 60             # seconds between checking config file

conf_mod_time = 0
last_config_time = 0
aliases = {}
excludes = []
limits = {}

def error_request(start_response, response_code, response_body):
    response_body = response_body + '\n'
    start_response(response_code,
                   [('Cache-control', 'max-age=' + str(negative_expire_secs)),
                    ('Content-Length', str(len(response_body)))])
    return [response_body]

def bad_request(start_response, reason):
    response_body = 'Bad Request: ' + reason
    return error_request(start_response, '400 Bad Request', response_body)

def good_request(start_response, response_body):
    response_code = '200 OK'
    start_response(response_code,
                  [('Content-Type', 'text/plain'),
                   ('Cache-Control', 'max-age=' + str(positive_expire_secs)),
                   ('Content-Length', str(len(response_body)))])
    return [response_body]

def parse_api_conf():
    global aliases, excludes, limits
    global conf_mod_time
    conffile = '/etc/cvmfsmon/api.conf'
    try:
        modtime = os.stat(conffile).st_mtime
        if modtime == conf_mod_time:
            # no change
            return
        conf_mod_time = modtime

        aliases = { 'local' : '127.0.0.1' }
        excludes = []
        limits = {
            'updated-warning': 8,
            'updated-critical': 24,
            'gc-warning': 10,
            'gc-critical': 20
        }

        for line in open(conffile, 'r').read().split('\n'):
            words = line.split()
            if words:
                if words[0] == 'serveralias' and len(words) > 1:
                    parts = words[1].split('=')
                    aliases[parts[0]] = parts[1]
                elif words[0] == 'excluderepo':
                    excludes.append(words[1])
                elif words[0] == 'limit' and len(words) > 1:
                    parts = words[1].split('=')
                    limits[parts[0]] = int(parts[1])

        print('processed ' + conffile)
        print('aliases: ' + str(aliases))
        print('excludes: ' + str(excludes))
        print('limits: ' + str(limits))
    except Exception, e:
	print('error reading ' + conffile + ', continuing: ' + str(e))
        conf_mod_time = 0

def dispatch(version, montests, parameters, start_response, environ):
    now = time.time()
    lock = threading.Lock()
    lock.acquire()
    global last_config_time
    if now - config_update_time > last_config_time:
        last_config_time = now
        lock.release()
        parse_api_conf()
    else:
        lock.release()

    if 'server' in parameters:
        serveralias = parameters['server'][0]
    else:
        serveralias = 'local'
    if serveralias in aliases:
        server = aliases[serveralias]
    else:
        return bad_request(start_response, 'unrecognized server alias ' + serveralias)

    socket.setdefaulttimeout(timeout_secs)

    url = 'http://' + server + '/cvmfs/info/v1/repositories.json'
    replicas = []
    repos = []
    try:
        request = urllib2.Request(url,
            headers={"Cache-control" : "max-age=" + str(request_max_secs)})
        json_data = urllib2.urlopen(request).read()
        repos_info = anyjson.deserialize(json_data)
        if 'replicas' in repos_info:
            for repo_info in repos_info['replicas']:
                # the url always has the visible name
                # use "str" to convert from unicode to string
                replicas.append(str(repo_info['url'].replace('/cvmfs/','')))
        if 'repositories' in repos_info:
            for repo_info in repos_info['repositories']:
                repos.append(str(repo_info['url'].replace('/cvmfs/','')))
    except:
        return error_request(start_response, '502 Bad Gateway', url + ' error: ' + str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1]))

    allresults = []
    for repo in replicas + repos:
        if repo in excludes:
            continue
        results = []
        if montests == 'ok':
            results.append([ 'ok', repo, 'OK', '' ])
            continue
        errormsg = ""
        doupdated = False
        if (repo in replicas) and ((montests == "updated") or (montests == "all")):
            doupdated = True
        repo_status = {}
        url = 'http://' + server + '/cvmfs/' + repo + '/.cvmfs_status.json'
        try:
            request = urllib2.Request(url, headers={"Cache-control" : "max-age=30"})
            status_json = urllib2.urlopen(request).read()
            repo_status = anyjson.deserialize(status_json)
        except urllib2.HTTPError, e:
            if e.code == 404:
                if doupdated:
                    # for backward compatibility, look for .cvmfs_last_snapshot
                    #   if .cvmfs_status.json was not found
                    try:
                        url2 = 'http://' + server + '/cvmfs/' + repo + '/.cvmfs_last_snapshot'
                        request = urllib2.Request(url2,
                            headers={"Cache-control" : "max-age=" + str(request_max_secs)})
                        snapshot_string = urllib2.urlopen(request).read()
                        repo_status = {"last_snapshot": snapshot_string}
                    except urllib2.HTTPError, e:
                        if e.code == 404:
                            errormsg = url + ' and .cvmfs_last_snapshot Not found'
                        else:
                            errormsg =  str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1])
                    except:
                        errormsg =  str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1])
                else:
                    errormsg = url + ' Not found'
            else:
                errormsg =  str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1])
        except:
            errormsg =  str(sys.exc_info()[0]) + ' ' + str(sys.exc_info()[1])

        if doupdated:
            results.append(cvmfsmon_updated.runtest(repo, limits, repo_status, errormsg))
        if (montests == "gc") or (montests == "all"):
            results.append(cvmfsmon_gc.runtest(repo, limits, repo_status, errormsg))
        if results == []:
            return bad_request(start_response, 'unrecognized montests ' + montests)
        allresults.extend(results)

    format = 'list'
    if 'format' in parameters:
        formats = parameters['format']
        l = len(formats)
        if l > 0:
            format = formats[l - 1]

    body = ""
    if format == 'status':
        worststatus = 'OK'
        for result in allresults:
            if len(result) == 0:
                continue
            status = result[2]
            if status == 'CRITICAL':
                worststatus = 'CRITICAL'
            elif (status == 'WARNING') and (worststatus != 'CRITICAL'):
                worststatus = 'WARNING'
        body = worststatus + '\n'
    elif format == 'details':
        details = {}
        for result in allresults:
            if len(result) == 0:
                continue
            test = result[0]
            status = result[2]
            repomsg = {'repo' : result[1], 'msg': result[3]}
            if status in details:
                if test in details[status]:
                    details[status][test].append(repomsg)
                else:
                    details[status][test] = [repomsg]
            else:
                details[status] = {}
                details[status][test] = [repomsg]

        output = StringIO.StringIO()
        pprint.pprint(details, output)
        body = output.getvalue()
        output.close()
        body = string.replace(body,"'", '"')
    else:  # list format
        repostatuses = {}
        for result in allresults:
            if len(result) == 0:
                continue
            repo = result[1]
            status = result[2]
            worststatus = 'OK'
            if repo in repostatuses:
                worststatus = repostatuses[repo]
            if status == 'CRITICAL':
                worststatus = status
            elif (status == 'WARNING') and (worststatus != 'CRITICAL'):
                worststatus = status
            repostatuses[repo] = worststatus

        statusrepos = {}
        for repo in repostatuses:
            status = repostatuses[repo]
            if not status in statusrepos:
                statusrepos[status] = []
            statusrepos[status].append(repo)
        for status in ['CRITICAL', 'WARNING', 'OK']:
            if status in statusrepos:
                statusrepos[status].sort()
                body += status + ':' + ",".join(statusrepos[status]) + '\n'

    return good_request(start_response, body)

<|endoftext|>


<commit_before>from django.shortcuts import render

# Create your views here.
from django.core import serializers
from django.http import HttpResponse
from django.http import JsonResponse
from .models import DailyQuote
from .models import Quote
from django.utils import timezone

def index(request):
    return HttpResponse("Hello, world. You're at the quotes index.")

def daily_quote(request):
	latest_daily_quote = DailyQuote.objects.all().order_by('-id')[:1][0]
	if(latest_daily_quote.date_used.date() >= timezone.now().date()):
		todays_quote = latest_daily_quote
	else:
		todays_quote = DailyQuote(quote=Quote.get_random(latest_daily_quote.quote.pk))
		todays_quote.save()
	return JsonResponse(todays_quote.to_dict())<commit_msg>fix bug where local timezone not being used to compare daily quotes<commit_after>from django.shortcuts import render

# Create your views here.
from django.core import serializers
from django.http import HttpResponse
from django.http import JsonResponse
from .models import DailyQuote
from .models import Quote
from django.utils import timezone

def index(request):
    return HttpResponse("Hello, world. You're at the quotes index.")

def daily_quote(request):
	latest_daily_quote = DailyQuote.objects.all().order_by('-id')[:1][0]
	if(latest_daily_quote.date_used.date() >= timezone.localtime(timezone.now()).date()):
		todays_quote = latest_daily_quote
	else:
		todays_quote = DailyQuote(quote=Quote.get_random(latest_daily_quote.quote.pk))
		todays_quote.save()
	return JsonResponse(todays_quote.to_dict())<|endoftext|>


<commit_before>from decimal import Decimal

class Currency( object ):
    def __init__(self, name):
        self.name= name
    def __repr__(self):
        return self.name

class ExchangeRate(object):
    def __init__(self, c1, c2, exchange_rate):
        '''c1 = exchange_rate * c2'''
        assert c1!=c2
        self.c1, self.c2= c1, c2
        self.exchange_rate= Decimal( exchange_rate )

    def convert(self, amount):
        assert isinstance(amount, Amount)
        if self.c1==amount.currency:
            amount.value*= self.exchange_rate
            amount.currency= self.c2
        elif self.c2==amount.currency:
            amount.value/= self.exchange_rate
            amount.currency= self.c1
        else:
            raise Exception("Can't exchange currencies with this ExchangeRate")
    def __repr__(self):
        return "%.3f %s/%s" % (self.exchange_rate, currency_dict[self.c1], currency_dict[self.c2])

class Amount(object):
    def __init__(self, value, currency):
        self.value, self.currency= Decimal(value), currency

    def convert(self, currencyequivalence, to_currency):
        if self.currency!=to_currency:
            currencyequivalence.convert(self)

    def clone():
        '''returns a copy of this amount'''
        return Amount( self.value, self.currency)

    def __repr__(self):
        return str(self.value)+" "+str(self.currency)

    def __iadd__(self, other):
        if type(other) in (int, float):
            self.value+=other
        elif isinstance(other, Amount):
            if self.currency!=other.currency:
                raise ValueError("Can't sum two amounts in different currencies")
            self.value+=other.value
        else:
            raise ValueError("Can't sum Amount to ", type(other))
        return self

    def __add__(self, other):
        a= self.clone()
        a+= other
        return a

    def __neg__(self):
        a= self.clone()
        a.value= -a.value
        return a

    def __isub__(self,other):
        self+= -other
        return self

    def __sub__(self, other):
        a= self.clone() + (-other)
        return a

    def __cmp__(self, other):
        if not isinstance(other, Amount) or other.currency!=self.currency:
            raise ValueError("can't compare the two amounts", str(self), str(other))
        return cmp(self.value,other.value)
<commit_msg>currency: check for bad Decimal conversion<commit_after>from decimal import Decimal
from mexbtcapi import log

def check_number_for_decimal_conversion(number):
    a= type(number) in (int, long)
    b= isinstance(number, (str, Decimal))
    if not (a or b):
        log.warning("You are using a number ("+str(number)+") that is not suitable to convert to Decimal!")

class Currency( object ):
    def __init__(self, name):
        self.name= name
    def __repr__(self):
        return self.name

class ExchangeRate(object):
    def __init__(self, c1, c2, exchange_rate):
        '''c2 = exchange_rate * c1'''
        assert c1!=c2
        check_number_for_decimal_conversion( exchange_rate )
        self.c1, self.c2= c1, c2
        self.exchange_rate= Decimal( exchange_rate )

    def convert(self, amount):
        assert isinstance(amount, Amount)
        if self.c1==amount.currency:
            return Amount(amount.value*self.exchange_rate, self.c2)
        elif self.c2==amount.currency:
            return Amount(amount.value/self.exchange_rate, self.c2)
        else:
            raise Exception("Can't exchange currencies with this ExchangeRate")
    def __repr__(self):
        return "%.3f %s/%s" % (self.exchange_rate, currency_dict[self.c1], currency_dict[self.c2])

class Amount(object):
    def __init__(self, value, currency):
        check_number_for_decimal_conversion( value )
        self.value, self.currency= Decimal(value), currency

    def convert(self, currencyequivalence, to_currency):
        if self.currency!=to_currency:
            currencyequivalence.convert(self)

    def clone():
        '''returns a copy of this amount'''
        return Amount( self.value, self.currency)

    def __repr__(self):
        return str(self.value)+" "+str(self.currency)

    def __iadd__(self, other):
        if type(other) in (int, float):
            self.value+=other
        elif isinstance(other, Amount):
            if self.currency!=other.currency:
                raise ValueError("Can't sum two amounts in different currencies")
            self.value+=other.value
        else:
            raise ValueError("Can't sum Amount to ", type(other))
        return self

    def __add__(self, other):
        a= self.clone()
        a+= other
        return a

    def __neg__(self):
        a= self.clone()
        a.value= -a.value
        return a

    def __isub__(self,other):
        self+= -other
        return self

    def __sub__(self, other):
        a= self.clone() + (-other)
        return a

    def __cmp__(self, other):
        if not isinstance(other, Amount) or other.currency!=self.currency:
            raise ValueError("can't compare the two amounts", str(self), str(other))
        return cmp(self.value,other.value)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals
import ir.main
import ir.view

__version__ = '3.2.9'
<commit_msg>Bump version to v3.3.0<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals
import ir.main
import ir.view

__version__ = '3.3.0'
<|endoftext|>


<commit_before>class AdminPlugin(object):
    # A dictionary which will contain the owner nicks and vhosts
    owners = None

    # A list of trusted vhosts
    trusted_vhosts = None

    def __init__(self, cardinal, config):
        self.owners = {}
        self.trusted_vhosts = []

        # If owners aren't defined, bail out
        if 'owners' not in config:
            return

        # Loop through the owners in the config file and add them to the
        # instance's owner array.
        for owner in config['owners']:
            owner = owner.split('@')
            self.owners[owner[0]] = owner[1]
            self.trusted_vhosts.append(owner[1])

    # A command to quickly check whether a user has permissions to access
    # these commands.
    def is_owner(self, user):
        if user.group(3) in self.trusted_vhosts:
            return True

        return False

    def eval(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            command = ' '.join(msg.split()[1:])
            if len(command) > 0:
                try:
                    output = str(eval(command))
                    cardinal.sendMsg(channel, output)
                except Exception, e:
                    cardinal.sendMsg(channel, 'Exception %s: %s' %
                                              (e.__class__, e))
                    raise

    eval.commands = ['eval']
    eval.help = ["A super dangerous command that runs eval() on the input. " +
                 "(admin only)",

                 "Syntax: .eval <command>"]

    def execute(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            command = ' '.join(msg.split()[1:])
            if len(command) > 0:
                try:
                    exec(command)
                    cardinal.sendMsg(channel, "Ran exec() on input.")
                except Exception, e:
                    cardinal.sendMsg(channel, 'Exception %s: %s' %
                                              (e.__class__, e))
                    raise

    execute.commands = ['exec']
    execute.help = ["A dangerous command that runs exec() on the input. " +
                    "(admin only)",

                    "Syntax: .exec <command>"]

    def load_plugins(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            cardinal.sendMsg(channel, "%s: Loading plugins..." % user.group(1))

            plugins = msg.split()
            plugins.pop(0)

            if len(plugins) == 0:
                plugins = []
                for plugin in cardinal.plugin_manager:
                    plugins.append(plugin['name'])

            failed = cardinal.plugin_manager.load(plugins)

            successful = [
                plugin for plugin in plugins if plugin not in failed
            ]

            if len(successful) > 0:
                cardinal.sendMsg(channel, "Plugins loaded succesfully: %s." %
                                          ', '.join(sorted(successful)))

            if len(failed) > 0:
                cardinal.sendMsg(channel, "Plugins failed to load: %s." %
                                          ', '.join(sorted(failed)))

    load_plugins.commands = ['load', 'reload']
    load_plugins.help = ["If no plugins are given after the command, reload " +
                         "all plugins. Otherwise, load (or reload) the " +
                         "selected plugins. (admin only)",

                         "Syntax: .reload [plugin [plugin ...]]"]

    def unload_plugins(self, cardinal, user, channel, msg):
        nick = user.group(1)

        if self.is_owner(user):
            plugins = msg.split()
            plugins.pop(0)

            if len(plugins) == 0:
                cardinal.sendMsg(channel, "%s: No plugins to unload." % nick)
                return

            cardinal.sendMsg(channel, "%s: Unloading plugins..." % nick)

            # Returns a list of plugins that weren't loaded to begin with
            unknown = cardinal.plugin_manager.unload(plugins)
            successful = [
                plugin for plugin in plugins if plugin not in unknown
            ]

            if len(successful) > 0:
                cardinal.sendMsg(channel, "Plugins unloaded succesfully: %s." %
                                          ', '.join(sorted(successful)))

            if len(unknown) > 0:
                cardinal.sendMsg(channel, "Unknown plugins: %s." %
                                          ', '.join(sorted(unknown)))

    unload_plugins.commands = ['unload']
    unload_plugins.help = ["Unload selected plugins. (admin only)",
                           "Syntax: .unload <plugin [plugin ...]>"]

    def disable_plugins(self, cardinal, user, channel, msg):
        if not self.is_owner(user):
            return

        channels = msg.split()
        channels.pop(0)

        if len(channels) < 2:
            cardinal.sendMsg(channel,
                             "Syntax: .disable <plugin> <channel [channel ...]>")
            return

        cardinal.sendMsg(channel, "%s: Disabling plugins..." % user.group(1))

        # First argument is plugin
        plugin = channels.pop(0)

        blacklisted = cardinal.plugin_manager.blacklist(plugin, channels)
        if not blacklisted:
            cardinal.sendMsg(channel, "Plugin %s does not exist" % plugin)
            return

        cardinal.sendMsg(channel, "Added to blacklist: %s." %
                                  ', '.join(sorted(channels)))

    disable_plugins.commands = ['disable']
    disable_plugins.help = ["Disable plugins in a channel. (admin only)",
                            "Syntax: .disable <plugin> <channel [channel ...]>"]

    def enable_plugins(self, cardinal, user, channel, msg):
        if not self.is_owner(user):
            return

        channels = msg.split()
        channels.pop(0)

        if len(channels) < 2:
            cardinal.sendMsg(channel,
                             "Syntax: .enable <plugin> <channel [channel ...]>")
            return

        cardinal.sendMsg(channel, "%s: Enabling plugins..." % user.group(1))

        # First argument is plugin
        plugin = channels.pop(0)

        not_blacklisted = cardinal.plugin_manager.unblacklist(plugin, channels)
        if not_blacklisted is False:
            cardinal.sendMsg("Plugin %s does not exist" % plugin)

        successful = [
            channel for channel in channels if channel not in not_blacklisted
        ]

        if len(successful) > 0:
            cardinal.sendMsg(channel, "Removed from blacklist: %s." %
                                      ', '.join(sorted(successful)))

        if len(not_blacklisted) > 0:
            cardinal.sendMsg(channel, "Wasn't in blacklist: %s." %
                                      ', '.join(sorted(not_blacklisted)))

    enable_plugins.commands = ['enable']
    enable_plugins.help = ["Enable plugins in a channel. (admin only)",
                           "Syntax: .enable <plugin> <channel [channel ...]>"]

    def join(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            channels = msg.split()
            channels.pop(0)
            for channel in channels:
                cardinal.join(channel)

    join.commands = ['join']
    join.help = ["Joins selected channels. (admin only)",
                 "Syntax: .join <channel [channel ...]>"]

    def part(self, cardinal, user, channel, msg):
        if not self.is_owner(user):
            return

        channels = msg.split()
        channels.pop(0)
        if len(channels) > 0:
            for channel in channels:
                cardinal.part(channel)
        elif channel != user:
            cardinal.part(channel)

    part.commands = ['part']
    part.help = ["Parts selected channels. (admin only)",
                 "Syntax: .join <channel [channel ...]>"]

    def quit(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            cardinal.disconnect(' '.join(msg.split(' ')[1:]))

    quit.commands = ['quit']
    quit.help = ["Quits the network with a quit message, if one is defined. " +
                 "(admin only)",

                 "Syntax: .quit [message]"]

    def debug_quit(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            cardinal.quit('Debug disconnect')
    debug_quit.commands = ['dbg_quit']
    debug_quit.help = ["Quits the network without setting disconnect flag " +
                       "(for testing reconnection, admin only)",

                       "Syntax: .dbg_quit"]

def setup(cardinal, config):
    """Returns an instance of the plugin.

    Keyword arguments:
      cardinal -- An instance of Cardinal. Passed in by PluginManager.
      config -- A config for this plugin.
    """
    return AdminPlugin(cardinal, config)
<commit_msg>Update admin plugin to use decorators<commit_after>from cardinal.decorators import command, help

class AdminPlugin(object):
    def __init__(self, cardinal, config):
        self.owners = {}
        self.trusted_vhosts = []

        # If owners aren't defined, bail out
        if 'owners' not in config:
            return

        # Loop through the owners in the config file and add them to the
        # instance's owner array.
        for owner in config['owners']:
            owner = owner.split('@')
            self.owners[owner[0]] = owner[1]
            self.trusted_vhosts.append(owner[1])

    # A command to quickly check whether a user has permissions to access
    # these commands.
    def is_owner(self, user):
        if user.group(3) in self.trusted_vhosts:
            return True

        return False

    @command('eval')
    @help("A super dangerous command that runs eval() on the input. "
          "(admin only)")
    @help("Syntax: .eval <command>")
    def eval(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            command = ' '.join(msg.split()[1:])
            if len(command) > 0:
                try:
                    output = str(eval(command))
                    cardinal.sendMsg(channel, output)
                except Exception, e:
                    cardinal.sendMsg(channel, 'Exception %s: %s' %
                                              (e.__class__, e))
                    raise

    @command('exec')
    @help("A dangerous command that runs exec() on the input. " +
          "(admin only)")
    @help("Syntax: .exec <command>")
    def execute(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            command = ' '.join(msg.split()[1:])
            if len(command) > 0:
                try:
                    exec(command)
                    cardinal.sendMsg(channel, "Ran exec() on input.")
                except Exception, e:
                    cardinal.sendMsg(channel, 'Exception %s: %s' %
                                              (e.__class__, e))
                    raise

    @command(['load', 'reload'])
    @help("If no plugins are given after the command, reload all plugins. "
          "Otherwise, load (or reload) the selected plugins. (admin only)")
    @help("Syntax: .load [plugin [plugin ...]]")
    def load_plugins(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            cardinal.sendMsg(channel, "%s: Loading plugins..." % user.group(1))

            plugins = msg.split()
            plugins.pop(0)

            if len(plugins) == 0:
                plugins = []
                for plugin in cardinal.plugin_manager:
                    plugins.append(plugin['name'])

            failed = cardinal.plugin_manager.load(plugins)

            successful = [
                plugin for plugin in plugins if plugin not in failed
            ]

            if len(successful) > 0:
                cardinal.sendMsg(channel, "Plugins loaded succesfully: %s." %
                                          ', '.join(sorted(successful)))

            if len(failed) > 0:
                cardinal.sendMsg(channel, "Plugins failed to load: %s." %
                                          ', '.join(sorted(failed)))

    @command('unload')
    @help("Unload selected plugins. (admin only)")
    @help("Syntax: .unload <plugin [plugin ...]>")
    def unload_plugins(self, cardinal, user, channel, msg):
        nick = user.group(1)

        if self.is_owner(user):
            plugins = msg.split()
            plugins.pop(0)

            if len(plugins) == 0:
                cardinal.sendMsg(channel, "%s: No plugins to unload." % nick)
                return

            cardinal.sendMsg(channel, "%s: Unloading plugins..." % nick)

            # Returns a list of plugins that weren't loaded to begin with
            unknown = cardinal.plugin_manager.unload(plugins)
            successful = [
                plugin for plugin in plugins if plugin not in unknown
            ]

            if len(successful) > 0:
                cardinal.sendMsg(channel, "Plugins unloaded succesfully: %s." %
                                          ', '.join(sorted(successful)))

            if len(unknown) > 0:
                cardinal.sendMsg(channel, "Unknown plugins: %s." %
                                          ', '.join(sorted(unknown)))

    @command('disable')
    @help("Disable plugins in a channel. (admin only)")
    @help("Syntax: .disable <plugin> <channel [channel ...]>")
    def disable_plugins(self, cardinal, user, channel, msg):
        if not self.is_owner(user):
            return

        channels = msg.split()
        channels.pop(0)

        if len(channels) < 2:
            cardinal.sendMsg(channel,
                             "Syntax: .disable <plugin> <channel [channel ...]>")
            return

        cardinal.sendMsg(channel, "%s: Disabling plugins..." % user.group(1))

        # First argument is plugin
        plugin = channels.pop(0)

        blacklisted = cardinal.plugin_manager.blacklist(plugin, channels)
        if not blacklisted:
            cardinal.sendMsg(channel, "Plugin %s does not exist" % plugin)
            return

        cardinal.sendMsg(channel, "Added to blacklist: %s." %
                                  ', '.join(sorted(channels)))

    @command('enable')
    @help("Enable plugins in a channel. (admin only)")
    @help("Syntax: .enable <plugin> <channel [channel ...]>")
    def enable_plugins(self, cardinal, user, channel, msg):
        if not self.is_owner(user):
            return

        channels = msg.split()
        channels.pop(0)

        if len(channels) < 2:
            cardinal.sendMsg(channel,
                             "Syntax: .enable <plugin> <channel [channel ...]>")
            return

        cardinal.sendMsg(channel, "%s: Enabling plugins..." % user.group(1))

        # First argument is plugin
        plugin = channels.pop(0)

        not_blacklisted = cardinal.plugin_manager.unblacklist(plugin, channels)
        if not_blacklisted is False:
            cardinal.sendMsg("Plugin %s does not exist" % plugin)

        successful = [
            channel for channel in channels if channel not in not_blacklisted
        ]

        if len(successful) > 0:
            cardinal.sendMsg(channel, "Removed from blacklist: %s." %
                                      ', '.join(sorted(successful)))

        if len(not_blacklisted) > 0:
            cardinal.sendMsg(channel, "Wasn't in blacklist: %s." %
                                      ', '.join(sorted(not_blacklisted)))

    @command('join')
    @help("Joins selected channels. (admin only)")
    @help("Syntax: .join <channel [channel ...]>")
    def join(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            channels = msg.split()
            channels.pop(0)
            for channel in channels:
                cardinal.join(channel)

    @command('part')
    @help("Parts selected channels. (admin only)")
    @help("Syntax: .join <channel [channel ...]>")
    def part(self, cardinal, user, channel, msg):
        if not self.is_owner(user):
            return

        channels = msg.split()
        channels.pop(0)
        if len(channels) > 0:
            for channel in channels:
                cardinal.part(channel)
        elif channel != user:
            cardinal.part(channel)

    @command('quit')
    @help("Quits the network with a quit message, if one is defined. "
          "(admin only)")
    @help("Syntax: .quit [message]")
    def quit(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            cardinal.disconnect(' '.join(msg.split(' ')[1:]))

    @command('dbg_quit')
    @help("Quits the network without setting disconnect flag "
         "(for testing reconnection, admin only)")
    @help("Syntax: .dbg_quit")
    def debug_quit(self, cardinal, user, channel, msg):
        if self.is_owner(user):
            cardinal.quit('Debug disconnect')

def setup(cardinal, config):
    """Returns an instance of the plugin.

    Keyword arguments:
      cardinal -- An instance of Cardinal. Passed in by PluginManager.
      config -- A config for this plugin.
    """
    return AdminPlugin(cardinal, config)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2013      Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.tools.ordereddict import OrderedDict

from .pages import LoginPage, AccountsPage, HistoryPage

__all__ = ['SGProfessionalBrowser', 'SGEnterpriseBrowser']

class SGPEBrowser(BaseBrowser):
    PROTOCOL = 'https'
    ENCODING = 'ISO-8859-1'

    def __init__(self, *args, **kwargs):
        self.PAGES = OrderedDict((
            ('%s://%s/Pgn/.+PageID=SoldeV3&.+' % (self.PROTOCOL, self.DOMAIN), AccountsPage),
            ('%s://%s/Pgn/.+PageID=ReleveCompteV3&.+' % (self.PROTOCOL, self.DOMAIN), HistoryPage),
            ('%s://%s/' % (self.PROTOCOL, self.DOMAIN), LoginPage),
        ))
        BaseBrowser.__init__(self, *args, **kwargs)

    def is_logged(self):
        if not self.page or self.is_on_page(LoginPage):
            return False

        error = self.page.get_error()
        if error is None:
            return True

        return False

    def login(self):
        assert isinstance(self.username, basestring)
        assert isinstance(self.password, basestring)
        assert self.password.isdigit()

        if not self.is_on_page(LoginPage):
            self.location('https://' + self.DOMAIN + '/', no_login=True)

            self.page.login(self.username, self.password)

        # force page change
        if not self.is_on_page(AccountsPage):
            self.accounts()
        if self.is_on_page(LoginPage):
            raise BrowserIncorrectPassword()

    def accounts(self):
        self.location('/Pgn/NavigationServlet?PageID=SoldeV3&MenuID=%s&Classeur=1&NumeroPage=1' % self.MENUID)

    def history(self, _id, page=1):
        if page > 1:
            pgadd = '&page_numero_page_courante=%s' % page
        else:
            pgadd = ''
        self.location('/Pgn/NavigationServlet?PageID=ReleveCompteV3&MenuID=%s&Classeur=1&Rib=%s&NumeroPage=1%s' % (self.MENUID, _id, pgadd))

    def get_accounts_list(self):
        if not self.is_on_page(AccountsPage):
            self.accounts()
        assert self.is_on_page(AccountsPage)
        return self.page.get_list()

    def get_account(self, _id):
        for a in self.get_accounts_list():
            if a.id == _id:
                yield a

    def iter_history(self, account):
        page = 1
        while page:
            self.history(account.id, page)
            assert self.is_on_page(HistoryPage)
            for transaction in self.page.iter_transactions(account):
                yield transaction
            if self.page.has_next():
                page += 1
            else:
                page = False

class SGProfessionalBrowser(SGPEBrowser):
    DOMAIN = 'professionnels.secure.societegenerale.fr'
    LOGIN_FORM = 'auth_reco'
    MENUID = 'SBORELCPT'

class SGEnterpriseBrowser(SGPEBrowser):
    DOMAIN = 'entreprises.secure.societegenerale.fr'
    LOGIN_FORM = 'auth'
    MENUID = 'BANRELCPT'
<commit_msg>sgpe: Check certificates<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2013      Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.tools.ordereddict import OrderedDict

from .pages import LoginPage, AccountsPage, HistoryPage

__all__ = ['SGProfessionalBrowser', 'SGEnterpriseBrowser']

class SGPEBrowser(BaseBrowser):
    PROTOCOL = 'https'
    ENCODING = 'ISO-8859-1'

    def __init__(self, *args, **kwargs):
        self.PAGES = OrderedDict((
            ('%s://%s/Pgn/.+PageID=SoldeV3&.+' % (self.PROTOCOL, self.DOMAIN), AccountsPage),
            ('%s://%s/Pgn/.+PageID=ReleveCompteV3&.+' % (self.PROTOCOL, self.DOMAIN), HistoryPage),
            ('%s://%s/' % (self.PROTOCOL, self.DOMAIN), LoginPage),
        ))
        BaseBrowser.__init__(self, *args, **kwargs)

    def is_logged(self):
        if not self.page or self.is_on_page(LoginPage):
            return False

        error = self.page.get_error()
        if error is None:
            return True

        return False

    def login(self):
        assert isinstance(self.username, basestring)
        assert isinstance(self.password, basestring)
        assert self.password.isdigit()

        if not self.is_on_page(LoginPage):
            self.location('https://' + self.DOMAIN + '/', no_login=True)

            self.page.login(self.username, self.password)

        # force page change
        if not self.is_on_page(AccountsPage):
            self.accounts()
        if self.is_on_page(LoginPage):
            raise BrowserIncorrectPassword()

    def accounts(self):
        self.location('/Pgn/NavigationServlet?PageID=SoldeV3&MenuID=%s&Classeur=1&NumeroPage=1' % self.MENUID)

    def history(self, _id, page=1):
        if page > 1:
            pgadd = '&page_numero_page_courante=%s' % page
        else:
            pgadd = ''
        self.location('/Pgn/NavigationServlet?PageID=ReleveCompteV3&MenuID=%s&Classeur=1&Rib=%s&NumeroPage=1%s' % (self.MENUID, _id, pgadd))

    def get_accounts_list(self):
        if not self.is_on_page(AccountsPage):
            self.accounts()
        assert self.is_on_page(AccountsPage)
        return self.page.get_list()

    def get_account(self, _id):
        for a in self.get_accounts_list():
            if a.id == _id:
                yield a

    def iter_history(self, account):
        page = 1
        while page:
            self.history(account.id, page)
            assert self.is_on_page(HistoryPage)
            for transaction in self.page.iter_transactions(account):
                yield transaction
            if self.page.has_next():
                page += 1
            else:
                page = False

class SGProfessionalBrowser(SGPEBrowser):
    DOMAIN = 'professionnels.secure.societegenerale.fr'
    LOGIN_FORM = 'auth_reco'
    MENUID = 'SBORELCPT'
    CERTHASH = 'd369315d357ba0018c2bd4d3394645669d99106c797d7390fded516b098a933e'

class SGEnterpriseBrowser(SGPEBrowser):
    DOMAIN = 'entreprises.secure.societegenerale.fr'
    LOGIN_FORM = 'auth'
    MENUID = 'BANRELCPT'
    CERTHASH = 'd5c21d47c7d5a300b40856be49d0b36b42eaae409c8891184652b888d16a05f5'
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
@package mi.core.instrument.playback
@file mi/core/instrument/playback.py
@author Ronald Ronquillo
@brief Playback process using ZMQ messaging.

Usage:
    playback datalog <module> <refdes> <event_url> <particle_url> [--allowed=<particles>]  [--max_events=<events>] <files>...
    playback ascii <module> <refdes> <event_url> <particle_url> [--allowed=<particles>] [--max_events=<events>] <files>...
    playback chunky <module> <refdes> <event_url> <particle_url> [--allowed=<particles>] [--max_events=<events>] <files>...
    playback zplsc <module> <refdes> <event_url> <particle_url> [--allowed=<particles>] [--max_events=<events>] <files>...

Options:
    -h, --help          Show this screen
    --allowed=<particles> Comma-separated list of publishable particles

    To run without installing:
    python -m mi.core.instrument.playback ...
"""
import glob
import importlib
import sys
import time
from datetime import datetime

import os
import re
from docopt import docopt
from mi.core.instrument.instrument_driver import DriverAsyncEvent
from mi.core.instrument.instrument_protocol import \
    MenuInstrumentProtocol,\
    CommandResponseInstrumentProtocol, \
    InstrumentProtocol
from mi.core.instrument.publisher import Publisher
from mi.logging import log
from ooi_port_agent.common import PacketType
from ooi_port_agent.packet import Packet, PacketHeader
from wrapper import EventKeys, encode_exception, DriverWrapper

__author__ = 'Ronald Ronquillo'
__license__ = 'Apache 2.0'

NTP_DIFF = (datetime(1970, 1, 1) - datetime(1900, 1, 1)).total_seconds()
Y2K = (datetime(2000, 1, 1) - datetime(1900, 1, 1)).total_seconds()
DATE_PATTERN = r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z?$'
DATE_MATCHER = re.compile(DATE_PATTERN)
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"

def string_to_ntp_date_time(datestr):
    """
    Extract an ntp date from a ISO8601 formatted date string.
    @param datestr an ISO8601 formatted string containing date information
    @retval an ntp date number (seconds since jan 1 1900)
    @throws InstrumentParameterException if datestr cannot be formatted to
    a date.
    """
    if not isinstance(datestr, basestring):
        raise IOError('Value %s is not a string.' % str(datestr))

    if not DATE_MATCHER.match(datestr):
        raise ValueError("date string not in ISO8601 format YYYY-MM-DDTHH:MM:SS.SSSSZ")

    try:
        # This assumes input date string are in UTC (=GMT)

        # if there is no decimal place, add one to match the date format
        if datestr.find('.') == -1:
            if datestr[-1] != 'Z':
                datestr += '.0Z'
            else:
                datestr = datestr[:-1] + '.0Z'

        # if there is no trailing 'Z' on the input string add one
        if datestr[-1:] != 'Z':
            datestr += 'Z'

        dt = datetime.strptime(datestr, DATE_FORMAT)
        timestamp = (dt - datetime(1900, 1, 1)).total_seconds()

    except ValueError as e:
        raise ValueError('Value %s could not be formatted to a date. %s' % (str(datestr), e))

    return timestamp

class PlaybackPacket(Packet):
    def get_data_length(self):
        return len(self.payload)

    def get_data(self):
        return self.payload

    def get_timestamp(self):
        return self.header.time

    def __repr__(self):
        return repr(self.payload)

    @staticmethod
    def packet_from_fh(file_handle):
        data_buffer = bytearray()
        while True:
            byte = file_handle.read(1)
            if byte == '':
                return None

            data_buffer.append(byte)
            sync_index = data_buffer.find(PacketHeader.sync)
            if sync_index != -1:
                # found the sync bytes, read the rest of the header
                data_buffer.extend(file_handle.read(PacketHeader.header_size - len(PacketHeader.sync)))

                if len(data_buffer) < PacketHeader.header_size:
                    return None

                header = PacketHeader.from_buffer(data_buffer, sync_index)
                # read the payload
                payload = file_handle.read(header.payload_size)
                if len(payload) == header.payload_size:
                    packet = PlaybackPacket(payload=payload, header=header)
                    return packet

class PlaybackWrapper(object):
    def __init__(self, module, refdes, event_url, particle_url, reader_klass, allowed, files, max_events, handler=None):
        version = DriverWrapper.get_version(module)
        headers = {'sensor': refdes, 'deliveryType': 'streamed', 'version': version, 'module': module}
        self.max_events = max_events
        self.event_publisher = Publisher.from_url(event_url, handler, headers)
        self.particle_publisher = Publisher.from_url(particle_url, handler, headers, allowed, max_events)
        self.protocol = self.construct_protocol(module)
        self.reader = reader_klass(files, self.got_data)

    def set_header_filename(self, filename):
        self.event_publisher.set_source(filename)
        self.particle_publisher.set_source(filename)

    def playback(self):
        for index, filename in enumerate(self.reader.read()):
            if filename is not None:
                self.set_header_filename(filename)
                if hasattr(self.protocol, 'got_filename'):
                    self.protocol.got_filename(filename)
            if index % 1000 == 0:
                self.publish()

        self.publish()
        if hasattr(self.particle_publisher, 'write'):
            self.particle_publisher.write()

    def zplsc_playback(self):
        for index, filename in enumerate(self.reader.read()):
            if filename:
                self.set_header_filename(filename)
                log.info("filename is: %s", filename)
                if hasattr(self.protocol, 'got_filename'):
                    self.protocol.got_filename(filename)

        pub_index = 0
        while True:
            self.publish()
            pub_index = pub_index + 1
            log.info("publish index is: %d", pub_index)

    def got_data(self, packet):
        try:
            self.protocol.got_data(packet)
        except KeyboardInterrupt:
            raise
        except Exception as e:
            log.exception(e)

    @staticmethod
    def find_base_class(base):
        targets = (MenuInstrumentProtocol, CommandResponseInstrumentProtocol, InstrumentProtocol, object)
        while True:
            if base in targets:
                return base
            base = base.__base__

    def construct_protocol(self, proto_module):
        module = importlib.import_module(proto_module)
        if hasattr(module, 'create_playback_protocol'):
            return module.create_playback_protocol(self.handle_event)

        log.error('Unable to import and create playback protocol from module: %r', module)
        sys.exit(1)

    def publish(self):
        for publisher in [self.event_publisher, self.particle_publisher]:
            remaining = publisher.publish()
            while remaining >= publisher._max_events:
                remaining = publisher.publish()

    def handle_event(self, event_type, val=None):
        """
        Construct and send an asynchronous driver event.
        @param event_type a DriverAsyncEvent type specifier.
        @param val event value for sample and test result events.
        """
        event = {
            'type': event_type,
            'value': val,
            'time': time.time()
        }

        if isinstance(event[EventKeys.VALUE], Exception):
            event[EventKeys.VALUE] = encode_exception(event[EventKeys.VALUE])

        if event[EventKeys.TYPE] == DriverAsyncEvent.ERROR:
            log.error(event)

        if event[EventKeys.TYPE] == DriverAsyncEvent.SAMPLE:
            if event[EventKeys.VALUE].get('stream_name') != 'raw':
                # don't publish raw
                self.particle_publisher.enqueue(event)
        else:
            self.event_publisher.enqueue(event)

class DatalogReader(object):
    def __init__(self, files, callback):
        self.callback = callback

        self.files = []
        for each in files:
            self.files.extend(glob.glob(each))

        self.files.sort()
        if not all([os.path.isfile(f) for f in self.files]):
            raise Exception('Not all files found')
        self._filehandle = None
        self.target_types = [PacketType.FROM_INSTRUMENT, PacketType.PA_CONFIG]
        self.file_name_list = []

    def read(self):
        while True:
            if self._filehandle is None and not self.files:
                log.info('Completed reading specified port agent logs, exiting...')
                raise StopIteration

            if self._filehandle is None:
                name = self.files.pop(0)
                log.info('Begin reading: %r', name)
                # yield the filename so we can pass it through to the driver
                yield name
                self.file_name_list.append(name)
                self._filehandle = open(name, 'r')

            if not self._process_packet():
                self._filehandle.close()
                self._filehandle = None

            yield

    def _process_packet(self):
        packet = PlaybackPacket.packet_from_fh(self._filehandle)
        if packet is None:
            return False
        if packet.header.packet_type in self.target_types:
            self.callback(packet)
        return True

class DigiDatalogAsciiReader(DatalogReader):
    def __init__(self, files, callback):
        self.ooi_ts_regex = re.compile(r'<OOI-TS (.+?) [TX][NS]>\r\n(.*?)<\\OOI-TS>', re.DOTALL)
        self.buffer = ''
        self.MAXBUF = 65535

        super(DigiDatalogAsciiReader, self).__init__(files, callback)

        if all((self.search_utc(f) for f in self.files)):
            self.files.sort(key=self.search_utc)

    # special case for RSN archived data
    # if all files have date_UTC in filename then sort by that
    @staticmethod
    def search_utc(f):
        match = re.search('(\d+T\d+_UTC)', f)
        if match is None:
            return None
        return match.group(1)

    def _process_packet(self):
        chunk = self._filehandle.read(1024)
        if chunk != '':
            self.buffer += chunk
            new_index = 0
            for match in self.ooi_ts_regex.finditer(self.buffer):
                payload = match.group(2)
                try:
                    packet_time = string_to_ntp_date_time(match.group(1))
                    header = PacketHeader(packet_type=PacketType.FROM_INSTRUMENT,
                                          payload_size=len(payload), packet_time=packet_time)
                    header.set_checksum(payload)
                    packet = PlaybackPacket(payload=payload, header=header)
                    self.callback(packet)
                except ValueError:
                    log.error('Unable to extract timestamp from record: %r' % match.group())
                new_index = match.end()

            if new_index > 0:
                self.buffer = self.buffer[new_index:]

            if len(self.buffer) > self.MAXBUF:
                self.buffer = self.buffer[-self.MAXBUF:]

            return True

        return False

class ChunkyDatalogReader(DatalogReader):
    def _process_packet(self):
        data = self._filehandle.read(1024)
        if data != '':
            header = PacketHeader(packet_type=PacketType.FROM_INSTRUMENT,
                                  payload_size=len(data), packet_time=0)
            header.set_checksum(data)
            packet = PlaybackPacket(payload=data, header=header)
            self.callback(packet)
            return True
        return False

class ZplscReader(DatalogReader):
    def __init__(self, files, callback):
        super(ZplscReader, self).__init__(files, callback)

    def _process_packet(self):

        for name in self.file_name_list:
            data = 'downloaded file:' + name + '\n'
            header = PacketHeader(packet_type=PacketType.FROM_INSTRUMENT,
                                  payload_size=len(data), packet_time=0)
            header.set_checksum(data)
            packet = PlaybackPacket(payload=data, header=header)
            self.callback(packet)

        return False

def main():
    options = docopt(__doc__)

    module = options['<module>']
    refdes = options['<refdes>']
    event_url = options['<event_url>']
    particle_url = options['<particle_url>']
    files = options.get('<files>')
    allowed = options.get('--allowed')
    if allowed is not None:
        allowed = [_.strip() for _ in allowed.split(',')]
    max_events = options.get('--max_events')
    if not max_events:
        max_events = Publisher.DEFAULT_MAX_EVENTS
    else:
        max_events = int(max_events)

    # when running with the profiler, files will be a string
    # coerce to list
    if isinstance(files, basestring):
        files = [files]

    if options['datalog']:
        reader = DatalogReader
    elif options['ascii']:
        reader = DigiDatalogAsciiReader
    elif options['chunky']:
        reader = ChunkyDatalogReader
    elif options['zplsc']:
        reader = ZplscReader
        zplsc_reader = True
    else:
        reader = None

    wrapper = PlaybackWrapper(module, refdes, event_url, particle_url, reader, allowed, files, max_events)
    try:
        wrapper.zplsc_playback()
    except:
        wrapper.playback()

if __name__ == '__main__':
    main()
<commit_msg>update as per Rene's comment<commit_after>#!/usr/bin/env python
"""
@package mi.core.instrument.playback
@file mi/core/instrument/playback.py
@author Ronald Ronquillo
@brief Playback process using ZMQ messaging.

Usage:
    playback datalog <module> <refdes> <event_url> <particle_url> [--allowed=<particles>]  [--max_events=<events>] <files>...
    playback ascii <module> <refdes> <event_url> <particle_url> [--allowed=<particles>] [--max_events=<events>] <files>...
    playback chunky <module> <refdes> <event_url> <particle_url> [--allowed=<particles>] [--max_events=<events>] <files>...
    playback zplsc <module> <refdes> <event_url> <particle_url> [--allowed=<particles>] [--max_events=<events>] <files>...

Options:
    -h, --help          Show this screen
    --allowed=<particles> Comma-separated list of publishable particles

    To run without installing:
    python -m mi.core.instrument.playback ...
"""
import glob
import importlib
import sys
import time
from datetime import datetime

import os
import re
from docopt import docopt
from mi.core.instrument.instrument_driver import DriverAsyncEvent
from mi.core.instrument.instrument_protocol import \
    MenuInstrumentProtocol,\
    CommandResponseInstrumentProtocol, \
    InstrumentProtocol
from mi.core.instrument.publisher import Publisher
from mi.logging import log
from ooi_port_agent.common import PacketType
from ooi_port_agent.packet import Packet, PacketHeader
from wrapper import EventKeys, encode_exception, DriverWrapper

__author__ = 'Ronald Ronquillo'
__license__ = 'Apache 2.0'

NTP_DIFF = (datetime(1970, 1, 1) - datetime(1900, 1, 1)).total_seconds()
Y2K = (datetime(2000, 1, 1) - datetime(1900, 1, 1)).total_seconds()
DATE_PATTERN = r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z?$'
DATE_MATCHER = re.compile(DATE_PATTERN)
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"

def string_to_ntp_date_time(datestr):
    """
    Extract an ntp date from a ISO8601 formatted date string.
    @param datestr an ISO8601 formatted string containing date information
    @retval an ntp date number (seconds since jan 1 1900)
    @throws InstrumentParameterException if datestr cannot be formatted to
    a date.
    """
    if not isinstance(datestr, basestring):
        raise IOError('Value %s is not a string.' % str(datestr))

    if not DATE_MATCHER.match(datestr):
        raise ValueError("date string not in ISO8601 format YYYY-MM-DDTHH:MM:SS.SSSSZ")

    try:
        # This assumes input date string are in UTC (=GMT)

        # if there is no decimal place, add one to match the date format
        if datestr.find('.') == -1:
            if datestr[-1] != 'Z':
                datestr += '.0Z'
            else:
                datestr = datestr[:-1] + '.0Z'

        # if there is no trailing 'Z' on the input string add one
        if datestr[-1:] != 'Z':
            datestr += 'Z'

        dt = datetime.strptime(datestr, DATE_FORMAT)
        timestamp = (dt - datetime(1900, 1, 1)).total_seconds()

    except ValueError as e:
        raise ValueError('Value %s could not be formatted to a date. %s' % (str(datestr), e))

    return timestamp

class PlaybackPacket(Packet):
    def get_data_length(self):
        return len(self.payload)

    def get_data(self):
        return self.payload

    def get_timestamp(self):
        return self.header.time

    def __repr__(self):
        return repr(self.payload)

    @staticmethod
    def packet_from_fh(file_handle):
        data_buffer = bytearray()
        while True:
            byte = file_handle.read(1)
            if byte == '':
                return None

            data_buffer.append(byte)
            sync_index = data_buffer.find(PacketHeader.sync)
            if sync_index != -1:
                # found the sync bytes, read the rest of the header
                data_buffer.extend(file_handle.read(PacketHeader.header_size - len(PacketHeader.sync)))

                if len(data_buffer) < PacketHeader.header_size:
                    return None

                header = PacketHeader.from_buffer(data_buffer, sync_index)
                # read the payload
                payload = file_handle.read(header.payload_size)
                if len(payload) == header.payload_size:
                    packet = PlaybackPacket(payload=payload, header=header)
                    return packet

class PlaybackWrapper(object):
    def __init__(self, module, refdes, event_url, particle_url, reader_klass, allowed, files, max_events, handler=None):
        version = DriverWrapper.get_version(module)
        headers = {'sensor': refdes, 'deliveryType': 'streamed', 'version': version, 'module': module}
        self.max_events = max_events
        self.event_publisher = Publisher.from_url(event_url, handler, headers)
        self.particle_publisher = Publisher.from_url(particle_url, handler, headers, allowed, max_events)
        self.protocol = self.construct_protocol(module)
        self.reader = reader_klass(files, self.got_data)

    def set_header_filename(self, filename):
        self.event_publisher.set_source(filename)
        self.particle_publisher.set_source(filename)

    def playback(self):
        for index, filename in enumerate(self.reader.read()):
            if filename is not None:
                self.set_header_filename(filename)
                if hasattr(self.protocol, 'got_filename'):
                    self.protocol.got_filename(filename)
            if index % 1000 == 0:
                self.publish()

        self.publish()
        if hasattr(self.particle_publisher, 'write'):
            self.particle_publisher.write()

    def zplsc_playback(self):
        for index, filename in enumerate(self.reader.read()):
            if filename:
                self.set_header_filename(filename)
                log.info("filename is: %s", filename)
                if hasattr(self.protocol, 'got_filename'):
                    self.protocol.got_filename(filename)

        pub_index = 0
        while True:
            self.publish()
            pub_index = pub_index + 1
            log.info("publish index is: %d", pub_index)

    def got_data(self, packet):
        try:
            self.protocol.got_data(packet)
        except KeyboardInterrupt:
            raise
        except Exception as e:
            log.exception(e)

    @staticmethod
    def find_base_class(base):
        targets = (MenuInstrumentProtocol, CommandResponseInstrumentProtocol, InstrumentProtocol, object)
        while True:
            if base in targets:
                return base
            base = base.__base__

    def construct_protocol(self, proto_module):
        module = importlib.import_module(proto_module)
        if hasattr(module, 'create_playback_protocol'):
            return module.create_playback_protocol(self.handle_event)

        log.error('Unable to import and create playback protocol from module: %r', module)
        sys.exit(1)

    def publish(self):
        for publisher in [self.event_publisher, self.particle_publisher]:
            remaining = publisher.publish()
            while remaining >= publisher._max_events:
                remaining = publisher.publish()

    def handle_event(self, event_type, val=None):
        """
        Construct and send an asynchronous driver event.
        @param event_type a DriverAsyncEvent type specifier.
        @param val event value for sample and test result events.
        """
        event = {
            'type': event_type,
            'value': val,
            'time': time.time()
        }

        if isinstance(event[EventKeys.VALUE], Exception):
            event[EventKeys.VALUE] = encode_exception(event[EventKeys.VALUE])

        if event[EventKeys.TYPE] == DriverAsyncEvent.ERROR:
            log.error(event)

        if event[EventKeys.TYPE] == DriverAsyncEvent.SAMPLE:
            if event[EventKeys.VALUE].get('stream_name') != 'raw':
                # don't publish raw
                self.particle_publisher.enqueue(event)
        else:
            self.event_publisher.enqueue(event)

class DatalogReader(object):
    def __init__(self, files, callback):
        self.callback = callback

        self.files = []
        for each in files:
            self.files.extend(glob.glob(each))

        self.files.sort()
        if not all([os.path.isfile(f) for f in self.files]):
            raise Exception('Not all files found')
        self._filehandle = None
        self.target_types = [PacketType.FROM_INSTRUMENT, PacketType.PA_CONFIG]
        self.file_name_list = []

    def read(self):
        while True:
            if self._filehandle is None and not self.files:
                log.info('Completed reading specified port agent logs, exiting...')
                raise StopIteration

            if self._filehandle is None:
                name = self.files.pop(0)
                log.info('Begin reading: %r', name)
                # yield the filename so we can pass it through to the driver
                yield name
                self.file_name_list.append(name)
                self._filehandle = open(name, 'r')

            if not self._process_packet():
                self._filehandle.close()
                self._filehandle = None

            yield

    def _process_packet(self):
        packet = PlaybackPacket.packet_from_fh(self._filehandle)
        if packet is None:
            return False
        if packet.header.packet_type in self.target_types:
            self.callback(packet)
        return True

class DigiDatalogAsciiReader(DatalogReader):
    def __init__(self, files, callback):
        self.ooi_ts_regex = re.compile(r'<OOI-TS (.+?) [TX][NS]>\r\n(.*?)<\\OOI-TS>', re.DOTALL)
        self.buffer = ''
        self.MAXBUF = 65535

        super(DigiDatalogAsciiReader, self).__init__(files, callback)

        if all((self.search_utc(f) for f in self.files)):
            self.files.sort(key=self.search_utc)

    # special case for RSN archived data
    # if all files have date_UTC in filename then sort by that
    @staticmethod
    def search_utc(f):
        match = re.search('(\d+T\d+_UTC)', f)
        if match is None:
            return None
        return match.group(1)

    def _process_packet(self):
        chunk = self._filehandle.read(1024)
        if chunk != '':
            self.buffer += chunk
            new_index = 0
            for match in self.ooi_ts_regex.finditer(self.buffer):
                payload = match.group(2)
                try:
                    packet_time = string_to_ntp_date_time(match.group(1))
                    header = PacketHeader(packet_type=PacketType.FROM_INSTRUMENT,
                                          payload_size=len(payload), packet_time=packet_time)
                    header.set_checksum(payload)
                    packet = PlaybackPacket(payload=payload, header=header)
                    self.callback(packet)
                except ValueError:
                    log.error('Unable to extract timestamp from record: %r' % match.group())
                new_index = match.end()

            if new_index > 0:
                self.buffer = self.buffer[new_index:]

            if len(self.buffer) > self.MAXBUF:
                self.buffer = self.buffer[-self.MAXBUF:]

            return True

        return False

class ChunkyDatalogReader(DatalogReader):
    def _process_packet(self):
        data = self._filehandle.read(1024)
        if data != '':
            header = PacketHeader(packet_type=PacketType.FROM_INSTRUMENT,
                                  payload_size=len(data), packet_time=0)
            header.set_checksum(data)
            packet = PlaybackPacket(payload=data, header=header)
            self.callback(packet)
            return True
        return False

class ZplscReader(DatalogReader):
    def __init__(self, files, callback):
        super(ZplscReader, self).__init__(files, callback)

    def _process_packet(self):

        for name in self.file_name_list:
            data = 'downloaded file:' + name + '\n'
            header = PacketHeader(packet_type=PacketType.FROM_INSTRUMENT,
                                  payload_size=len(data), packet_time=0)
            header.set_checksum(data)
            packet = PlaybackPacket(payload=data, header=header)
            self.callback(packet)

        return False

def main():
    options = docopt(__doc__)

    module = options['<module>']
    refdes = options['<refdes>']
    event_url = options['<event_url>']
    particle_url = options['<particle_url>']
    files = options.get('<files>')
    allowed = options.get('--allowed')
    if allowed is not None:
        allowed = [_.strip() for _ in allowed.split(',')]
    max_events = options.get('--max_events')
    if not max_events:
        max_events = Publisher.DEFAULT_MAX_EVENTS
    else:
        max_events = int(max_events)

    # when running with the profiler, files will be a string
    # coerce to list
    if isinstance(files, basestring):
        files = [files]

    zplsc_reader = False

    if options['datalog']:
        reader = DatalogReader
    elif options['ascii']:
        reader = DigiDatalogAsciiReader
    elif options['chunky']:
        reader = ChunkyDatalogReader
    elif options['zplsc']:
        reader = ZplscReader
        zplsc_reader = True
    else:
        reader = None

    wrapper = PlaybackWrapper(module, refdes, event_url, particle_url, reader, allowed, files, max_events)
    if zplsc_reader:
        wrapper.zplsc_playback()
    else:
        wrapper.playback()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>__version__ = '4.9.14'
<commit_msg>Bump version to 4.10.0<commit_after>__version__ = '4.10.0'
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.1.32"
<commit_msg>update version number for pypi<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = "0.1.33"
<|endoftext|>