<commit_before>#!/usr/bin/env python
# Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert
#          Steve Reinhardt
#          Ali Saidi

import os, os.path, shutil, signal, socket, sys, time
from os import environ as env
from os.path import join as joinpath, expanduser

class rsync:
    def __init__(self):
        self.sudo = False
        self.rsync = 'rsync'
        self.compress = False
        self.archive = True
        self.delete = False
        self.options = ''

    def do(self, src, dst):
        args = []
        if self.sudo:
            args.append('sudo')

        args.append(self.rsync)
        if (self.archive):
            args.append('-a')
        if (self.compress):
            args.append('-z')
        if (self.delete):
            args.append('--delete')
        if len(self.options):
            args.append(self.options)
        args.append(src)
        args.append(dst)

        return os.spawnvp(os.P_WAIT, args[0], args)

def cleandir(dir):
    for root, dirs, files in os.walk(dir, False):
        for name in files:
            os.remove(joinpath(root, name))
        for name in dirs:
            os.rmdir(joinpath(root, name))

def date():
    return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime())

def remfile(file):
    if os.path.isfile(file):
        os.unlink(file)

def readval(filename):
    file = open(filename, 'r')
    value = file.readline().strip()
    file.close()
    return value

if __name__ == '__main__':
    rootdir = env.setdefault('ROOTDIR', os.getcwd())
    jobid = env['PBS_JOBID']
    jobname = env['PBS_JOBNAME']
    jobdir = joinpath(rootdir, jobname)
    basedir = joinpath(rootdir, 'Base')
    user = env['USER']

    env['POOLJOB'] = 'True'
    env['OUTPUT_DIR'] = jobdir
    env['JOBFILE'] = joinpath(basedir, 'test.py')
    env['JOBNAME'] = jobname

    def echofile(filename, string):
        try:
            f = file(joinpath(jobdir, filename), 'w')
            print >>f, string
            f.flush()
            f.close()
        except IOError,e:
            sys.exit(e)

    if os.path.isdir("/work"):
        workbase = "/work"
    else:
        workbase = "/tmp/"

    workdir = joinpath(workbase, '%s.%s' % (user, jobid))

    os.umask(0022)

    echofile('.start', date())
    echofile('.jobid', jobid)
    echofile('.host', socket.gethostname())

    if os.path.isdir(workdir):
        cleandir(workdir)
    else:
        os.mkdir(workdir)

    if os.path.isdir('/z/dist'):
        sync = rsync()
        sync.delete = True
        sync.sudo = True
        sync.do('poolfs::dist/m5/', '/z/dist/m5/')

    try:
        os.chdir(workdir)
    except OSError,e:
        sys.exit(e)

    os.symlink(joinpath(jobdir, 'output'), 'status.out')

    args = [ joinpath(basedir, 'm5'), joinpath(basedir, 'run.mpy') ]
    if not len(args):
        sys.exit("no arguments")

    print 'starting job... %s' % date()
    print ' '.join(args)
    print
    sys.stdout.flush()

    childpid = os.fork()
    if not childpid:
        # Execute command
        sys.stdin.close()
        fd = os.open(joinpath(jobdir, "output"),
                     os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(fd, sys.stdout.fileno())
        os.dup2(fd, sys.stderr.fileno())
        os.execvp(args[0], args)

    def handler(signum, frame):
        if childpid != 0:
            os.kill(childpid, signum)

    signal.signal(signal.SIGHUP, handler)
    signal.signal(signal.SIGINT, handler)
    signal.signal(signal.SIGQUIT, handler)
    signal.signal(signal.SIGTERM, handler)
    signal.signal(signal.SIGSTOP, handler)
    signal.signal(signal.SIGCONT, handler)
    signal.signal(signal.SIGUSR1, handler)
    signal.signal(signal.SIGUSR2, handler)

    done = 0
    while not done:
        try:
            thepid,ec = os.waitpid(childpid, 0)
            if ec:
                print 'Exit code ', ec
                echofile('.failure', date())
            else:
                echofile('.success', date())
            done = 1
        except OSError:
            pass

    print '\njob complete... %s' % date()
    echofile('.stop', date())
<commit_msg>output dir changes to python files<commit_after>#!/usr/bin/env python
# Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert
#          Steve Reinhardt
#          Ali Saidi

import os, os.path, shutil, signal, socket, sys, time
from os import environ as env
from os.path import join as joinpath, expanduser

class rsync:
    def __init__(self):
        self.sudo = False
        self.rsync = 'rsync'
        self.compress = False
        self.archive = True
        self.delete = False
        self.options = ''

    def do(self, src, dst):
        args = []
        if self.sudo:
            args.append('sudo')

        args.append(self.rsync)
        if (self.archive):
            args.append('-a')
        if (self.compress):
            args.append('-z')
        if (self.delete):
            args.append('--delete')
        if len(self.options):
            args.append(self.options)
        args.append(src)
        args.append(dst)

        return os.spawnvp(os.P_WAIT, args[0], args)

def cleandir(dir):
    for root, dirs, files in os.walk(dir, False):
        for name in files:
            os.remove(joinpath(root, name))
        for name in dirs:
            os.rmdir(joinpath(root, name))

def date():
    return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime())

def remfile(file):
    if os.path.isfile(file):
        os.unlink(file)

def readval(filename):
    file = open(filename, 'r')
    value = file.readline().strip()
    file.close()
    return value

if __name__ == '__main__':
    rootdir = env.setdefault('ROOTDIR', os.getcwd())
    jobid = env['PBS_JOBID']
    jobname = env['PBS_JOBNAME']
    jobdir = joinpath(rootdir, jobname)
    basedir = joinpath(rootdir, 'Base')
    user = env['USER']

    env['POOLJOB'] = 'True'
    env['OUTPUT_DIR'] = jobdir
    env['JOBFILE'] = joinpath(basedir, 'test.py')
    env['JOBNAME'] = jobname

    def echofile(filename, string):
        try:
            f = file(joinpath(jobdir, filename), 'w')
            print >>f, string
            f.flush()
            f.close()
        except IOError,e:
            sys.exit(e)

    if os.path.isdir("/work"):
        workbase = "/work"
    else:
        workbase = "/tmp/"

    workdir = joinpath(workbase, '%s.%s' % (user, jobid))

    os.umask(0022)

    echofile('.start', date())
    echofile('.jobid', jobid)
    echofile('.host', socket.gethostname())

    if os.path.isdir(workdir):
        cleandir(workdir)
    else:
        os.mkdir(workdir)

    if os.path.isdir('/z/dist'):
        sync = rsync()
        sync.delete = True
        sync.sudo = True
        sync.do('poolfs::dist/m5/', '/z/dist/m5/')

    try:
        os.chdir(workdir)
    except OSError,e:
        sys.exit(e)

    os.symlink(joinpath(jobdir, 'output'), 'status.out')

    args = [ joinpath(basedir, 'm5'), '-d', '%s' % jobdir, joinpath(basedir, 'run.mpy') ]
    if not len(args):
        sys.exit("no arguments")

    print 'starting job... %s' % date()
    print ' '.join(args)
    print
    sys.stdout.flush()

    childpid = os.fork()
    if not childpid:
        # Execute command
        sys.stdin.close()
        fd = os.open(joinpath(jobdir, "output"),
                     os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
        os.dup2(fd, sys.stdout.fileno())
        os.dup2(fd, sys.stderr.fileno())
        os.execvp(args[0], args)

    def handler(signum, frame):
        if childpid != 0:
            os.kill(childpid, signum)

    signal.signal(signal.SIGHUP, handler)
    signal.signal(signal.SIGINT, handler)
    signal.signal(signal.SIGQUIT, handler)
    signal.signal(signal.SIGTERM, handler)
    signal.signal(signal.SIGSTOP, handler)
    signal.signal(signal.SIGCONT, handler)
    signal.signal(signal.SIGUSR1, handler)
    signal.signal(signal.SIGUSR2, handler)

    done = 0
    while not done:
        try:
            thepid,ec = os.waitpid(childpid, 0)
            if ec:
                print 'Exit code ', ec
                echofile('.failure', date())
            else:
                echofile('.success', date())
            done = 1
        except OSError:
            pass

    print '\njob complete... %s' % date()
    echofile('.stop', date())
<|endoftext|>


<commit_before>"""Match a large set of multi-word expressions in O(1) time.

The idea is to associate each word in the vocabulary with a tag, noting whether
they begin, end, or are inside at least one pattern. An additional tag is used
for single-word patterns. Complete patterns are also stored in a hash set.

When we process a document, we look up the words in the vocabulary, to associate
the words with the tags.  We then search for tag-sequences that correspond to
valid candidates. Finally, we look up the candidates in the hash set.

For instance, to search for the phrases "Barack Hussein Obama" and "Hilary Clinton", we
would associate "Barack" and "Hilary" with the B tag, Hussein with the I tag,
and Obama and Clinton with the L tag.

The document "Barack Clinton and Hilary Clinton" would have the tag sequence
[{B}, {L}, {}, {B}, {L}], so we'd get two matches. However, only the second candidate
is in the phrase dictionary, so only one is returned as a match.

The algorithm is O(n) at run-time for document of length n because we're only ever
matching over the tag patterns. So no matter how many phrases we're looking for,
our pattern set stays very small (exact size depends on the maximum length we're
looking for, as the query language currently has no quantifiers)
"""
from __future__ import print_function, unicode_literals, division

import plac

from preshed.maps import PreshMap
from spacy.strings import hash_string
from spacy.en import English
from spacy.matcher import Matcher

from spacy.attrs import FLAG63 as U_ENT
from spacy.attrs import FLAG62 as L_ENT
from spacy.attrs import FLAG61 as I_ENT
from spacy.attrs import FLAG60 as B_ENT

def get_bilou(length):
    if length == 1:
        return [U_ENT]
    else:
        return [B_ENT] + [I_ENT] * (length - 2) + [L_ENT]

def make_matcher(vocab, max_length):
    abstract_patterns = []
    for length in range(1, max_length+1):
        abstract_patterns.append([{tag: True} for tag in get_bilou(length)])
    return Matcher(vocab, {'Candidate': ('CAND', {}, abstract_patterns)})

def get_matches(matcher, pattern_ids, doc):
    matches = []
    for label, start, end in matcher(doc):
        candidate = doc[start : end]
        if pattern_ids[hash_string(candidate.text)] == True:
            start = candidate[0].idx
            end = candidate[-1].idx + len(candidate[-1])
            matches.append((start, end, candidate.root.tag_, candidate.text))
    return matches

def merge_matches(doc, matches):
    for start, end, tag, text in matches:
        doc.merge(start, end, tag, text, 'MWE')

def main():
    nlp = English(parser=False, tagger=False, entity=False)

    gazetteer = [u'M.I.A.', 'Shiny Happy People', 'James E. Jones']
    example_text = u'The artist M.I.A. did a cover of Shiny Happy People. People is not an entity.'
    pattern_ids = PreshMap()
    max_length = 0
    for pattern_str in gazetteer:
        pattern = nlp.tokenizer(pattern_str)
        bilou_tags = get_bilou(len(pattern))
        for word, tag in zip(pattern, bilou_tags):
            lexeme = nlp.vocab[word.orth]
            lexeme.set_flag(tag, True)
        pattern_ids[hash_string(pattern.text)] = True
        max_length = max(max_length, len(pattern))

    matcher = make_matcher(nlp.vocab, max_length)

    doc = nlp(example_text)
    matches = get_matches(matcher, pattern_ids, doc)
    merge_matches(doc, matches)
    for token in doc:
        print(token.text, token.ent_type_)

if __name__ == '__main__':
    plac.call(main)
<commit_msg>* Fix multi_word_matches script<commit_after>"""Match a large set of multi-word expressions in O(1) time.

The idea is to associate each word in the vocabulary with a tag, noting whether
they begin, end, or are inside at least one pattern. An additional tag is used
for single-word patterns. Complete patterns are also stored in a hash set.

When we process a document, we look up the words in the vocabulary, to associate
the words with the tags.  We then search for tag-sequences that correspond to
valid candidates. Finally, we look up the candidates in the hash set.

For instance, to search for the phrases "Barack Hussein Obama" and "Hilary Clinton", we
would associate "Barack" and "Hilary" with the B tag, Hussein with the I tag,
and Obama and Clinton with the L tag.

The document "Barack Clinton and Hilary Clinton" would have the tag sequence
[{B}, {L}, {}, {B}, {L}], so we'd get two matches. However, only the second candidate
is in the phrase dictionary, so only one is returned as a match.

The algorithm is O(n) at run-time for document of length n because we're only ever
matching over the tag patterns. So no matter how many phrases we're looking for,
our pattern set stays very small (exact size depends on the maximum length we're
looking for, as the query language currently has no quantifiers)
"""
from __future__ import print_function, unicode_literals, division
from ast import literal_eval
from bz2 import BZ2File
import time
import math

import plac

from preshed.maps import PreshMap
from spacy.strings import hash_string
from spacy.en import English
from spacy.matcher import Matcher

from spacy.attrs import FLAG63 as B_ENT
from spacy.attrs import FLAG62 as L_ENT
from spacy.attrs import FLAG61 as I_ENT

from spacy.attrs import FLAG60 as B2_ENT
from spacy.attrs import FLAG59 as B3_ENT
from spacy.attrs import FLAG58 as B4_ENT
from spacy.attrs import FLAG57 as B5_ENT
from spacy.attrs import FLAG56 as B6_ENT
from spacy.attrs import FLAG55 as B7_ENT
from spacy.attrs import FLAG54 as B8_ENT
from spacy.attrs import FLAG53 as B9_ENT
from spacy.attrs import FLAG52 as B10_ENT

from spacy.attrs import FLAG51 as I3_ENT
from spacy.attrs import FLAG50 as I4_ENT
from spacy.attrs import FLAG49 as I5_ENT
from spacy.attrs import FLAG48 as I6_ENT
from spacy.attrs import FLAG47 as I7_ENT
from spacy.attrs import FLAG46 as I8_ENT
from spacy.attrs import FLAG45 as I9_ENT
from spacy.attrs import FLAG44 as I10_ENT

from spacy.attrs import FLAG43 as L2_ENT
from spacy.attrs import FLAG42 as L3_ENT
from spacy.attrs import FLAG41 as L4_ENT
from spacy.attrs import FLAG40 as L5_ENT
from spacy.attrs import FLAG39 as L6_ENT
from spacy.attrs import FLAG38 as L7_ENT
from spacy.attrs import FLAG37 as L8_ENT
from spacy.attrs import FLAG36 as L9_ENT
from spacy.attrs import FLAG35 as L10_ENT

def get_bilou(length):
    if length == 1:
        return [U_ENT]
    elif length == 2:
        return [B2_ENT, L2_ENT]
    elif length == 3:
        return [B3_ENT, I3_ENT, L3_ENT]
    elif length == 4:
        return [B4_ENT, I4_ENT, I4_ENT, L4_ENT]
    elif length == 5:
        return [B5_ENT, I5_ENT, I5_ENT, L5_ENT]
    elif length == 6:
        return [B6_ENT, I6_ENT, I6_ENT, I6_ENT, I6_ENT, L6_ENT]
    elif length == 7:
        return [B7_ENT, I7_ENT, I7_ENT, I7_ENT, I7_ENT, I7_ENT, L7_ENT]
    elif length == 8:
        return [B8_ENT, I8_ENT, I8_ENT, I8_ENT, I8_ENT, I8_ENT, I8_ENT, L8_ENT]
    elif length == 9:
        return [B9_ENT, I9_ENT, I9_ENT, I9_ENT, I9_ENT, I9_ENT, I9_ENT, L9_ENT]
    elif length == 10:
        return [B10_ENT, I10_ENT, I10_ENT, I10_ENT, I10_ENT, I10_ENT, I10_ENT, L10_ENT]

def make_matcher(vocab, max_length):
    abstract_patterns = []
    for length in range(2, max_length):
        abstract_patterns.append([{tag: True} for tag in get_bilou(length)])
    return Matcher(vocab, {'Candidate': ('CAND', {}, abstract_patterns)})

def get_matches(matcher, pattern_ids, doc):
    matches = []
    for label, start, end in matcher(doc):
        candidate = doc[start : end]
        if pattern_ids[hash_string(candidate.text)] == True:
            start = candidate[0].idx
            end = candidate[-1].idx + len(candidate[-1])
            matches.append((start, end, candidate.root.tag_, candidate.text))
    return matches

def merge_matches(doc, matches):
    for start, end, tag, text in matches:
        doc.merge(start, end, tag, text, 'MWE')

def read_gazetteer(loc):
    for line in open(loc):
        phrase = literal_eval('u' + line.strip())
        if ' (' in phrase and phrase.endswith(')'):
            phrase = phrase.split(' (', 1)[0]
        yield phrase

def read_text(bz2_loc):
    with BZ2File(bz2_loc) as file_:
        for line in file_:
            yield line.decode('utf8')

def main(patterns_loc, text_loc):
    nlp = English(parser=False, tagger=False, entity=False)

    pattern_ids = PreshMap()
    max_length = 10
    i = 0
    for pattern_str in read_gazetteer(patterns_loc):
        pattern = nlp.tokenizer(pattern_str)
        if len(pattern) < 2 or len(pattern) >= max_length:
            continue
        bilou_tags = get_bilou(len(pattern))
        for word, tag in zip(pattern, bilou_tags):
            lexeme = nlp.vocab[word.orth]
            lexeme.set_flag(tag, True)
        pattern_ids[hash_string(pattern.text)] = True
        i += 1
        if i >= 10000001:
            break

    matcher = make_matcher(nlp.vocab, max_length)

    t1 = time.time()

    for text in read_text(text_loc):
        doc = nlp.tokenizer(text)
        matches = get_matches(matcher, pattern_ids, doc)
        merge_matches(doc, matches)
    t2 = time.time()
    print('10 ^ %d patterns took %d s' % (round(math.log(i, 10)), t2-t1))

if __name__ == '__main__':
    plac.call(main)
<|endoftext|>


<commit_before>#! /usr/bin/python

from asana import *

import re, argparse

#argument parsing
parser = argparse.ArgumentParser(
	description='Create a view of all tasks by createor',
	formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('name', help='Name of the project that will contain the view')
parser.add_argument('--project-regex', '-p', help='Regex of which projects to generate report for')
parser.add_argument('--api-key', '-a', help='API key to use')

options = parser.parse_args()

Entity.set_api(AsanaAPI(options.api_key, debug=True, cache=True))

print 'Loading projects and tasks...'

projects = Project.find({
	'name': lambda n: re.search(options.projects, n)
})

tasks = []

for project in projects:
	tasks.extend(project.tasks)

print 'Creating view project...'

view_project = Project({
	'name': 'Creator View',
	'workspace': projects[0].workspace.id,
	'team': projects[0].team.id
})

view_project.save()

#initialize each creator to an empty array
tasks_by_creator = {t.created_by: [] for t in tasks}

print 'Adding tasks to meeting...'

for task in tasks:
	tasks_by_creator[task.created_by].append(task)

for creator in tasks_by_creator.keys():

	for task in tasks_by_creator[creator]:
		view_project.add_task(task)

	# need to add section after tasks to correct order
	section = Section({'name':creator.name + ':'})

	view_project.add_task(section)

print 'Done!'<commit_msg>Update example<commit_after>#! /usr/bin/python

from asana import Entity, AsanaAPI, Project, Section

import re, argparse

#argument parsing
parser = argparse.ArgumentParser(
	description='Create a view of all tasks by createor',
	formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('name', help='Name of the project that will contain the view')
parser.add_argument('--project-regex', '-p', help='Regex of which projects to generate report for')
parser.add_argument('--api-key', '-a', help='API key to use')

options = parser.parse_args()

Entity.set_api(AsanaAPI(options.api_key, debug=True, cache=True))

print 'Loading projects and tasks...'

projects = Project.find({
	'name': lambda n: re.search(options.project_regex, n)
})

tasks = []

for project in projects:
	tasks.extend(project.tasks)

print 'Creating view project...'

view_project = Project({
	'name': 'Creator View',
	'workspace': projects[0].workspace.id,
	'team': projects[0].team.id
})

view_project.save()

#initialize each creator to an empty array
tasks_by_creator = {t.created_by: [] for t in tasks}

print 'Adding tasks to meeting...'

for task in tasks:
	tasks_by_creator[task.created_by].append(task)

for creator in tasks_by_creator.keys():

	for task in tasks_by_creator[creator]:
		view_project.add_task(task)

	# need to add section after tasks to correct order
	section = Section({'name':creator.name + ':'})

	view_project.add_task(section)

print 'Done!'<|endoftext|>


<commit_before>#! /usr/bin/env python

"""
    start socket based minimal readline exec server
"""
# this part of the program only executes on the server side
#

progname = 'socket_readline_exec_server-1.2'
debug = 0

import sys, socket, os
try:
    import fcntl
except ImportError:
    fcntl = None

if debug: #  and not os.isatty(sys.stdin.fileno()):
    f = open('/tmp/execnet-socket-pyout.log', 'a', 0)
    old = sys.stdout, sys.stderr
    sys.stdout = sys.stderr = f
    #import py
    #compile = py.code.compile

def print_(*args):
    print(" ".join(args))

if sys.version_info > (3, 0):
    exec("""def exec_(source, locs):
    exec(source, locs)""")
else:
    exec("""def exec_(source, locs):
    exec source in locs""")

def exec_from_one_connection(serversock):
    print_(progname, 'Entering Accept loop', serversock.getsockname())
    clientsock,address = serversock.accept()
    print_(progname, 'got new connection from %s %s' % address)
    clientfile = clientsock.makefile('r+',0)
    print_("reading line")
    # rstrip so that we can use \r\n for telnet testing
    source = clientfile.readline().rstrip()
    clientfile.close()
    g = {'clientsock' : clientsock, 'address' : address}
    source = eval(source)
    if source:
        co = compile(source+'\n', source, 'exec')
        print_(progname, 'compiled source, executing')
        try:
            exec_(co, g)
        finally:
            print_(progname, 'finished executing code')
            # background thread might hold a reference to this (!?)
            #clientsock.close()

def bind_and_listen(hostport):
    if isinstance(hostport, str):
        host, port = hostport.split(':')
        hostport = (host, int(port))
    serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    # set close-on-exec
    if hasattr(fcntl, 'FD_CLOEXEC'):
        old = fcntl.fcntl(serversock.fileno(), fcntl.F_GETFD)
        fcntl.fcntl(serversock.fileno(), fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
    # allow the address to be re-used in a reasonable amount of time
    if os.name == 'posix' and sys.platform != 'cygwin':
        serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    serversock.bind(hostport)
    serversock.listen(5)
    return serversock

def startserver(serversock, loop=False):
    try:
        while 1:
            try:
                exec_from_one_connection(serversock)
            except (KeyboardInterrupt, SystemExit):
                raise
            except:
                if debug:
                    import traceback
                    traceback.print_exc()
                else:
                    excinfo = sys.exc_info()
                    print_("got exception", excinfo[1])
            if not loop:
                break
    finally:
        print_("leaving socketserver execloop")
        serversock.shutdown(2)

if __name__ == '__main__':
    import sys
    if len(sys.argv)>1:
        hostport = sys.argv[1]
    else:
        hostport = ':8888'
    serversock = bind_and_listen(hostport)
    startserver(serversock, loop=False)

<commit_msg>convert argument to string<commit_after>#! /usr/bin/env python

"""
    start socket based minimal readline exec server
"""
# this part of the program only executes on the server side
#

progname = 'socket_readline_exec_server-1.2'
debug = 0

import sys, socket, os
try:
    import fcntl
except ImportError:
    fcntl = None

if debug: #  and not os.isatty(sys.stdin.fileno()):
    f = open('/tmp/execnet-socket-pyout.log', 'a', 0)
    old = sys.stdout, sys.stderr
    sys.stdout = sys.stderr = f
    #import py
    #compile = py.code.compile

def print_(*args):
    print(" ".join(str(arg) for arg in args))

if sys.version_info > (3, 0):
    exec("""def exec_(source, locs):
    exec(source, locs)""")
else:
    exec("""def exec_(source, locs):
    exec source in locs""")

def exec_from_one_connection(serversock):
    print_(progname, 'Entering Accept loop', serversock.getsockname())
    clientsock,address = serversock.accept()
    print_(progname, 'got new connection from %s %s' % address)
    clientfile = clientsock.makefile('r+',0)
    print_("reading line")
    # rstrip so that we can use \r\n for telnet testing
    source = clientfile.readline().rstrip()
    clientfile.close()
    g = {'clientsock' : clientsock, 'address' : address}
    source = eval(source)
    if source:
        co = compile(source+'\n', source, 'exec')
        print_(progname, 'compiled source, executing')
        try:
            exec_(co, g)
        finally:
            print_(progname, 'finished executing code')
            # background thread might hold a reference to this (!?)
            #clientsock.close()

def bind_and_listen(hostport):
    if isinstance(hostport, str):
        host, port = hostport.split(':')
        hostport = (host, int(port))
    serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    # set close-on-exec
    if hasattr(fcntl, 'FD_CLOEXEC'):
        old = fcntl.fcntl(serversock.fileno(), fcntl.F_GETFD)
        fcntl.fcntl(serversock.fileno(), fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
    # allow the address to be re-used in a reasonable amount of time
    if os.name == 'posix' and sys.platform != 'cygwin':
        serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    serversock.bind(hostport)
    serversock.listen(5)
    return serversock

def startserver(serversock, loop=False):
    try:
        while 1:
            try:
                exec_from_one_connection(serversock)
            except (KeyboardInterrupt, SystemExit):
                raise
            except:
                if debug:
                    import traceback
                    traceback.print_exc()
                else:
                    excinfo = sys.exc_info()
                    print_("got exception", excinfo[1])
            if not loop:
                break
    finally:
        print_("leaving socketserver execloop")
        serversock.shutdown(2)

if __name__ == '__main__':
    import sys
    if len(sys.argv)>1:
        hostport = sys.argv[1]
    else:
        hostport = ':8888'
    serversock = bind_and_listen(hostport)
    startserver(serversock, loop=False)

<|endoftext|>


<commit_before><commit_msg>make test work in python 2.5<commit_after><|endoftext|>


<commit_before><commit_msg>Remove vestigial print statements.<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python3
"""
Author: Nate Levesque <public@thenaterhood.com>
File: testAnal.py
Language: Python3
Description:

"""
import unittest
import json
from urllib.request import urlopen

domain = "192.168.1.103"
url = "http://192.168.1.103/engine/"

class testWebEngine(unittest.TestCase):

	def setUp(self):
		return

	def test_smoke(self):
		self.assertTrue( True )

	def test_sanitizeShort(self):
		# Test the ability to sanitize a short string
		decodedData = getData( url+"api_saneString.php?testvar=justtest" )
		self.assertEqual( decodedData["testvar"], "justtest" )

	def test_sanitizeMessy(self):
		# Test the ability to remove non-alphanumberic characters
		decodedData = getData( url+"api_saneString.php?testvar=)just*test$*@(*%" )
		self.assertEqual( decodedData["testvar"], "justtest" )

	def test_sessionDomain(self):
		# Test that the session class contains expected data
		decodedData = getData( url+"api_session.php" )
		self.assertEqual( decodedData["domain"], domain )

	def test_sessionData(self):
		# Test that the session class can pick up data
		decodedData = getData( url+"api_session.php?name=test" )
		self.assertEqual( decodedData["name"], "test" )

	def test_sessionDataMessy(self):
		# Test that the session class can pick up messy data correctly
		decodedData = getData( url+"api_session.php?name=te$%@!st&konami=pride" )
		self.assertEqual( decodedData["name"], "test" )

	def test_article_missing(self):
		# Test that the article class can return data and deal with missing posts
		decodedData = getData( url+"api_post.php?node=test&element=title" )
		self.assertEqual( decodedData["title"], "Oops! Post Not Found!" )

	def test_inventory_retrieve(self):
		# Test that an item can be retrieved from the inventory with the proper data
		decodedData = getData( url+"api_inventory.php")
		self.assertEqual( decodedData["title"], "Lovin' those Facebook Likes")
		self.assertEqual( decodedData["tags"], "facebook, privacy, social networking")

	def test_redirect(self):
		# Test that a redirect can be correctly initialized
		decodedData = urlopen( url+"api_redirect.php?from=origin&to=destination").read().decode('utf-8')
		self.assertEqual( decodedData, "origin to destination")

def getData( address ):
	page = urlopen( address ).read().decode('utf-8')
	return json.loads(page)

if __name__ == "__main__":
    unittest.main()
<commit_msg>update unit testing for new url from refactoring<commit_after>#!/usr/bin/python3
"""
Author: Nate Levesque <public@thenaterhood.com>
File: testAnal.py
Language: Python3
Description:

"""
import unittest
import json
from urllib.request import urlopen

domain = "192.168.1.103"
url = "http://192.168.1.103/engine/lib/"

class testWebEngine(unittest.TestCase):

	def setUp(self):
		return

	def test_smoke(self):
		self.assertTrue( True )

	def test_sanitizeShort(self):
		# Test the ability to sanitize a short string
		decodedData = getData( url+"api_saneString.php?testvar=justtest" )
		self.assertEqual( decodedData["testvar"], "justtest" )

	def test_sanitizeMessy(self):
		# Test the ability to remove non-alphanumberic characters
		decodedData = getData( url+"api_saneString.php?testvar=)just*test$*@(*%" )
		self.assertEqual( decodedData["testvar"], "justtest" )

	def test_sessionDomain(self):
		# Test that the session class contains expected data
		decodedData = getData( url+"api_session.php" )
		self.assertEqual( decodedData["domain"], domain )

	def test_sessionData(self):
		# Test that the session class can pick up data
		decodedData = getData( url+"api_session.php?name=test" )
		self.assertEqual( decodedData["name"], "test" )

	def test_sessionDataMessy(self):
		# Test that the session class can pick up messy data correctly
		decodedData = getData( url+"api_session.php?name=te$%@!st&konami=pride" )
		self.assertEqual( decodedData["name"], "test" )

	def test_article_missing(self):
		# Test that the article class can return data and deal with missing posts
		decodedData = getData( url+"api_post.php?node=test&element=title" )
		self.assertEqual( decodedData["title"], "Oops! Post Not Found!" )

	def test_inventory_retrieve(self):
		# Test that an item can be retrieved from the inventory with the proper data
		decodedData = getData( url+"api_inventory.php")
		self.assertEqual( decodedData["title"], "Lovin' those Facebook Likes")
		self.assertEqual( decodedData["tags"], "facebook, privacy, social networking")

	def test_redirect(self):
		# Test that a redirect can be correctly initialized
		decodedData = urlopen( url+"api_redirect.php?from=origin&to=destination").read().decode('utf-8')
		self.assertEqual( decodedData, "origin to destination")

def getData( address ):
	page = urlopen( address ).read().decode('utf-8')
	return json.loads(page)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before><commit_msg>add missing args to tests<commit_after><|endoftext|>


<commit_before><commit_msg>simplify. not sure why I needed this<commit_after><|endoftext|>


<commit_before><commit_msg>a couple more tests again again<commit_after><|endoftext|>


<commit_before><commit_msg>remove timesince test<commit_after><|endoftext|>


<commit_before><commit_msg>checkpoint<commit_after><|endoftext|>


<commit_before><commit_msg>fixed bug: reused test case name in signal_tests<commit_after><|endoftext|>


<commit_before><commit_msg>Remove duplicate import<commit_after><|endoftext|>


<commit_before><commit_msg>stored state using pickle<commit_after><|endoftext|>


<commit_before><commit_msg>fix the fake module to have verison and docstring<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- python -*-
#
# Git Version: @git@

#-----------------------------------------------------------------------
# XALT: A tool to track the programs on a cluster.
# Copyright (C) 2013-2014 Robert McLay and Mark Fahey
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser  General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
#-----------------------------------------------------------------------

#  xalt_syslog takes the output found in xalt syslog file
#  provided with the --syslog_file=ans argument and puts it
#  into the database
#
#  One has to set up syslog to capture the XALT output
#  Example is provided in the xalt_syslog.conf file that can
#  be put in the /etc/rsyslog.d directory.
#
#

from __future__  import print_function
import os, sys, re, MySQLdb, json, time, argparse, base64, zlib, shlex

dirNm, execName = os.path.split(os.path.realpath(sys.argv[0]))
sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../libexec")))
sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../site")))

from XALTdb        import XALTdb
from xalt_site_pkg import translate
from xalt_util     import *
from xalt_global   import *
from progressBar   import ProgressBar
from Rmap_XALT     import Rmap

import inspect

def __LINE__():
    try:
        raise Exception
    except:
        return sys.exc_info()[2].tb_frame.f_back.f_lineno

def __FILE__():
    return inspect.currentframe().f_code.co_filename

#print ("file: '%s', line: %d" % (__FILE__(), __LINE__()), file=sys.stderr)

ConfigBaseNm = "xalt_db"
ConfigFn     = ConfigBaseNm + ".conf"
logger       = config_logger()

class CmdLineOptions(object):
  """ Command line Options class """

  def __init__(self):
    """ Empty Ctor """
    pass

  def execute(self):
    """ Specify command line arguments and parse the command line"""
    parser = argparse.ArgumentParser()
    parser.add_argument("--syslog",      dest='syslog',   action="store",      help="Location and name of syslog file")
    parser.add_argument("--syshost",     dest='syshost',  action="store",      default=".*",
                                                                               help="Store just records that come from syshost")
    parser.add_argument("--leftover_fn", dest='leftover', action="store",      default='leftover.log',
                                                                               help="Name of the leftover file")
    parser.add_argument("--timer",       dest='timer',    action="store_true", help="Time runtime")
    parser.add_argument("--reverseMapD", dest='rmapD',    action="store",      help="Path to the directory containing the json reverseMap")
    parser.add_argument("--dbname",      dest='dbname',   action="store",      default="xalt", help="Name of the database")
    parser.add_argument("--confFn",      dest='confFn',   action="store",      default="xalt_db.conf", help="Name of the database")
    args = parser.parse_args()
    return args

class Record(object):
  """
  This class holds the pieces of a single record as it comes in from
  syslog
  """
  def __init__(self, t):
    nblks          = int(t['nb'])
    self.__nblks   = nblks
    self.__kind    = t['kind']
    self.__syshost = t['syshost']
    self.__blkCnt  = 0

    blkA = []
    for i in range(nblks):
      blkA.append(False)

    self.__blkA    = blkA
    self.addBlk(t)

  def addBlk(self,t):
    idx               = int(t['idx'])
    self.__blkA[idx]  = t['value']
    self.__blkCnt    += 1

  def completed(self):
    return (self.__blkCnt >= self.__nblks)

  def value(self):
    return "".join(self.__blkA)

  def prt(self, prefix, key):
    sA    = []
    nblks = self.__nblks
    blkA  = self.__blkA

    sPA   = []
    sPA.append(prefix)
    sPA.append(" kind:")
    sPA.append(self.__kind)
    sPA.append(" syshost:")
    sPA.append(self.__syshost)
    sPA.append(" key:")
    sPA.append(key)
    sPA.append(" nb:")
    sPA.append(str(nblks))
    ss = "".join(sPA)

    for idx in range(nblks):
      value = blkA[idx]
      if (value):
        sA.append(ss)
        sA.append(" idx:")
        sA.append(str(idx))
        sA.append(" value:")
        sA.append(value)
        sA.append("\n")
    return "".join(sA)

def parseSyslogV1(s, clusterName):
  t = { 'kind' : None, 'value' : None, 'syshost' : None, 'version' : 1}

  idx = s.find("link:")
  if (idx == -1):
    idx = s.find("run:")
  if (idx == -1):
    return t, False

  array        = s[idx:].split(":")
  t['kind']    = array[0].strip()
  t['syshost'] = array[1].strip()
  t['value']   = base64.b64decode(array[2])

  if (clusterName != ".*" and clusterName != t['syshost']):
    return t, False

  return t, True

def parseSyslogV2(s, clusterName, recordT):
  t = { 'kind' : None, 'syshost' : None, 'value' : None, 'version' : 2}

  idx = s.find(" V:2 ")
  if (idx  == -1):
    return t, False

  # Strip off "XALT_LOGGING V:2" from string.
  s                      = s[idx+5:]

  # Setup parser
  lexer                  = shlex.shlex(s)
  lexer.whitespace_split = True
  lexer.whitespace       = ' :'

  # Pick off two values at a time.
  try:
    while True:
      key    = next(lexer)
      value  = next(lexer)
      t[key] = value
  except StopIteration as e:
    pass

  # get the key from the input, then place an entry in the *recordT* table.
  # or just add the block to the current record.
  key = t['key']
  r    = recordT.get(key, None)
  if (r):
    r.addBlk(t)
  else:
    r  = Record(t)
    recordT[key] = r

  if (clusterName != ".*" and clusterName != t['syshost']):
    return t, False

  # If the block is completed then grap the value, remove the entry from *recordT*
  # and return a completed table.
  if (r.completed()):

    rv   = r.value()
    b64v = base64.b64decode(rv)
    vv   = zlib.decompress(b64v)

    t['value'] = vv
    recordT.pop(key)
    return t, True

  # Entry is not complete.
  return t, False

def parseSyslog(s, clusterName, recordT):
  if ("XALT_LOGGING" in s) and ("V:2" in s):
    return parseSyslogV2(s, clusterName, recordT)
  return parseSyslogV1(s, clusterName)

def main():
  """
  read from syslog file into XALT db.
  """

  sA = []
  sA.append("CommandLine:")
  for v in sys.argv:
    sA.append('"'+v+'"')
  XALT_Stack.push(" ".join(sA))

  args       = CmdLineOptions().execute()
  xalt       = XALTdb(args.confFn)
  syslogFile = args.syslog

  # should add a check if file exists
  num    = int(capture("cat "+syslogFile+" | wc -l"))
  icnt   = 0

  t1     = time.time()

  rmapT  = Rmap(args.rmapD).reverseMapT()

  lnkCnt = 0
  runCnt = 0
  badCnt = 0
  count  = 0

  recordT = {}

  if (num == 0):
    return

  pbar   = ProgressBar(maxVal=num)

  fnA = [ args.leftover, syslogFile ]

  for fn in fnA:
    if (not os.path.isfile(fn)):
      continue

    f=open(fn, 'r')
    for line in f:
      if (not ("XALT_LOGGING" in line)):
        continue
      t, done = parseSyslog(line, args.syshost, recordT)
      if (not done):
        continue

      try:
        XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost'])

        if ( t['kind'] == "link" ):
          XALT_Stack.push("link_to_db()")
          xalt.link_to_db(rmapT, json.loads(t['value']))
          XALT_Stack.pop()
          lnkCnt += 1
        elif ( t['kind'] == "run" ):
          XALT_Stack.push("run_to_db()")
          xalt.run_to_db(rmapT, json.loads(t['value']))
          XALT_Stack.pop()
          runCnt += 1
        else:
          print("Error in xalt_syslog_to_db", file=sys.stderr)
        XALT_Stack.pop()
      except Exception as e:
        print(e, file=sys.stderr)
        badCnt += 1

      count += 1
      pbar.update(count)

    f.close()

  pbar.fini()

  t2 = time.time()
  rt = t2 - t1
  if (args.timer):
    print("Time: ", time.strftime("%T", time.gmtime(rt)))
  print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)

  leftover = args.leftover
  if (os.path.isfile(leftover)):
    os.rename(leftover, leftover + ".old")

  # if there is anything left in recordT file write it out to the leftover file.

  if (recordT):
    f = open(leftover, "w")
    for key in recordT:
      r = recordT[key]
      s = r.prt("XALT_LOGGING V=2", key)
      f.write(s)
    f.close()

if ( __name__ == '__main__'): main()
<commit_msg>improve progress bar printing<commit_after>#!/usr/bin/env python
# -*- python -*-
#
# Git Version: @git@

#-----------------------------------------------------------------------
# XALT: A tool to track the programs on a cluster.
# Copyright (C) 2013-2014 Robert McLay and Mark Fahey
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser  General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
#-----------------------------------------------------------------------

#  xalt_syslog takes the output found in xalt syslog file
#  provided with the --syslog_file=ans argument and puts it
#  into the database
#
#  One has to set up syslog to capture the XALT output
#  Example is provided in the xalt_syslog.conf file that can
#  be put in the /etc/rsyslog.d directory.
#
#

from __future__  import print_function
import os, sys, re, MySQLdb, json, time, argparse, base64, zlib, shlex

dirNm, execName = os.path.split(os.path.realpath(sys.argv[0]))
sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../libexec")))
sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../site")))

from XALTdb        import XALTdb
from xalt_site_pkg import translate
from xalt_util     import *
from xalt_global   import *
from progressBar   import ProgressBar
from Rmap_XALT     import Rmap

import inspect

def __LINE__():
    try:
        raise Exception
    except:
        return sys.exc_info()[2].tb_frame.f_back.f_lineno

def __FILE__():
    return inspect.currentframe().f_code.co_filename

#print ("file: '%s', line: %d" % (__FILE__(), __LINE__()), file=sys.stderr)

ConfigBaseNm = "xalt_db"
ConfigFn     = ConfigBaseNm + ".conf"
logger       = config_logger()

class CmdLineOptions(object):
  """ Command line Options class """

  def __init__(self):
    """ Empty Ctor """
    pass

  def execute(self):
    """ Specify command line arguments and parse the command line"""
    parser = argparse.ArgumentParser()
    parser.add_argument("--syslog",      dest='syslog',   action="store",      help="Location and name of syslog file")
    parser.add_argument("--syshost",     dest='syshost',  action="store",      default=".*",
                                                                               help="Store just records that come from syshost")
    parser.add_argument("--leftover_fn", dest='leftover', action="store",      default='leftover.log',
                                                                               help="Name of the leftover file")
    parser.add_argument("--timer",       dest='timer',    action="store_true", help="Time runtime")
    parser.add_argument("--reverseMapD", dest='rmapD',    action="store",      help="Path to the directory containing the json reverseMap")
    parser.add_argument("--dbname",      dest='dbname',   action="store",      default="xalt", help="Name of the database")
    parser.add_argument("--confFn",      dest='confFn',   action="store",      default="xalt_db.conf", help="Name of the database")
    args = parser.parse_args()
    return args

class Record(object):
  """
  This class holds the pieces of a single record as it comes in from
  syslog
  """
  def __init__(self, t):
    nblks          = int(t['nb'])
    self.__nblks   = nblks
    self.__kind    = t['kind']
    self.__syshost = t['syshost']
    self.__blkCnt  = 0

    blkA = []
    for i in range(nblks):
      blkA.append(False)

    self.__blkA    = blkA
    self.addBlk(t)

  def addBlk(self,t):
    idx               = int(t['idx'])
    self.__blkA[idx]  = t['value']
    self.__blkCnt    += 1

  def completed(self):
    return (self.__blkCnt >= self.__nblks)

  def value(self):
    return "".join(self.__blkA)

  def prt(self, prefix, key):
    sA    = []
    nblks = self.__nblks
    blkA  = self.__blkA

    sPA   = []
    sPA.append(prefix)
    sPA.append(" kind:")
    sPA.append(self.__kind)
    sPA.append(" syshost:")
    sPA.append(self.__syshost)
    sPA.append(" key:")
    sPA.append(key)
    sPA.append(" nb:")
    sPA.append(str(nblks))
    ss = "".join(sPA)

    for idx in range(nblks):
      value = blkA[idx]
      if (value):
        sA.append(ss)
        sA.append(" idx:")
        sA.append(str(idx))
        sA.append(" value:")
        sA.append(value)
        sA.append("\n")
    return "".join(sA)

def parseSyslogV1(s, clusterName):
  t = { 'kind' : None, 'value' : None, 'syshost' : None, 'version' : 1}

  idx = s.find("link:")
  if (idx == -1):
    idx = s.find("run:")
  if (idx == -1):
    return t, False

  array        = s[idx:].split(":")
  t['kind']    = array[0].strip()
  t['syshost'] = array[1].strip()
  t['value']   = base64.b64decode(array[2])

  if (clusterName != ".*" and clusterName != t['syshost']):
    return t, False

  return t, True

def parseSyslogV2(s, clusterName, recordT):
  t = { 'kind' : None, 'syshost' : None, 'value' : None, 'version' : 2}

  idx = s.find(" V:2 ")
  if (idx  == -1):
    return t, False

  # Strip off "XALT_LOGGING V:2" from string.
  s                      = s[idx+5:]

  # Setup parser
  lexer                  = shlex.shlex(s)
  lexer.whitespace_split = True
  lexer.whitespace       = ' :'

  # Pick off two values at a time.
  try:
    while True:
      key    = next(lexer)
      value  = next(lexer)
      t[key] = value
  except StopIteration as e:
    pass

  # get the key from the input, then place an entry in the *recordT* table.
  # or just add the block to the current record.
  key = t['key']
  r    = recordT.get(key, None)
  if (r):
    r.addBlk(t)
  else:
    r  = Record(t)
    recordT[key] = r

  if (clusterName != ".*" and clusterName != t['syshost']):
    return t, False

  # If the block is completed then grap the value, remove the entry from *recordT*
  # and return a completed table.
  if (r.completed()):

    rv   = r.value()
    b64v = base64.b64decode(rv)
    vv   = zlib.decompress(b64v)

    t['value'] = vv
    recordT.pop(key)
    return t, True

  # Entry is not complete.
  return t, False

def parseSyslog(s, clusterName, recordT):
  if ("XALT_LOGGING" in s) and ("V:2" in s):
    return parseSyslogV2(s, clusterName, recordT)
  return parseSyslogV1(s, clusterName)

def main():
  """
  read from syslog file into XALT db.
  """

  sA = []
  sA.append("CommandLine:")
  for v in sys.argv:
    sA.append('"'+v+'"')
  XALT_Stack.push(" ".join(sA))

  args       = CmdLineOptions().execute()
  xalt       = XALTdb(args.confFn)
  syslogFile = args.syslog

  # should add a check if file exists
  num    = int(capture("cat "+syslogFile+" | wc -l"))
  icnt   = 0

  t1     = time.time()

  rmapT  = Rmap(args.rmapD).reverseMapT()

  lnkCnt = 0
  runCnt = 0
  badCnt = 0
  count  = 0

  recordT = {}

  if (num == 0):
    return

  pbar   = ProgressBar(maxVal=num)

  fnA = [ args.leftover, syslogFile ]

  for fn in fnA:
    if (not os.path.isfile(fn)):
      continue

    f=open(fn, 'r')
    for line in f:
      count += 1
      pbar.update(count)
      if (not ("XALT_LOGGING" in line)):
        continue
      t, done = parseSyslog(line, args.syshost, recordT)
      if (not done):
        continue

      try:
        XALT_Stack.push("XALT_LOGGING: " + t['kind'] + " " + t['syshost'])

        if ( t['kind'] == "link" ):
          XALT_Stack.push("link_to_db()")
          xalt.link_to_db(rmapT, json.loads(t['value']))
          XALT_Stack.pop()
          lnkCnt += 1
        elif ( t['kind'] == "run" ):
          XALT_Stack.push("run_to_db()")
          xalt.run_to_db(rmapT, json.loads(t['value']))
          XALT_Stack.pop()
          runCnt += 1
        else:
          print("Error in xalt_syslog_to_db", file=sys.stderr)
        XALT_Stack.pop()
      except Exception as e:
        print(e, file=sys.stderr)
        badCnt += 1

    f.close()

  pbar.fini()

  t2 = time.time()
  rt = t2 - t1
  if (args.timer):
    print("Time: ", time.strftime("%T", time.gmtime(rt)))
  print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt, ", badCnt: ", badCnt)

  leftover = args.leftover
  if (os.path.isfile(leftover)):
    os.rename(leftover, leftover + ".old")

  # if there is anything left in recordT file write it out to the leftover file.

  if (recordT):
    f = open(leftover, "w")
    for key in recordT:
      r = recordT[key]
      s = r.prt("XALT_LOGGING V=2", key)
      f.write(s)
    f.close()

if ( __name__ == '__main__'): main()
<|endoftext|>


<commit_before><commit_msg>Fix canceling appoitnment status<commit_after><|endoftext|>


<commit_before><commit_msg>reformatting code<commit_after><|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.db import models
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.utils.safestring import mark_safe

# User Actions Constants
USER_ACTION_CREATE = 1
USER_ACTION_EDIT = 2
USER_ACTION_DELETE = 3
USER_ACTION_IMPORT = 4
USER_ACTION_CHOICES = (
    (USER_ACTION_CREATE, 'created'),
    (USER_ACTION_EDIT, 'modified'),
    (USER_ACTION_DELETE, 'deleted'),
    (USER_ACTION_IMPORT, 'imported'),
)

class UserActionManager(models.Manager):
    """
    Manager for UserAction model.
    """

    def log_action(self, user, obj, action, message):
        self.model.objects.create(content_type=ContentType.objects.get_for_model(
            obj), object_id=obj.id, user=user, action=action, message=message)

    def log_bulk_action(self, user, obj_type, action, message):
        self.model.objects.create(content_type=ContentType.objects.get_for_model(
            content_type), user=user, action=action, message=message)

    def log_create(self, user, obj, message):
        self.log_action(user, obj, USER_ACTION_CREATE, message)

    def log_edit(self, user, obj, message):
        self.log_action(user, obj, USER_ACTION_EDIT, message)

    def log_delete(self, user, obj, message):
        self.log_action(user, obj, USER_ACTION_DELETE, message)

    def log_import(self, user, obj_type, message):
        self.log_bulk_action(user, obj_type, USER_ACTION_IMPORT, message)

class UserAction(models.Model):
    """
    A record of a user action (add/edit/delete/import).
    """
    time = models.DateTimeField(auto_now_add=True, editable=False)
    user = models.ForeignKey(
        User, related_name='actions', on_delete=models.CASCADE)
    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
    object_id = models.PositiveIntegerField(blank=True, null=True)
    action = models.PositiveSmallIntegerField(choices=USER_ACTION_CHOICES)
    message = models.TextField(blank=True)

    objects = UserActionManager()

    class Meta:
        ordering = ['-time']

    def get_icon(self):
        """
        Return an HTML element based on the user action.
        """
        if self.action in [USER_ACTION_CREATE, USER_ACTION_IMPORT]:
            return mark_safe('')

        if self.action == USER_ACTION_EDIT:
            return mark_safe('')

        if self.action == USER_ACTION_DELETE:
            return mark_safe('')

        return ''

    def __str__(self):
        if self.message:
            return '{} {}'.format(self.user, self.message)

        return '{} {} {}'.format(self.user, self.get_action_display(), self.content_type)
<commit_msg>One more fix.<commit_after>from __future__ import unicode_literals

from django.db import models
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.utils.safestring import mark_safe

# User Actions Constants
USER_ACTION_CREATE = 1
USER_ACTION_EDIT = 2
USER_ACTION_DELETE = 3
USER_ACTION_IMPORT = 4
USER_ACTION_CHOICES = (
    (USER_ACTION_CREATE, 'created'),
    (USER_ACTION_EDIT, 'modified'),
    (USER_ACTION_DELETE, 'deleted'),
    (USER_ACTION_IMPORT, 'imported'),
)

class UserActionManager(models.Manager):
    """
    Manager for UserAction model.
    """

    def log_action(self, user, obj, action, message):
        self.model.objects.create(content_type=ContentType.objects.get_for_model(
            obj), object_id=obj.id, user=user, action=action, message=message)

    def log_bulk_action(self, user, obj_type, action, message):
        self.model.objects.create(content_type=ContentType.objects.get_for_model(
            obj_type), user=user, action=action, message=message)

    def log_create(self, user, obj, message):
        self.log_action(user, obj, USER_ACTION_CREATE, message)

    def log_edit(self, user, obj, message):
        self.log_action(user, obj, USER_ACTION_EDIT, message)

    def log_delete(self, user, obj, message):
        self.log_action(user, obj, USER_ACTION_DELETE, message)

    def log_import(self, user, obj_type, message):
        self.log_bulk_action(user, obj_type, USER_ACTION_IMPORT, message)

class UserAction(models.Model):
    """
    A record of a user action (add/edit/delete/import).
    """
    time = models.DateTimeField(auto_now_add=True, editable=False)
    user = models.ForeignKey(
        User, related_name='actions', on_delete=models.CASCADE)
    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
    object_id = models.PositiveIntegerField(blank=True, null=True)
    action = models.PositiveSmallIntegerField(choices=USER_ACTION_CHOICES)
    message = models.TextField(blank=True)

    objects = UserActionManager()

    class Meta:
        ordering = ['-time']

    def get_icon(self):
        """
        Return an HTML element based on the user action.
        """
        if self.action in [USER_ACTION_CREATE, USER_ACTION_IMPORT]:
            return mark_safe('')

        if self.action == USER_ACTION_EDIT:
            return mark_safe('')

        if self.action == USER_ACTION_DELETE:
            return mark_safe('')

        return ''

    def __str__(self):
        if self.message:
            return '{} {}'.format(self.user, self.message)

        return '{} {} {}'.format(self.user, self.get_action_display(), self.content_type)
<|endoftext|>


<commit_before><commit_msg>Added the format converter for using the old unsupervised model<commit_after><|endoftext|>


<commit_before><commit_msg>Main with origin<commit_after><|endoftext|>


<commit_before>import numpy as np

def LatLonAlt_from_XYZ(xyz):

        # see wikipedia geodetic_datum and Datum transformations of
        # GPS positions PDF in docs folder
        gps_b = 6356752.31424518
        gps_a = 6378137
        e_squared = 6.69437999014e-3
        e_prime_squared = 6.73949674228e-3
        gps_p = np.sqrt(xyz[0]**2 + xyz[1]**2)
        gps_theta = np.arctan2(xyz[2] * gps_a, gps_p * gps_b)
        latitude = np.arctan2(xyz[2] + e_prime_squared * gps_b *
                              np.sin(gps_theta)**3, gps_p - e_squared * gps_a *
                              np.cos(gps_theta)**3)

        longitude = np.arctan2(xyz[1], xyz[0])
        gps_N = gps_a / np.sqrt(1 - e_squared * np.sin(latitude)**2)
        altitude = ((gps_p / np.cos(latitude)) - gps_N)
        return latitude, longitude, altitude

def XYZ_from_LatLonAlt(latitude, longitude, altitude):

        # see wikipedia geodetic_datum and Datum transformations of
        # GPS positions PDF in docs folder
        gps_b = 6356752.31424518
        gps_a = 6378137
        e_squared = 6.69437999014e-3
        e_prime_squared = 6.73949674228e-3
        gps_N = gps_a / np.sqrt(1 - e_squared *
                                np.sin(latitude)**2)
        xyz = np.zeros(3)
        xyz[0] = ((gps_N + altitude) * np.cos(latitude) * np.cos(longitude))
        xyz[1] = ((gps_N + altitude) * np.cos(latitude) * np.sin(longitude))
        xyz[2] = ((gps_b**2 / gps_a**2 * gps_N + altitude) * np.sin(latitude))

        return xyz
<commit_msg>util cleanup<commit_after>import numpy as np

# parameters for transforming between xyz & lat/lon/alt
gps_b = 6356752.31424518
gps_a = 6378137
e_squared = 6.69437999014e-3
e_prime_squared = 6.73949674228e-3

def LatLonAlt_from_XYZ(xyz):

        # see wikipedia geodetic_datum and Datum transformations of
        # GPS positions PDF in docs folder
        gps_p = np.sqrt(xyz[0]**2 + xyz[1]**2)
        gps_theta = np.arctan2(xyz[2] * gps_a, gps_p * gps_b)
        latitude = np.arctan2(xyz[2] + e_prime_squared * gps_b *
                              np.sin(gps_theta)**3, gps_p - e_squared * gps_a *
                              np.cos(gps_theta)**3)

        longitude = np.arctan2(xyz[1], xyz[0])
        gps_N = gps_a / np.sqrt(1 - e_squared * np.sin(latitude)**2)
        altitude = ((gps_p / np.cos(latitude)) - gps_N)
        return latitude, longitude, altitude

def XYZ_from_LatLonAlt(latitude, longitude, altitude):

        # see wikipedia geodetic_datum and Datum transformations of
        # GPS positions PDF in docs folder
        gps_N = gps_a / np.sqrt(1 - e_squared * np.sin(latitude)**2)
        xyz = np.zeros(3)
        xyz[0] = ((gps_N + altitude) * np.cos(latitude) * np.cos(longitude))
        xyz[1] = ((gps_N + altitude) * np.cos(latitude) * np.sin(longitude))
        xyz[2] = ((gps_b**2 / gps_a**2 * gps_N + altitude) * np.sin(latitude))

        return xyz
<|endoftext|>


<commit_before>from numpy import float
#http://code.activestate.com/recipes/52219-associating-multiple-values-with-each-key-in-a-dic/
d = {}                                     #creates an empty dictionary
counts = {}                                #empty dictionary thaat will hold the value counts of dictionary d
with open ("Yeast.txt") as f:              #while the text is open
    for line in f:                         #iterate through each line
        (key,val) = line.split()           #split each lines so that its key then value
        d.setdefault(key,[]).append(val)   #method that adds multiple vals to a key

for x in d:                                #for each value in dictionary
    print("{} : {}".format(x,d[x]))        #prints key and all values attached to it

print("test\n")
print(d.get('1369'))                       #prints values ['815', '859', '941'] for key 1369

for key, val, in d.items():                #prints the key and the number of values there
    print(key, len(filter(bool,val)))      #len(filter(bool,val))) necessary methods for counting the values present in the dictionary
    NumOfItems= len(filter(bool,val))
    counts[key]= NumOfItems                #places the number of items contained to a value into the "counts" dictionary.
print(len(d))                              #prints the length of dictionary

for x in counts:                           #for loop that prints out the gene and how many genes it connects to, iterating through the "counts" dictionary
    print("Gene {} connects to {} genes".format(x,counts[x]))
print(counts.get('1369'))                  #test to see if I can access gene number 1369 and see how many other genes it connects to

                                           #here the user inputs what gene it wants to access:
answer = input('Which gene would you like to know about?\n')
answer=str(answer)                         #convert the answer to a string to fix problem of accessing dictionary with an int value without single quotes (' ')
print("{} has {} genes connect to it".format(answer,counts.get(answer)))
print("{} has the following genes connect to it: {}".format(answer,d.get(answer)))
                                           #prints information of gene - what genes connects to it and how many of them

entropy = counts.get(answer)               #code segment for calculating entropy value of that gene
print(entropy)
entropy= 1/float(entropy)                  #float type necessary unless you want python to tell you 0 entropy calculated
print("The entropy of gene {} is {} because {} genes connect to it".format(answer,entropy, counts.get(answer)))

totalentropy = 0                           #total entropy counter set to 0
for x in counts:                           #for loop that accesses count values of each value in counts dictionary and adds them to totalentropy
    entropy=counts[x]
    totalentropy+=entropy
print(totalentropy)                        #test to see what denominator is
totalentropy=1/float(totalentropy)         #total entropy calculation
print("The total entropy of the entire system is {}".format(totalentropy))<commit_msg>Massive program structure overhaul. Methods, menus, etc.<commit_after>import sys                             #Modules
from numpy import float

d = {}                                 #Data structures
counts = {}

def Quit():                            #Kill program >:)
    sys.exit()

def PrintNetwork(): #prints all the genes and what they connect to
    for x in d:  # for each value in dictionary
        print("{} : {}".format(x, d[x]))  # prints key and all values attached to it
    Menu()

def PrintSet(): #print an individual key
    key = input("Which gene would you like to see all of its connections to?\n")
    key = str(key)
    print(d.get(key))
    Menu()

def DlenPrint(): #print length of the dictionary
    print(len(d))
    Menu()

def PrintSystemsCounts():
    for x in counts:  # for loop that prints out the gene and how many genes it connects to, iterating through the "counts" dictionary
        print("Gene {} connects to {} genes".format(x, counts[x]))
    Menu()

def PrintSetsCounts():
    key=input("Which gene would you like to see the number of connections to?\n")
    key = str(key)
    print(counts.get(key))
    Menu()

def PrintGeneInformation():
    key = input("Which gene would you like more information about?\n")
    key = str(key)
    print("{} has {} genes connect to it".format(key, counts.get(key)))
    print("{} has the following genes connect to it: {}".format(key, d.get(key)))
    Menu()

def PrintIndividualEntropy():
    key=input("Which gene's entropy would you like to know?\n")
    key = str(key)
    entropy = counts.get(key)  # code segment for calculating entropy value of that gene
    entropy = 1 / float(entropy)  # float type necessary unless you want python to tell you 0 entropy calculated
    print("The entropy of gene {} is {} because {} genes connect to it".format(key, entropy, counts.get(key)))
    Menu()

def PrintSystemEntropy():
    totalentropy = 0  # total entropy counter set to 0
    for x in counts:  # for loop that accesses count values of each value in counts dictionary and adds them to totalentropy
        entropy = counts[x]
        totalentropy += entropy
    print("1/{}".format(totalentropy))  # test to see what denominator is
    totalentropy = 1 / float(totalentropy)  # total entropy calculation
    print("The total entropy of the entire system is {}".format(totalentropy))
    Menu()

def Menu():            #Main Menu
    menu = {1: PrintNetwork, 2: PrintSet, 3: DlenPrint, 4: PrintSystemsCounts, 5: PrintSetsCounts,
            6: PrintGeneInformation, 7: PrintIndividualEntropy, 8: PrintSystemEntropy, 0: Quit}
    num = input()
    menu[num]()

def main():
    with open("Yeast.txt") as f:  # while the text is open
        for line in f:  # iterate through each line
            (key, val) = line.split()  # split each lines so that its key then value
            d.setdefault(key, []).append(val)  # method that adds multiple vals to a key
    for key, val, in d.items():  # prints the key and the number of values there
        NumOfItems = len(filter(bool, val))
        counts[key] = NumOfItems
    Menu()

if __name__ == "__main__":
    # execute only if run as a script
    main()<|endoftext|>


<commit_before>#! /usr/bin/env python

'''Fetch a bunch of feeds in quick succession'''

# Based on the example at:
# http://pycurl.cvs.sourceforge.net/pycurl/pycurl/examples/retriever-multi.py?view=markup

import pyev
import pycurl					# We need to talk to curl
import signal
import socket
import logging					# Early integration of logging is good
import urlparse
from cStringIO import StringIO	# To fake file descriptors into strings

# Our logger
logger = logging.getLogger('downpour')
# Signals that are equivalent of stopping
SIGSTOP = (signal.SIGPIPE, signal.SIGINT, signal.SIGTERM)
# The loop we'll be using for everything
loop = pyev.default_loop()

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.FileHandler('downpour.log', 'w+')
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logger.addHandler(handler)

class Request(object):
	retryMax   = 3
	retryBase  = 2
	retryScale = 1

	def __init__(self, url):
		self.url       = url
		self.sock      = None
		self.fetcher   = None
		self.ioWatcher = pyev.Io(0, pyev.EV_READ | pyev.EV_WRITE, loop, self.io)
		# For backing off for retrying:
		# scale * (base ** retries)
		self.retries   = 0

	def success(self, c, content):
		pass

	def error(self, c, errno, errmsg):
		pass

	def done(self):
		try:
			self.sock.close()
		except AttributeError as e:
			logger.error(repr(e))

	#################
	# curl callbacks
	#################
	def socket(self, family, socktype, protocol):
		'''Pycurl wants a socket, so make one, watch it and return it.'''
		logger.debug('Watching socket for %s' % self.url)
		if self.sock:
			self.sock.close()
		self.sock = socket.socket(family, socktype, protocol)
		self.ioWatcher.stop()
		self.ioWatcher.set(self.sock, pyev.EV_READ | pyev.EV_WRITE)
		self.ioWatcher.start()
		return self.sock

	#################
	# libev callbacks
	#################
	def io(self, watcher, revents):
		#logger.debug('IO Event')
		# Temporarily stop our watcher, call, then restart
		#self.ioWatcher.stop()
		self.fetcher.socketAction(self.sock.fileno())
		#self.ioWatcher.start()

class Fetcher(object):
	def __init__(self, poolSize = 10):
		# Go ahead and make a curl multi handle
		self.multi = pycurl.CurlMulti()
		self.multi.setopt(pycurl.M_TIMERFUNCTION, self.curlTimer)
		self.multi.setopt(pycurl.M_SOCKETFUNCTION, self.curlSocket)
		# Make a sharing option for DNS stuff
		self.share = pycurl.CurlShare()
		self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
		# A queue of our requests, and the number of requests in flight
		self.queue = []
		self.retryQueue = []
		self.num = 0
		# Now instantiate a pool of easy handles
		self.pool = []
		for i in range(poolSize):
			c = pycurl.Curl()
			# It will need a file to write to
			c.fp = None
			# Set some options
			c.setopt(pycurl.CONNECTTIMEOUT, 15)
			c.setopt(pycurl.FOLLOWLOCATION, 1)
			c.setopt(pycurl.SHARE, self.share)
			c.setopt(pycurl.FRESH_CONNECT, 1)
			c.setopt(pycurl.FORBID_REUSE, 1)
			c.setopt(pycurl.MAXREDIRS, 5)
			c.setopt(pycurl.TIMEOUT, 15)
			c.setopt(pycurl.NOSIGNAL, 1)
			# Now add it to the pool
			self.pool.append(c)
		self.multi.handles = self.pool[:]
		# Now listen for certain events
		self.signalWatchers = [pyev.Signal(sig, loop, self.signal) for sig in SIGSTOP]
		self.timerWatcher = pyev.Timer(1000.0, 0.0, loop, self.timer)
		#self.timeoutTimer = pyev.Timer(60.0, 60.0, loop, self.checkTimeouts)

	def __del__(self):
		'''Clean up the pool of curl handlers we allocated'''
		logger.info('Cleaning up')
		for c in self.pool:
			if c.fp is not None:
				c.fp.close()
				c.fp = None
			c.close()
		self.multi.close()

	#################
	# Inheritance Interface
	#################
	def __len__(self):
		return self.num + len(self.queue)

	def extend(self, requests):
		self.queue.extend(requests)
		self.serveNext()

	def pop(self):
		'''Get the next request'''
		return self.queue.pop(0)

	def push(self, r):
		'''Queue a request'''
		self.queue.append(r)
		self.serveNext()

	def onSuccess(self, c):
		pass

	def onError(self, c):
		pass

	def onDone(self, c):
		pass

	#################
	# Our interface
	#################
	def start(self):
		logger.info('Starting fetcher...')
		for w in self.signalWatchers:
			w.start()
		self.serveNext()
		loop.start()

	def stop(self):
		logger.info('Stopping fetcher...')
		for w in self.signalWatchers:
			w.stop()
		loop.stop()

	#################
	# libev callbacks
	#################
	def signal(self, watcher, revents):
		logger.info('Signal caught')
		self.stop()

	def timer(self, watcher, revents):
		logger.info('Timer fired')
		self.perform()

	def retry(self, watcher, revents):
		try:
			c = self.retryQueue.pop()
			logger.info('Retrying %s' % c.request.url)
			self.serve(c, c.request)
		except ValueError:
			logger.warn('Tried popping off empty retryQueue')

	#################
	# handle complete
	#################
	def success(self, c):
		content = c.fp.getvalue()
		logger.debug('Success %s => %s...' % (c.request.url, content[0:100]))
		c.request.success(c, c.fp.getvalue())
		self.onSuccess(c)
		self.done(c)

	def error(self, c, errno, errmsg):
		if c.request.retries <= c.request.retryMax:
			# Append it to the list to retry
			r = c.request
			t = r.retryScale * (r.retryBase ** r.retries)
			c.request.retries += 1
			logger.debug('Retrying %s in %is (%s)' % (r.url, t, errmsg))
			self.retryQueue.append(c)
			self.multi.remove_handle(c)
			c.timer = pyev.Timer(t, 0, loop, self.retry)
			c.timer.start()
		else:
			logger.debug('Error %s => (%i) %s' % (c.request.url, errno, errmsg))
			c.request.error(c, errno, errmsg)
			self.onError(c)
			self.done(c)

	def done(self, c):
		logger.debug('Done with %s' % c.request.url)
		self.onDone(c)
		c.fp.close()
		c.fp = None
		c.request.done()
		self.pool.append(c)
		self.multi.remove_handle(c)
		self.serveNext()

	#################
	# curl callbacks
	#################
	def curlTimer(self, timeout):
		t = timeout / 1000.0
		if t < self.timerWatcher.remaining():
			logger.debug('Resetting timer to fire in %fs' % t)
			self.timerWatcher.stop()
			self.timerWatcher.set(t, 1.0)
			self.timerWatcher.start()

	def curlSocket(self, sock, action, userp, socketp):
		pass

	def perform(self):
		try:
			return self.multi.perform()
		except OSError as e:
			logger.error(repr(e))

	def socketAction(self, sock):
		try:
			ret, num = self.multi.perform()
			if num < self.num:
				logger.info('%i < %i => one or more handles has completed' % (num, self.num))
				self.infoRead()
		except socket.error as e:
			logger.error('%s' % repr(e))
		except OSError as e:
			logger.error('%s' % repr(e))

	def infoRead(self):
		#logger.debug('Checking with curl for finished handlers')
		num, ok, err = self.multi.info_read()
		logger.debug('infoRead : %i <=> %i' % (num, self.num))
		self.num = num
		for c in ok:
			# Handle successulf
			self.success(c)
		for c, errno, errmsg in err:
			# Handle failed
			self.error(c, errno, errmsg)

	def serveNext(self):
		while len(self.queue) and len(self.pool):
			# While there are requests to service, and handles to service them
			logger.debug('Queue : %i\tPool: %i' % (len(self.queue), len(self.pool)))
			# Look for the next request
			r = self.pop()
			if r == None:
				# pop() can return None to signal there are no more
				break
			# Get a handle, and attach a request and fp to it
			c = self.pool.pop()
			self.serve(c, r)
		self.perform()

	def serve(self, c, r):
		# The request should know who the fetcher is
		r.fetcher = self
		c.request = r
		c.fp = StringIO()
		# Set some options
		c.setopt(pycurl.URL, c.request.url)
		c.setopt(pycurl.HTTPHEADER, ['Host: %s' % urlparse.urlparse(r.url).hostname])
		c.setopt(pycurl.OPENSOCKETFUNCTION, c.request.socket)
		c.setopt(pycurl.WRITEFUNCTION, c.fp.write)
		# Indicate that we have one more in flight
		self.num += 1
		self.multi.add_handle(c)
		self.multi.socket_action(pycurl.SOCKET_TIMEOUT, 0)
		self.perform()

if __name__ == '__main__':
	handler   = logging.StreamHandler()
	handler.setLevel(logging.DEBUG)
	handler.setFormatter(formatter)
	logger.addHandler(handler)
	logger.setLevel(logging.DEBUG)
	f = file('urls.txt')
	urls = f.read().strip().split()
	f.close()
	f = Fetcher(20)
	f.extend([Request(url) for url in urls])
	f.start()
<commit_msg>Trying to catch this weird EAGAIN (OSError 11) bug.<commit_after>#! /usr/bin/env python

'''Fetch a bunch of feeds in quick succession'''

# Based on the example at:
# http://pycurl.cvs.sourceforge.net/pycurl/pycurl/examples/retriever-multi.py?view=markup

import pyev
import pycurl					# We need to talk to curl
import signal
import socket
import logging					# Early integration of logging is good
import urlparse
from cStringIO import StringIO	# To fake file descriptors into strings

# Our logger
logger = logging.getLogger('downpour')
# Signals that are equivalent of stopping
SIGSTOP = (signal.SIGPIPE, signal.SIGINT, signal.SIGTERM)
# The loop we'll be using for everything
loop = pyev.default_loop()

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.FileHandler('downpour.log', 'w+')
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logger.addHandler(handler)

class Request(object):
	retryMax   = 3
	retryBase  = 2
	retryScale = 1

	def __init__(self, url):
		self.url       = url
		self.sock      = None
		self.fetcher   = None
		self.ioWatcher = pyev.Io(0, pyev.EV_READ | pyev.EV_WRITE, loop, self.io)
		# For backing off for retrying:
		# scale * (base ** retries)
		self.retries   = 0

	def success(self, c, content):
		pass

	def error(self, c, errno, errmsg):
		pass

	def done(self):
		try:
			self.sock.close()
		except AttributeError as e:
			logger.error(repr(e))

	#################
	# curl callbacks
	#################
	def socket(self, family, socktype, protocol):
		'''Pycurl wants a socket, so make one, watch it and return it.'''
		logger.debug('Watching socket for %s' % self.url)
		if self.sock:
			self.sock.close()
		self.sock = socket.socket(family, socktype, protocol)
		self.ioWatcher.stop()
		self.ioWatcher.set(self.sock, pyev.EV_READ | pyev.EV_WRITE)
		self.ioWatcher.start()
		return self.sock

	#################
	# libev callbacks
	#################
	def io(self, watcher, revents):
		#logger.debug('IO Event')
		# Temporarily stop our watcher, call, then restart
		#self.ioWatcher.stop()
		self.fetcher.socketAction(self.sock.fileno())
		#self.ioWatcher.start()

class Fetcher(object):
	def __init__(self, poolSize = 10):
		# Go ahead and make a curl multi handle
		self.multi = pycurl.CurlMulti()
		self.multi.setopt(pycurl.M_TIMERFUNCTION, self.curlTimer)
		self.multi.setopt(pycurl.M_SOCKETFUNCTION, self.curlSocket)
		# Make a sharing option for DNS stuff
		self.share = pycurl.CurlShare()
		self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
		# A queue of our requests, and the number of requests in flight
		self.queue = []
		self.retryQueue = []
		self.num = 0
		# Now instantiate a pool of easy handles
		self.pool = []
		for i in range(poolSize):
			c = pycurl.Curl()
			# It will need a file to write to
			c.fp = None
			# Set some options
			c.setopt(pycurl.CONNECTTIMEOUT, 15)
			c.setopt(pycurl.FOLLOWLOCATION, 1)
			c.setopt(pycurl.SHARE, self.share)
			c.setopt(pycurl.FRESH_CONNECT, 1)
			c.setopt(pycurl.FORBID_REUSE, 1)
			c.setopt(pycurl.MAXREDIRS, 5)
			c.setopt(pycurl.TIMEOUT, 15)
			c.setopt(pycurl.NOSIGNAL, 1)
			# Now add it to the pool
			self.pool.append(c)
		self.multi.handles = self.pool[:]
		# Now listen for certain events
		self.signalWatchers = [pyev.Signal(sig, loop, self.signal) for sig in SIGSTOP]
		self.timerWatcher = pyev.Timer(1000.0, 0.0, loop, self.timer)
		#self.timeoutTimer = pyev.Timer(60.0, 60.0, loop, self.checkTimeouts)

	def __del__(self):
		'''Clean up the pool of curl handlers we allocated'''
		logger.info('Cleaning up')
		for c in self.pool:
			if c.fp is not None:
				c.fp.close()
				c.fp = None
			c.close()
		self.multi.close()

	#################
	# Inheritance Interface
	#################
	def __len__(self):
		return self.num + len(self.queue)

	def extend(self, requests):
		self.queue.extend(requests)
		self.serveNext()

	def pop(self):
		'''Get the next request'''
		return self.queue.pop(0)

	def push(self, r):
		'''Queue a request'''
		self.queue.append(r)
		self.serveNext()

	def onSuccess(self, c):
		pass

	def onError(self, c):
		pass

	def onDone(self, c):
		pass

	#################
	# Our interface
	#################
	def start(self):
		logger.info('Starting fetcher...')
		for w in self.signalWatchers:
			w.start()
		self.serveNext()
		loop.start()

	def stop(self):
		logger.info('Stopping fetcher...')
		for w in self.signalWatchers:
			w.stop()
		loop.stop()

	#################
	# libev callbacks
	#################
	def signal(self, watcher, revents):
		logger.info('Signal caught')
		self.stop()

	def timer(self, watcher, revents):
		logger.info('Timer fired')
		self.perform()

	def retry(self, watcher, revents):
		try:
			c = self.retryQueue.pop()
			logger.info('Retrying %s' % c.request.url)
			self.serve(c, c.request)
		except ValueError:
			logger.warn('Tried popping off empty retryQueue')

	#################
	# handle complete
	#################
	def success(self, c):
		content = c.fp.getvalue()
		logger.debug('Success %s => %s...' % (c.request.url, content[0:100]))
		c.request.success(c, c.fp.getvalue())
		self.onSuccess(c)
		self.done(c)

	def error(self, c, errno, errmsg):
		if c.request.retries <= c.request.retryMax:
			# Append it to the list to retry
			r = c.request
			t = r.retryScale * (r.retryBase ** r.retries)
			c.request.retries += 1
			logger.debug('Retrying %s in %is (%s)' % (r.url, t, errmsg))
			self.retryQueue.append(c)
			self.multi.remove_handle(c)
			c.timer = pyev.Timer(t, 0, loop, self.retry)
			c.timer.start()
		else:
			logger.debug('Error %s => (%i) %s' % (c.request.url, errno, errmsg))
			c.request.error(c, errno, errmsg)
			self.onError(c)
			self.done(c)

	def done(self, c):
		logger.debug('Done with %s' % c.request.url)
		self.onDone(c)
		c.fp.close()
		c.fp = None
		c.request.done()
		self.pool.append(c)
		self.multi.remove_handle(c)
		self.serveNext()

	#################
	# curl callbacks
	#################
	def curlTimer(self, timeout):
		t = timeout / 1000.0
		if t < self.timerWatcher.remaining():
			logger.debug('Resetting timer to fire in %fs' % t)
			self.timerWatcher.stop()
			self.timerWatcher.set(t, 1.0)
			self.timerWatcher.start()

	def curlSocket(self, sock, action, userp, socketp):
		pass

	def perform(self):
		try:
			return self.multi.perform()
		except OSError as e:
			logger.error(repr(e))

	def socketAction(self, sock):
		try:
			ret, num = self.multi.perform()
			if num < self.num:
				logger.info('%i < %i => one or more handles has completed' % (num, self.num))
				self.infoRead()
		except socket.error as e:
			logger.error('%s' % repr(e))
		except OSError as e:
			logger.error('%s' % repr(e))

	def infoRead(self):
		#logger.debug('Checking with curl for finished handlers')
		try:
			num, ok, err = self.multi.info_read()
		except OSError as e:
			logger.error('%s' % repr(e))
			return
		logger.debug('infoRead : %i <=> %i' % (num, self.num))
		self.num = num
		for c in ok:
			# Handle successulf
			self.success(c)
		for c, errno, errmsg in err:
			# Handle failed
			self.error(c, errno, errmsg)

	def serveNext(self):
		while len(self.queue) and len(self.pool):
			# While there are requests to service, and handles to service them
			logger.debug('Queue : %i\tPool: %i' % (len(self.queue), len(self.pool)))
			# Look for the next request
			r = self.pop()
			if r == None:
				# pop() can return None to signal there are no more
				break
			# Get a handle, and attach a request and fp to it
			c = self.pool.pop()
			self.serve(c, r)
		self.perform()

	def serve(self, c, r):
		# The request should know who the fetcher is
		r.fetcher = self
		c.request = r
		c.fp = StringIO()
		# Set some options
		c.setopt(pycurl.URL, c.request.url)
		c.setopt(pycurl.HTTPHEADER, ['Host: %s' % urlparse.urlparse(r.url).hostname])
		c.setopt(pycurl.OPENSOCKETFUNCTION, c.request.socket)
		c.setopt(pycurl.WRITEFUNCTION, c.fp.write)
		# Indicate that we have one more in flight
		self.num += 1
		self.multi.add_handle(c)
		self.multi.socket_action(pycurl.SOCKET_TIMEOUT, 0)
		self.perform()

if __name__ == '__main__':
	handler   = logging.StreamHandler()
	handler.setLevel(logging.DEBUG)
	handler.setFormatter(formatter)
	logger.addHandler(handler)
	logger.setLevel(logging.DEBUG)
	f = file('urls.txt')
	urls = f.read().strip().split()
	f.close()
	f = Fetcher(20)
	f.extend([Request(url) for url in urls])
	f.start()
<|endoftext|>


<commit_before>""" Summary:            Friends is question #3 from the Reelio test.

    P-Version:          QA'd on Python 3.4.1

    Knowledge domains:  Relationship networks, performance profiling
   						(e.g. import cProfile|profile, profile.run('main()' -or-
   						python -m cProfile [-o output_file] [-s sort_order] myscript.py)

    Candidates Eval'd:	Data model, map/reduce (functools module), OOD

    Considerations:		Short on time so trading off succinct requirement for functional requirement.
                        Intended usage of the system implies need for conceptual parallelization being designed in.  As implemented, there's still some blocking, but it's better than a monolithic container object.  With this design, full logical & Physical parallization is easier done later.

    Decision History:   Going OO, but not creating multi-threading per the alpha containers.  Also, getting user file data persistent in memory would be key for a real performance test.
                        As-is the object routing per alpha prefix is effectively an unbalanced/simplistic routing option that allows for parallelization, and implies built-in option for ranges with more granularity.
                        I naively applied the data model used in the data dictionary per reading I did on Hadoop a while back - could use a 2nd pass.

    Data Format:		Friend records --  {'Bob Brown': ['Sam Smith', 'Joe Shepard']}
                        Alpha lookup for object routing --  {0: 'A', 1: 'B', 2: 'C', 3: 'D', 4: 'E', 5: 'F', 6: 'G', 7: 'H', 8: 'I', 9: 'J', 10: 'K', 11: 'L', 12: 'M', 13: 'N', 14: 'O', 15: 'P', 16: 'Q', 17: 'R', 18: 'S', 19: 'T', 20: 'U', 21: 'V', 22: 'W', 23: 'X', 24: 'Y', 25: 'Z'}

    Script Usage:       python Friends.py <filename> <name1> <name2>

    Note:               There is gratuitous overdesign here but it was more natural to me than code shaving the test so I went for the exercise as relevant to production environments.
"""

import itertools
import sys

__author__ = "Michael Rais"
__version__ = "0.1-alpha"
__maintainer__ = "Michael Rais"
__email__ = "mrais@inbox.com"
__status__ = "Development"

class FriendMap():

    def __init__(self):
        self.range, self.data = '', {}

    def get_friends(self, source):
        source = source.title()  # Titlecase everything
        return self.data.get(source)

    def add_friend_map(self, source, target):
        source, target = source.title(), target.title()  # Titlecase everything
        # If source already exists, check if target associated or not
        sourceGet = self.data.get(source)
        if sourceGet:
            # Target not already associated - if it is do nothing.
            if target not in sourceGet:
                oldValue = sourceGet
                oldValue.append(target)
                # This can't be combined with above statement
                newValue = oldValue
                self.data[source] = newValue
        else:
            self.data[source] = [target]

    def get_friend_map(self, source, target):
        source, target = source.title(), target.title()  # Titlecase everything
        returnval = False
        # Determine if relationship exists (target in source value list)
        sourceGet = self.data.get(source)
        if sourceGet:
            if target in sourceGet:
                returnval = True
        return returnval

    # method with object routing for performance
    def add_friendship(alphabetMap, alphaObjs, friend1, friend2):
        friend1, friend2 = friend1.title(), friend2.title()
        fRoute1, fRoute2 = friend1[0:1], friend2[0:1]
        objectLookup = dict(map(reversed, alphabetMap.items()))
        fRoute1, fRoute2 = objectLookup.get(fRoute1), objectLookup.get(fRoute2)
        alphaObjs[fRoute1].add_friend_map(friend1, friend2)
        alphaObjs[fRoute2].add_friend_map(friend2, friend1)

    # method with object routing for performance
    def get_friendship(alphabetMap, alphaObjs, friend1, friend2):
        friend1, friend2 = friend1.title(), friend2.title()
        fRoute1, fRoute2 = friend1[0:1], friend2[0:1]
        objectLookup = dict(map(reversed, alphabetMap.items()))
        fRoute1, fRoute2 = objectLookup.get(fRoute1), objectLookup.get(fRoute2)
        if (alphaObjs[fRoute1].get_friend_map(friend1, friend2) or alphaObjs[fRoute2].get_friend_map(friend2, friend1)):
            # Assumption is first positive triggers return True.  If not, logic can
            # be further optimized by splitting conditional.
            return True
        else:
            return False

def main():
    # This map must not change without corresponding object management update
    # to captured chars (e.g. friend1[0:1], friend2[0:1] becomes friend1[0:2],
    # friend2[0:2] when you update alphabetMap to AA, AB, AC, AD, ...)
    alphabetMap = 'A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z'.split(
        sep=",")
    alphabetMap = dict(zip(range(len(alphabetMap)), alphabetMap))
    alphaObjs = []  # Object container

    # Create one alpha object per alpha, for data structures.  Thinking future
    # parallelization & scalability, though data spread is unbalanced.  Ranges
    # can be partitioned further.
    for x, y in alphabetMap.items():
        alphaObjs.append(FriendMap())
        alphaObjs[x].range = y

    # Security considerations for argument capture beyond scope of example.
    if len(sys.argv) != 4:
        print("Script usage is: " +
              sys.argv[0] + " 'filename' 'fname lname' 'fname lname'")
        exit(0)

    oFile = open(sys.argv[1], 'r')
    try:
        fileArray = list(oFile.readlines())
        for line in fileArray:
            lineArray = [x for x in line.replace('\n', '').split(sep=",")]
            FriendMap.add_friendship(
                alphabetMap, alphaObjs, lineArray[0].strip(), lineArray[1].strip())
    finally:
        oFile.close()

    if FriendMap.get_friendship(alphabetMap, alphaObjs, sys.argv[2].strip(), sys.argv[3].strip()):
        print("yes")
    else:
        print("no")

    qaCheck = FriendMap.add_friendship(alphabetMap, alphaObjs, 'John Doe', 'Lady Jane')
    assert (FriendMap.get_friendship(alphabetMap, alphaObjs, 'Lady jane', 'John Doe')
            ) == True, "ERROR:  Simple constrained validation failed.  Answer may not be accurate."

if __name__ == '__main__':
    main()
<commit_msg>Changed dictionary to set<commit_after>""" Summary:            Friends is question #3 from the Reelio test.

    P-Version:          QA'd on Python 3.4.1

    Knowledge domains:  Relationship networks, performance profiling
   						(e.g. import cProfile|profile, profile.run('main()' -or-
   						python -m cProfile [-o output_file] [-s sort_order] myscript.py)

    Candidates Eval'd:	Data model, map/reduce (functools module), OOD

    Considerations:		Short on time so trading off succinct requirement for functional requirement.
                        Intended usage of the system implies need for conceptual parallelization being designed in.  As implemented, there's still some blocking, but it's better than a monolithic container object.  With this design, full logical & Physical parallization is easier done later.

    Decision History:   Going OO, but not creating multi-threading per the alpha containers.  Also, getting user file data persistent in memory would be key for a real performance test.
                        As-is the object routing per alpha prefix is effectively an unbalanced/simplistic routing option that allows for parallelization, and implies built-in option for ranges with more granularity.
                        I naively applied the data model used in the data dictionary per reading I did on Hadoop a while back - could use a 2nd pass.
                        It was suggested I use sets inside dictionary, instead of lists.
                            (implies coming up with a naming key of some kind (e.g. Name::email@)) or embedding single-user data structures in set positions.

    Data Format:		Friend records --  {'Bob Brown': ['Sam Smith', 'Joe Shepard']}
                        Alpha lookup for object routing --  {0: 'A', 1: 'B', 2: 'C', 3: 'D', 4: 'E', 5: 'F', 6: 'G', 7: 'H', 8: 'I', 9: 'J', 10: 'K', 11: 'L', 12: 'M', 13: 'N', 14: 'O', 15: 'P', 16: 'Q', 17: 'R', 18: 'S', 19: 'T', 20: 'U', 21: 'V', 22: 'W', 23: 'X', 24: 'Y', 25: 'Z'}

    Script Usage:       python Friends.py <filename> <name1> <name2>

    Note:               There is gratuitous overdesign here but it was more natural to me than code shaving the test so I went for the exercise as relevant to production environments.
"""

import itertools
import sys

__author__ = "Michael Rais"
__version__ = "0.1-alpha"
__maintainer__ = "Michael Rais"
__email__ = "mrais@inbox.com"
__status__ = "Development"

class FriendMap():

    def __init__(self):
        self.range, self.data = '', {}

    def get_friends(self, source):
        source = source.title()  # Titlecase everything
        return self.data.get(source)

    def add_friend_map(self, source, target):
        source, target = source.title(), target.title()  # Titlecase everything
        # If source already exists, check if target associated or not
        sourceGet = self.data.get(source)
        if sourceGet:
            # Target not already associated - if it is do nothing.
            if target in sourceGet:
                oldValue = sourceGet
                oldValue.add(target)
                # This can't be combined with above statement
                newValue = oldValue
                self.data[source] = newValue
        else:
            item = set()
            item.add(target)
            self.data[source] = item

    def get_friend_map(self, source, target):
        source, target = source.title(), target.title()  # Titlecase everything
        returnval = False
        # Determine if relationship exists (target in source value list)
        sourceGet = self.data.get(source)
        if sourceGet:
            if target in sourceGet:
                returnval = True
        return returnval

# method with object routing for performance - these would never be in class of data structure object itself.
def add_friendship(alphabetMap, alphaObjs, friend1, friend2):
    friend1, friend2 = friend1.title(), friend2.title()
    fRoute1, fRoute2 = friend1[0:1], friend2[0:1]
    objectLookup = dict(map(reversed, alphabetMap.items()))
    fRoute1, fRoute2 = objectLookup.get(fRoute1), objectLookup.get(fRoute2)
    alphaObjs[fRoute1].add_friend_map(friend1, friend2)
    alphaObjs[fRoute2].add_friend_map(friend2, friend1)

# method with object routing for performance
def get_friendship(alphabetMap, alphaObjs, friend1, friend2):
    friend1, friend2 = friend1.title(), friend2.title()
    fRoute1, fRoute2 = friend1[0:1], friend2[0:1]
    objectLookup = dict(map(reversed, alphabetMap.items()))
    fRoute1, fRoute2 = objectLookup.get(fRoute1), objectLookup.get(fRoute2)
    if (alphaObjs[fRoute1].get_friend_map(friend1, friend2) or alphaObjs[fRoute2].get_friend_map(friend2, friend1)):
        # First positive in conditional exits conditional and return True.
        return True
    else:
        return False

def main():
    # This map must not change without corresponding object management update
    # to captured chars (e.g. friend1[0:1], friend2[0:1] becomes friend1[0:2],
    # friend2[0:2] when you update alphabetMap to AA, AB, AC, AD, ...)
    alphabetMap = 'A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z'.split(
        sep=",")
    alphabetMap = dict(zip(range(len(alphabetMap)), alphabetMap))
    alphaObjs = []  # Object container

    # Create one alpha object per alpha, for data structures.  Thinking future
    # parallelization & scalability, though data spread is unbalanced.  Ranges
    # can be partitioned further.
    for x, y in alphabetMap.items():
        alphaObjs.append(FriendMap())
        alphaObjs[x].range = y

    # Security considerations for argument capture beyond scope of example.
    if len(sys.argv) != 4:
        print("Script usage is: " +
              sys.argv[0] + " 'filename' 'fname lname' 'fname lname'")
        exit(0)

    oFile = open(sys.argv[1], 'r')
    try:
        fileArray = list(oFile.readlines())
        for line in fileArray:
            lineArray = [x for x in line.replace('\n', '').split(sep=",")]
            add_friendship(
                alphabetMap, alphaObjs, lineArray[0].strip(), lineArray[1].strip())
    finally:
        oFile.close()

    if get_friendship(alphabetMap, alphaObjs, sys.argv[2].strip(), sys.argv[3].strip()):
        print("yes")
    else:
        print("no")

    qaCheck = add_friendship(alphabetMap, alphaObjs, 'John Doe', 'Lady Jane')
    #assert (get_friendship(alphabetMap, alphaObjs, 'Lady jane', 'John Doe')
            #) == True, "ERROR:  Simple constrained validation failed.  Answer may not be accurate."

if __name__ == '__main__':
    main()
<|endoftext|>