<commit_before>#===============================================================================
# !/usr/bin/env python
#
# twitterRelationGraphs.py
#
# This script calls the twitter API to obtain the followers and friends graph for
# a given set of users. It also obtains the tweets of these users.
#
# Dependency:
# 1. Python 2.7
# 2. Python-Twitter-0.8.2 http://code.google.com/p/python-twitter/
#     - Refer to their documentation for installation of the dependencies
#     - Currently using a modified version of library at: <NEEDS LINK>
#     - Rebuild and reinstall with the mod ver
#
# Copyright (c) 2011 Andy Luong.
#===============================================================================

import twitter, sys
from operator import itemgetter
import codecs
import time
import argparse

#===============================================================================
# Globals
#===============================================================================
api = None
args = None

#===============================================================================
# Book Keeping
#===============================================================================
processedUsers = set()
processedOthers = set()
unprocessableUsers = set()
failGetTweets = 0

#===============================================================================
# Output Files
#===============================================================================
outFollowersGraph = None
outFollowersTweets = None
outFriendsGraph = None
outFriendsTweets = None

#===============================================================================
# Parameters
#===============================================================================
# Minimum number of followers the target user must have before we process their graphs
# Number must be at least 1 to be processed
minNumFollowers = 30

# Minimum number of tweets the follower/friend must have
minNumTweets = 30

# Maximum number of followers the target user has (celebs...)
maxNumFollowers = 1000

# Number of users to extract
numUsersToProcess = 20

# Number of tweets to extract for follower/friend (max = 200)
numTweetsPerTimeline = 200

# Twitter API Call delay between GetTimeline calls
apiCallDelaySeconds = 1

# max number continuous of failures before we apiCallDelaySeconds
maxNumFailures = 3

# Twitter API Call delay after maxNumFailures
failDelaySeconds = 7

#===============================================================================
# Authenticate twitter API calls
# This is required in order for most sophisticated API calls
# Obtain Keys after registering an application at: https://dev.twitter.com/apps
# Return: True if successful, False otherwise
#===============================================================================
def authenticate(ck, cs, atk, ats):
    global api

    api = twitter.Api(
        consumer_key = ck,
        consumer_secret = cs,
        access_token_key = atk,
        access_token_secret = ats)

    if not api.VerifyCredentials():
        print "There is an error with your authentication keys."
        return False

    else:
        print "You have been successfully authenticated."
        return True

#===============================================================================
# Checks how many remaining Twitter API are remaining
# If the 'remaining hits' is less than the minimum required, sleep script
#===============================================================================
def checkApiLimit():
    if api == None:
        return

    minRequiredHits = numUsersToProcess - 2

    if api.GetRateLimitStatus()[u'remaining_hits'] - minRequiredHits < 0:
        limit_sleep = round(api.GetRateLimitStatus()[u'reset_time_in_seconds'] - time.time() + 5)
        print "API limit reached, sleep for %d seconds..." % limit_sleep
        time.sleep(limit_sleep)

#===============================================================================
# Get the last 'numTweetsPerTimeline' tweets for a given user
# If the Twitter API call fails, try up 3 times
#===============================================================================
def getTweets(user):
    global failGetTweets

    print ("\tRetrieving Timeline for User: " +
           str(user.GetId()) + "/" + user.GetName() +
           "\tNumTweets: " + str(user.GetStatusesCount()))

    if user.GetId() in processedOthers:
        print "\t\tAlready Extracted tweets, so skipping..."
        return "SKIP"

    attempts = 0
    maxAttempts = 3

    while attempts < maxAttempts:
        try:
            attempts += 1
            tweets = api.GetUserTimeline(id = user.GetId(), count = numTweetsPerTimeline)
            print "\t\tSuccessfully Extracted %d tweets" % len(tweets)
            failGetTweets = 0

            return tweets

        except (KeyboardInterrupt, SystemExit):
            raise

        except:
            print ("\t\tAttempt %d: Failed retrieval for User: " + str(user.GetId())) % attempts
            failGetTweets = failGetTweets + 1

            if failGetTweets >= maxNumFailures:
                print "\nDue to high API call failure, sleeping script for %d seconds\n" % failDelaySeconds
                time.sleep(failDelaySeconds)
                failGetTweets = 0

    return []

#===============================================================================
# Remove users that have less than 'minNumTweets' of tweets
# Sort the remaining follower by number of tweets (descending)
# Return: Subset of Followers
#===============================================================================
def filterUsers(users):
    users = [(user.GetStatusesCount(), user) for user in users if (user.GetStatusesCount() >= minNumTweets and
                                                                   user.GetFollowersCount() <= maxNumFollowers)]
    users = sorted(users, key = itemgetter(0), reverse = True)
    users = [user[1] for user in users]

    return users

#===============================================================================
# For a given user, obtain their follower graph
# Notation: other = follower/friend
# Output the follower/friend graph (ID_user, SN_user, ID_other, SN_other)
# Output the follower/friend tweets, for each tweet (ID_other, SN_other, date, geo, text )
#===============================================================================
def outputGraphsAndTweets(user, fGraph, outGraph, outTweet):
    for other in fGraph:
        otherID = ("ID_" + other[0] + "\t" +
                      "SN_\'" + other[1] + "\'")

        #Write the graph
        outGraph.write("ID_" + str(user.GetId()) + "\t" +
                            "SN_\'" + user.GetName() + "\'\t" +
                            otherID +
                            "\n")

        for tweet in other[2]:

            geo = tweet.GetLocation()

            if geo == None or geo == '':
                geo = "NoCoords"
            geo = "<" + geo + ">"

            tweet_s = (otherID + "\t" +
                       tweet.GetCreatedAt() + "\t" +
                       geo + "\t" +
                       tweet.GetText().lower())

            #Write the Tweets
            outTweet.write(tweet_s + "\n")

#===============================================================================
# If the user has minNumFollowers <= # followers <= maxNumFollowers, find their followers and tweets
# Notation: other = follower/friend
# Return: list of followers/friends where each follower/friend(id, screen_name, list(tweets) )
#===============================================================================
def relationshipGraph(user, gType):
    if gType.lower() == 'followers':
        print "Processing Followers Graph for User: " + str(user.GetId()) + "/" + user.GetName()
    elif gType.lower() == 'friends':
        print "Processing Friends Graph for User: " + str(user.GetId()) + "/" + user.GetName()

    fg = []

    if (user.GetFollowersCount() >= minNumFollowers and
        user.GetFollowersCount() <= maxNumFollowers):
        others = []
        attempts = 0
        maxAttempts = 2

        while attempts < maxAttempts:
            try:
                attempts += 1

                if gType.lower() == 'followers':
                    others = api.GetFollowers(user.GetId())
                elif gType.lower() == 'friends':
                    others = api.GetFriends(user.GetId())
                else:
                    print "Unsupported Relationship Graph Processing: " + gType
                    return []

            except twitter.TwitterError:
                print ("\t\tAttempt %d: Failed retrieval for User: " + str(user.GetId())) % attempts
                failGetTweets = failGetTweets + 1

                if failGetTweets >= maxNumFailures:
                    print "\nDue to high API call failure, sleeping script for %d seconds\n" % failDelaySeconds
                    time.sleep(failDelaySeconds)
                    failGetTweets = 0

            except (KeyboardInterrupt, SystemExit):
                raise

        if others == []:
            return []

        others = filterUsers(others)
        count = 0

        for i in range(0, len(others)):
            #Delay between calls
            time.sleep(apiCallDelaySeconds)

            other = others[i]
            tweets = getTweets(other)
            processedOthers.add(other.GetId())

            if tweets == "SKIP":
                fg += [(str(other.GetId()), other.GetName(), [])]
                count += 1
            elif not tweets == []:
                fg += [(str(other.GetId()), other.GetName(), tweets)]
                count += 1

            if count >= numUsersToProcess:
                break
    else:
        print "\tSkipping User because of too few or many followers (%d)." % user.GetFollowersCount()

    return fg

#===============================================================================
# For each user, process their followers/friends graph, as specified
#===============================================================================
def processUser(userID):
    print "Processing User: " + userID

    try:
        user = api.GetUser(userID)

        if args.f:
            followers = relationshipGraph(user, 'followers')
            outputGraphsAndTweets(user, followers, outFollowersGraph, outFollowersTweets)

        if args.g:
            friends = relationshipGraph(user, 'friends')
            outputGraphsAndTweets(user, friends, outFriendsGraph, outFriendsTweets)

        processedUsers.add(str(user.GetId()))

    except twitter.TwitterError:
        print "\tUnprocessable User: ", userID
        unprocessableUsers.add(userID)

    except (KeyboardInterrupt, SystemExit):
        raise

#===============================================================================
# Closes output files
# Writes a log of processed users to 'processedUsers.log'
# Writes a log of unprocessed users to 'unprocessedUsers.log'
#===============================================================================
def cleanup():
    if outFollowersGraph:
        outFollowersGraph.close()

    if outFriendsGraph:
        outFriendsGraph.close()

    if outFollowersTweets:
        outFollowersTweets.close()

    flog = open('unprocessedUsers.log', 'w')
    for user in unprocessableUsers:
        flog.write(user + "\n")
    flog.close()

    flog = open('processedUsers.log', 'w')
    for user in processedUsers:
        flog.write(user + "\n")
    flog.close()

#===============================================================================
# Reads in the file with a list of users and attempts to process each user
#===============================================================================
def processUsers(users_file):
    print "Processing Users..."
    fin = open(users_file, 'r')

    for user in fin:
        user = user.strip()

        if not user in processedUsers and not len(user) == 0:
            checkApiLimit()
            processUser(user)

    fin.close()

def main():
    parser = argparse.ArgumentParser(description = 'Build Twitter Relation Graphs.')
    setup = parser.add_argument_group('Script Setup')
    graphs = parser.add_mutually_exclusive_group(required = True)
    params = parser.add_argument_group('Script Parameters (Optional)')

    setup.add_argument('-k',
                        action = "store",
                        help = 'Authentication Keys Input File',
                        metavar = 'FILE',
                        required = True)
    setup.add_argument('-u',
                        action = "store",
                        help = 'Users Input File',
                        metavar = 'FILE',
                        required = True)
    graphs.add_argument('-f',
                        action = "store",
                        metavar = 'FILE',
                        help = 'Graph File, Tweets File',
                        nargs = 2)
    graphs.add_argument('-g',
                        action = "store",
                        metavar = 'FILE',
                        help = 'Graph File, Tweets File',
                        nargs = 2)
    #Parameters
    params.add_argument('-minNumFollowers',
                        action = "store",
                        help = 'Minimum number of followers the target user must have before we process their graphs. Default %(default)s',
                        metavar = 'INT',
                        type = int,
                        default = 50)
    params.add_argument('-minNumTweets',
                        action = "store",
                        help = 'Minimum number of tweets the follower/friend must have. Default %(default)s',
                        metavar = 'INT',
                        type = int,
                        default = 30,)
    params.add_argument('-maxNumFollowers',
                        action = "store",
                        help = 'Maximum number of followers the target user has. Default %(default)s',
                        metavar = 'INT',
                        type = int,
                        default = 1000)
    params.add_argument('-numUsersToProcess',
                       action = "store",
                       help = 'Number of followers/friends to extract for a user. Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 20)
    params.add_argument('-numTweetsPerTimeline',
                       action = "store",
                       help = 'Number of tweets to extract for follower/friend (max = 200). Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 200)
    params.add_argument('-apiCallDelaySeconds',
                       action = "store",
                       help = 'Twitter API Call delay between GetTimeline calls. Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 1)
    params.add_argument('-maxNumFailures',
                       action = "store",
                       help = 'Max number continuous of failures before we apiCallDelaySeconds. Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 3)
    params.add_argument('-failDelaySeconds',
                       action = "store",
                       help = 'Twitter API Call delay after maxNumFailures. Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 7)
    global args
    args = parser.parse_args()

    #Parse Key File
    fin = open(args.k, 'r')
    keys = fin.readlines()
    fin.close()
    if not authenticate(keys[0].strip(),
                        keys[1].strip(),
                        keys[2].strip(),
                        keys[3].strip()):
        sys.exit()

    #Create output files
    if args.f:
        global outFollowersGraph, outFollowersTweets
        outFollowersGraph = codecs.open(args.f[0], encoding = 'utf-8', mode = 'w')
        outFollowersTweets = codecs.open(args.f[1], encoding = 'utf-8', mode = 'w')

    if args.g:
        global outFriendsGraph, outFriendsTweets
        outFriendsGraph = codecs.open(args.g[0], encoding = 'utf-8', mode = 'w')
        outFriendsTweets = codecs.open(args.g[1], encoding = 'utf-8', mode = 'w')

    #Set Parameters
    global minNumTweets, minNumFollowers, maxNumFollowers, numUsersToProcess, numTweetsPerTimeline
    global apiCallDelaySeconds, maxNumFailures, failDelaySeconds
    minNumTweets = args.minNumTweets
    minNumFollowers = max(1, args.minNumFollowers)
    maxNumFollowers = args.maxNumFollowers
    numUsersToProcess = args.numUsersToProcess
    numTweetsPerTimeline = min(200, args.numTweetsPerTimeline)
    apiCallDelaySeconds = args.apiCallDelaySeconds
    maxNumFailures = args.maxNumFailures
    failDelaySeconds = args.failDelaySeconds

    #Process Users
    processUsers(args.u)

    #Cleanup
    cleanup()

main()

<commit_msg>Added tweaking of default parameters Added more attempts for limit checking Allow user to build follower and friend graph simutaneously<commit_after>#===============================================================================
# !/usr/bin/env python
#
# twitterRelationGraphs.py
#
# This script calls the twitter API to obtain the followers and friends graph for
# a given set of users. It also obtains the tweets of these users.
#
# Dependency:
# 1. Python 2.7
# 2. Python-Twitter-0.8.2 http://code.google.com/p/python-twitter/
#     - Refer to their documentation for installation of the dependencies
#     - Currently using a modified version of library at: <NEEDS LINK>
#     - Rebuild and reinstall with the mod ver
#
# Copyright (c) 2011 Andy Luong.
#===============================================================================

import twitter, sys
from operator import itemgetter
import codecs
import time
import argparse

#===============================================================================
# Globals
#===============================================================================
api = None
args = None

#===============================================================================
# Book Keeping
#===============================================================================
processedUsers = set()
processedOthers = set()
unprocessableUsers = set()
failGetTweets = 0

#===============================================================================
# Output Files
#===============================================================================
outFollowersGraph = None
outFollowersTweets = None
outFriendsGraph = None
outFriendsTweets = None

#===============================================================================
# Parameters
#===============================================================================
# Minimum number of followers the target user must have before we process their graphs
# Number must be at least 1 to be processed
minNumFollowers = 30

# Minimum number of tweets the follower/friend must have
minNumTweets = 30

# Maximum number of followers the target user has (celebs...)
maxNumFollowers = 1000

# Number of users to extract
numUsersToProcess = 100

# Number of tweets to extract for follower/friend (max = 200)
numTweetsPerTimeline = 200

# Twitter API Call delay between GetTimeline calls
apiCallDelaySeconds = 3

# max number continuous of failures before we apiCallDelaySeconds
maxNumFailures = 3

# Twitter API Call delay after maxNumFailures
failDelaySeconds = 7

#===============================================================================
# Authenticate twitter API calls
# This is required in order for most sophisticated API calls
# Obtain Keys after registering an application at: https://dev.twitter.com/apps
# Return: True if successful, False otherwise
#===============================================================================
def authenticate(ck, cs, atk, ats):
    global api

    api = twitter.Api(
        consumer_key = ck,
        consumer_secret = cs,
        access_token_key = atk,
        access_token_secret = ats)

    if not api.VerifyCredentials():
        print "There is an error with your authentication keys."
        return False

    else:
        print "You have been successfully authenticated."
        return True

#===============================================================================
# Checks how many remaining Twitter API are remaining
# If the 'remaining hits' is less than the minimum required, sleep script
#===============================================================================
def checkApiLimit():
    if api == None:
        return
    attempts = 0
    while attempts < 3:
        try:
            attempts += 1
            minRequiredHits = numUsersToProcess - 2
            print "Remaining Hits: " + str(api.GetRateLimitStatus()[u'remaining_hits'])

            if api.GetRateLimitStatus()[u'remaining_hits'] - minRequiredHits < 0:
                limit_sleep = round(api.GetRateLimitStatus()[u'reset_time_in_seconds'] - time.time() + 5)
                print "API limit reached, sleep for %d seconds..." % limit_sleep
                time.sleep(limit_sleep)
                return
        except twitter.TwitterError:
            print "Error checking for Rate Limit, Sleeping for 60 seconds..."
            time.sleep(60)

#===============================================================================
# Get the last 'numTweetsPerTimeline' tweets for a given user
# If the Twitter API call fails, try up 3 times
#===============================================================================
def getTweets(user):
    global failGetTweets

    print ("\tRetrieving Timeline for User: " +
           str(user.GetId()) + "/" + user.GetName() +
           "\tNumTweets: " + str(user.GetStatusesCount()))

    if user.GetId() in processedOthers:
        print "\t\tAlready Extracted tweets, so skipping..."
        return "SKIP"

    attempts = 0
    maxAttempts = 3

    while attempts < maxAttempts:
        try:
            attempts += 1
            tweets = api.GetUserTimeline(id = user.GetId(), count = numTweetsPerTimeline)
            print "\t\tSuccessfully Extracted %d tweets" % len(tweets)
            failGetTweets = 0

            return tweets

        except (KeyboardInterrupt, SystemExit):
            raise

        except:
            print ("\t\tAttempt %d: Failed retrieval for User: " + str(user.GetId())) % attempts
            failGetTweets = failGetTweets + 1

            time.sleep(2);

            if failGetTweets >= maxNumFailures:
                print "\nDue to high API call failure, sleeping script for %d seconds\n" % failDelaySeconds
                time.sleep(failDelaySeconds)
                failGetTweets = 0

    return []

#===============================================================================
# Remove users that have less than 'minNumTweets' of tweets and ...
# minNumFollowers <= # followers <= maxNumFollowers
# Sort the remaining follower by number of tweets (descending)
# Return: Subset of Followers
#===============================================================================
def filterUsers(users):
    users = [(user.GetStatusesCount(), user) for user in users if (user.GetStatusesCount() >= minNumTweets and
                                                                   user.GetFollowersCount() >= minNumFollowers and
                                                                   user.GetFollowersCount() <= maxNumFollowers)]
    users = sorted(users, key = itemgetter(0), reverse = True)
    users = [user[1] for user in users]

    return users

#===============================================================================
# For a given user, obtain their follower graph
# Notation: other = follower/friend
# Output the follower/friend graph (ID_user, SN_user, ID_other, SN_other)
# Output the follower/friend tweets, for each tweet (ID_other, SN_other, date, geo, text ) - Optional
#===============================================================================
def outputGraphsAndTweets(user, fGraph, outGraph, outTweet):
    for other in fGraph:
        otherID = ("ID_" + other[0] + "\t" +
                   "SN_\'" + other[1] + "\'")

        #Write the graph
        outGraph.write("ID_" + str(user.GetId()) + "\t" +
                       "SN_\'" + user.GetName() + "\'\t" +
                       otherID +
                       "\n")

        #Write the Tweets
        if outTweet:
            for tweet in other[2]:

                geo = tweet.GetLocation()

                if geo == None or geo == '':
                    geo = "NoCoords"
                geo = "<" + geo + ">"

                tweet_s = (otherID + "\t" +
                           tweet.GetCreatedAt() + "\t" +
                           geo + "\t" +
                           tweet.GetText().lower())

                outTweet.write(tweet_s + "\n")

#===============================================================================
# If the user has minNumFollowers <= # followers <= maxNumFollowers, find their followers and tweets
# Notation: other = follower/friend
# Return: list of followers/friends where each follower/friend(id, screen_name, list(tweets) )
#===============================================================================
def relationshipGraph(user, gType, bTweets):
    global failGetTweets

    if gType.lower() == 'followers':
        print "Processing Followers Graph for User: " + str(user.GetId()) + "/" + user.GetName()
    elif gType.lower() == 'friends':
        print "Processing Friends Graph for User: " + str(user.GetId()) + "/" + user.GetName()

    fg = []

    if (user.GetFollowersCount() >= minNumFollowers and
        user.GetFollowersCount() <= maxNumFollowers):
        others = []
        attempts = 0
        maxAttempts = 2

        while attempts < maxAttempts:
            try:
                attempts += 1

                if gType.lower() == 'followers':
                    others = api.GetFollowers(user.GetId())
                    break
                elif gType.lower() == 'friends':
                    others = api.GetFriends(user.GetId())
                    break
                else:
                    print "Unsupported Relationship Graph Processing: " + gType
                    return []

            except twitter.TwitterError:
                print ("\t\tAttempt %d: Failed retrieval for User: " + str(user.GetId())) % attempts
                failGetTweets = failGetTweets + 1

                if failGetTweets >= maxNumFailures:
                    print "\nDue to high API call failure, sleeping script for %d seconds\n" % failDelaySeconds
                    time.sleep(failDelaySeconds)
                    failGetTweets = 0

            except (KeyboardInterrupt, SystemExit):
                raise

        if others == []:
            return []

        others = filterUsers(others)
        count = 0

        for i in range(0, len(others)):
            other = others[i]

            #Default Value such that we always output follower or friend graph
            tweets = "SKIP"

            #Grab Tweets from Others
            if bTweets:
                #Delay between calls
                time.sleep(apiCallDelaySeconds)
                tweets = getTweets(other)
                processedOthers.add(other.GetId())

            if tweets == "SKIP":
                fg += [(str(other.GetId()), other.GetName(), [])]
                count += 1
            elif not tweets == []:
                fg += [(str(other.GetId()), other.GetName(), tweets)]
                count += 1

            if count >= numUsersToProcess:
                break
    else:
        print "\tSkipping User because of too few or many followers (%d)." % user.GetFollowersCount()

    return fg

#===============================================================================
# For each user, process their followers/friends graph, as specified
#===============================================================================
def processUser(userID):
    print "Processing User: " + userID

    try:
        user = api.GetUser(userID)

        if args.f:
            followers = relationshipGraph(user, 'followers', not outFollowersTweets == None)
            outputGraphsAndTweets(user, followers, outFollowersGraph, outFollowersTweets)

        if args.g:
            friends = relationshipGraph(user, 'friends', not outFriendsTweets == None)
            outputGraphsAndTweets(user, friends, outFriendsGraph, outFriendsTweets)

        processedUsers.add(str(user.GetId()))

    except twitter.TwitterError:
        print "\tUnprocessable User: ", userID
        unprocessableUsers.add(userID)

    except (KeyboardInterrupt, SystemExit):
        raise

#===============================================================================
# Closes output files
# Writes a log of processed users to 'processedUsers.log'
# Writes a log of unprocessed users to 'unprocessedUsers.log'
#===============================================================================
def cleanup():
    if outFollowersGraph:
        outFollowersGraph.close()

    if outFriendsGraph:
        outFriendsGraph.close()

    if outFollowersTweets:
        outFollowersTweets.close()

    if outFriendsTweets:
        outFollowersTweets.close()

    flog = open(args.u + 'unprocessedUsers.log', 'w')
    for user in unprocessableUsers:
        flog.write(user + "\n")
    flog.close()

    flog = open(args.u + 'processedUsers.log', 'w')
    for user in processedUsers:
        flog.write(user + "\n")
    flog.close()

#===============================================================================
# Reads in the file with a list of users and attempts to process each user
#===============================================================================
def processUsers(users_file):
    print "Processing Users..."
    fin = open(users_file, 'r')

    for user in fin:
        user = user.strip()

        if not user in processedUsers and not len(user) == 0:
            checkApiLimit()
            processUser(user)

    fin.close()

def main():
    parser = argparse.ArgumentParser(description = 'Build Twitter Relation Graphs.')
    setup = parser.add_argument_group('Script Setup')
    #graphs = parser.add_mutually_exclusive_group(required = True)
    graphs = parser.add_argument_group('Relationships to Compute')
    params = parser.add_argument_group('Script Parameters (Optional)')

    setup.add_argument('-k',
                        action = "store",
                        help = 'Authentication Keys Input File',
                        metavar = 'FILE',
                        required = True)
    setup.add_argument('-u',
                        action = "store",
                        help = 'Users Input File',
                        metavar = 'FILE',
                        required = True)
    graphs.add_argument('-f',
                        action = "store",
                        metavar = 'FILE',
                        help = 'Followers Graph File, Followers Tweets File (opt)',
                        nargs = '+')
    graphs.add_argument('-g',
                        action = "store",
                        metavar = 'FILE',
                        help = 'Friends Graph File, Friends Tweets File (opt)',
                        nargs = '+')
    #Parameters
    params.add_argument('-minNumFollowers',
                        action = "store",
                        help = 'Minimum number of followers the target user must have before we process their graphs. Default %(default)s',
                        metavar = 'INT',
                        type = int,
                        default = 50)
    params.add_argument('-minNumTweets',
                        action = "store",
                        help = 'Minimum number of tweets the follower/friend must have. Default %(default)s',
                        metavar = 'INT',
                        type = int,
                        default = 30,)
    params.add_argument('-maxNumFollowers',
                        action = "store",
                        help = 'Maximum number of followers the target user has. Default %(default)s',
                        metavar = 'INT',
                        type = int,
                        default = 1000)
    params.add_argument('-numUsersToProcess',
                       action = "store",
                       help = 'Number of followers/friends to extract for a user. Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 100)
    params.add_argument('-numTweetsPerTimeline',
                       action = "store",
                       help = 'Number of tweets to extract for follower/friend (max = 200). Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 200)
    params.add_argument('-apiCallDelaySeconds',
                       action = "store",
                       help = 'Twitter API Call delay between GetTimeline calls. Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 3)
    params.add_argument('-maxNumFailures',
                       action = "store",
                       help = 'Max number continuous of failures before we apiCallDelaySeconds. Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 3)
    params.add_argument('-failDelaySeconds',
                       action = "store",
                       help = 'Twitter API Call delay after maxNumFailures. Default %(default)s',
                       metavar = 'INT',
                       type = int,
                       default = 7)
    global args
    args = parser.parse_args()
    #print args
    #sys.exit()

    #Parse Key File
    fin = open(args.k, 'r')
    keys = fin.readlines()
    fin.close()
    if not authenticate(keys[0].strip(),
                        keys[1].strip(),
                        keys[2].strip(),
                        keys[3].strip()):
        sys.exit()

    #Create output files
    if args.f:
        global outFollowersGraph, outFollowersTweets
        outFollowersGraph = codecs.open(args.f[0], encoding = 'utf-8', mode = 'w')
        if len(args.f) >= 2:
            outFollowersTweets = codecs.open(args.f[1], encoding = 'utf-8', mode = 'w')

    if args.g:
        global outFriendsGraph, outFriendsTweets
        outFriendsGraph = codecs.open(args.g[0], encoding = 'utf-8', mode = 'w')
        if  len(args.g) >= 2:
            outFriendsTweets = codecs.open(args.g[1], encoding = 'utf-8', mode = 'w')

    #Set Parameters
    global minNumTweets, minNumFollowers, maxNumFollowers, numUsersToProcess, numTweetsPerTimeline
    global apiCallDelaySeconds, maxNumFailures, failDelaySeconds
    minNumTweets = args.minNumTweets
    minNumFollowers = max(1, args.minNumFollowers)
    maxNumFollowers = args.maxNumFollowers
    numUsersToProcess = args.numUsersToProcess
    numTweetsPerTimeline = min(200, args.numTweetsPerTimeline)
    apiCallDelaySeconds = args.apiCallDelaySeconds
    maxNumFailures = args.maxNumFailures
    failDelaySeconds = args.failDelaySeconds

    #Process Users
    processUsers(args.u)

    #Cleanup
    cleanup()

main()

<|endoftext|>


<commit_before>from firedrake import *
from mpi4py import MPI
from argparse import ArgumentParser
import numpy as np
import pandas as pd
import sys

parser = ArgumentParser(description=("""Convergence test for the 3D Helmholtz."""),
                        add_help=False)

parser.add_argument("--verify",
                    default=False,
                    type=bool,
                    action="store",
                    help=(
                        "Verify static condensation with GMRES "
                        "converges in one iteration."
                    ))

parser.add_argument("--degree",
                    default=None,
                    type=int,
                    action="store",
                    help="Degree of approximation.")

parser.add_argument("--write",
                    default=False,
                    action="store_true",
                    help="Write solution to output?")

parser.add_argument("--help",
                    action="store_true",
                    help="Show help")

args, _ = parser.parse_known_args()

if args.help:
    help = parser.format_help()
    print("%s\n" % help)
    sys.exit(1)

def run_convergence_test(degree, write=False):

    name = "HelmholtzProblem"
    param_set = "scpc_hypre"
    params = {"snes_type": "ksponly",
              "pmat_type": "matfree",
              "ksp_type": "preonly",
              "pc_type": "python",
              "pc_python_type": "scpc.SCCG",
              # HYPRE on the reduced system
              "static_condensation": {"ksp_type": "cg",
                                      "ksp_rtol": 1e-16,
                                      "ksp_monitor": True,
                                      "pc_type": "hypre",
                                      "pc_hypre_type": "boomeramg",
                                      "pc_hypre_boomeramg_no_CF": True,
                                      "pc_hypre_boomeramg_coarsen_type": "HMIS",
                                      "pc_hypre_boomeramg_interp_type": "ext+i",
                                      "pc_hypre_boomeramg_P_max": 4,
                                      "pc_hypre_boomeramg_agg_nl": 1}}

    if args.verify:
        # Wrap PC in GMRES to monitor convergence.
        print("Wrapping solver options in GMRES outer loop.")
        params["ksp_type"] = "gmres"
        params["ksp_monitor"] = True

    r_params = range(0, 6)
    l2_errors = []
    gmres_its = []
    sc_ksp_its = []
    num_dofs = []
    num_cells = []
    reductions = []
    for r in r_params:
        mesh_size = 2 ** r
        mesh = UnitCubeMesh(mesh_size, mesh_size, mesh_size)
        V = FunctionSpace(mesh, "CG", degree)
        x, y, z = SpatialCoordinate(mesh)

        u = Function(V).assign(0.0)
        v = TestFunction(V)
        f = Function(V)
        f.interpolate((1 + 27*pi*pi)*cos(3*pi*x)*cos(3*pi*y)*cos(3*pi*z))
        F = inner(grad(v), grad(u))*dx + v*u*dx - inner(v, f)*dx
        r0 = assemble(F)
        problem = NonlinearVariationalProblem(F, u)
        solver = NonlinearVariationalSolver(problem,
                                            solver_parameters=params)

        output_file = "%s-d-%s-size-%s-params-%s" % (name,
                                                     degree,
                                                     r,
                                                     param_set)

        print("\nSolving 3D %s problem of degree %s, mesh size %s, "
              "and parameter set %s\n" % (name,
                                          degree,
                                          r,
                                          param_set))
        solver.solve()
        r1 = solver.snes.ksp.buildResidual()
        reductions.append(r1.norm()/r0.dat.norm)
        u_h = problem.u
        u_a = Function(FunctionSpace(mesh, "CG", degree + 1), name="analytic")
        sol = cos(3*pi*x)*cos(3*pi*y)*cos(3*pi*z)
        u_a.interpolate(sol)

        gmres_its.append(solver.snes.getLinearSolveIterations())
        sc_ksp_its.append(solver.snes.ksp.getPC().getPythonContext().sc_ksp.getIterationNumber())
        num_dofs.append(problem.u.dof_dset.layout_vec.getSize())
        num_cells.append(mesh.comm.allreduce(mesh.cell_set.size,
                                             op=MPI.SUM))

        print("\n Solver complete...\n")

        print("\n Computing L2 errors\n")
        l2_errors.append(errornorm(sol, u_h, norm_type="L2"))

        # Only write output for the final solve
        if write:
            if r == len(r_params):
                print("\n Writing output to pvd file\n")
                File(output_file + ".pvd").write(u_h, u_a)

    print("\nComputing convergence rates\n")
    l2_errors = np.array(l2_errors)
    rates = list(np.log2(l2_errors[:-1] / l2_errors[1:]))
    # Insert '---' in first slot, as there is rate to compute
    rates.insert(0, '---')
    degrees = [degree]*len(l2_errors)

    data = {"Mesh": r_params,
            "ResidualReductions": reductions,
            "L2Errors": l2_errors,
            "ConvRates": rates,
            "GMRESIterations": gmres_its,
            "SCPCIterations": sc_ksp_its,
            "NumDOFS": num_dofs,
            "NumCells": num_cells,
            "Degree": degrees}

    df = pd.DataFrame(data)
    result = "helmholtz_conv-d-%d.csv" % degree
    df.to_csv(result, index=False, mode="w")

if args.degree:
    run_convergence_test(degree=args.degree, write=args.write)
else:
    # If no degree provided, run full convergence test
    for degree in range(4, 8):
        run_convergence_test(degree=degree, write=args.write)
<commit_msg>Print via PETSc for cleaner logs<commit_after>from firedrake import *
from firedrake.petsc import PETSc
from mpi4py import MPI
from argparse import ArgumentParser
import numpy as np
import pandas as pd
import sys

parser = ArgumentParser(description=("""Convergence test for the 3D Helmholtz."""),
                        add_help=False)

parser.add_argument("--verify",
                    default=False,
                    type=bool,
                    action="store",
                    help=(
                        "Verify static condensation with GMRES "
                        "converges in one iteration."
                    ))

parser.add_argument("--degree",
                    default=None,
                    type=int,
                    action="store",
                    help="Degree of approximation.")

parser.add_argument("--write",
                    default=False,
                    action="store_true",
                    help="Write solution to output?")

parser.add_argument("--help",
                    action="store_true",
                    help="Show help")

args, _ = parser.parse_known_args()

if args.help:
    help = parser.format_help()
    print("%s\n" % help)
    sys.exit(1)

def run_convergence_test(degree, write=False):

    name = "HelmholtzProblem"
    param_set = "scpc_hypre"
    params = {"snes_type": "ksponly",
              "pmat_type": "matfree",
              "ksp_type": "preonly",
              "pc_type": "python",
              "pc_python_type": "scpc.SCCG",
              # HYPRE on the reduced system
              "static_condensation": {"ksp_type": "cg",
                                      "ksp_rtol": 1e-16,
                                      "ksp_monitor": True,
                                      "pc_type": "hypre",
                                      "pc_hypre_type": "boomeramg",
                                      "pc_hypre_boomeramg_no_CF": True,
                                      "pc_hypre_boomeramg_coarsen_type": "HMIS",
                                      "pc_hypre_boomeramg_interp_type": "ext+i",
                                      "pc_hypre_boomeramg_P_max": 4,
                                      "pc_hypre_boomeramg_agg_nl": 1}}

    if args.verify:
        # Wrap PC in GMRES to monitor convergence.
        print("Wrapping solver options in GMRES outer loop.")
        params["ksp_type"] = "gmres"
        params["ksp_monitor"] = True

    r_params = range(0, 6)
    l2_errors = []
    gmres_its = []
    sc_ksp_its = []
    num_dofs = []
    num_cells = []
    reductions = []
    for r in r_params:
        mesh_size = 2 ** r
        mesh = UnitCubeMesh(mesh_size, mesh_size, mesh_size)
        V = FunctionSpace(mesh, "CG", degree)
        x, y, z = SpatialCoordinate(mesh)

        u = Function(V).assign(0.0)
        v = TestFunction(V)
        f = Function(V)
        f.interpolate((1 + 27*pi*pi)*cos(3*pi*x)*cos(3*pi*y)*cos(3*pi*z))
        F = inner(grad(v), grad(u))*dx + v*u*dx - inner(v, f)*dx
        r0 = assemble(F)
        problem = NonlinearVariationalProblem(F, u)
        solver = NonlinearVariationalSolver(problem,
                                            solver_parameters=params)

        output_file = "%s-d-%s-size-%s-params-%s" % (name,
                                                     degree,
                                                     r,
                                                     param_set)

        PETSc.Sys.Print("\nSolving 3D %s problem of degree %s, mesh size %s, "
                        "and parameter set %s\n" % (name,
                                                    degree,
                                                    r,
                                                    param_set))
        solver.solve()
        r1 = solver.snes.ksp.buildResidual()
        reductions.append(r1.norm()/r0.dat.norm)
        u_h = problem.u
        u_a = Function(FunctionSpace(mesh, "CG", degree + 1), name="analytic")
        sol = cos(3*pi*x)*cos(3*pi*y)*cos(3*pi*z)
        u_a.interpolate(sol)

        gmres_its.append(solver.snes.getLinearSolveIterations())
        sc_ksp_its.append(solver.snes.ksp.getPC().getPythonContext().sc_ksp.getIterationNumber())
        num_dofs.append(problem.u.dof_dset.layout_vec.getSize())
        num_cells.append(mesh.comm.allreduce(mesh.cell_set.size,
                                             op=MPI.SUM))

        PETSc.Sys.Print("\n Solver complete...\n")

        PETSc.Sys.Print("\n Computing L2 errors\n")
        l2_errors.append(errornorm(sol, u_h, norm_type="L2"))

        # Only write output for the final solve
        if write:
            if COMM_WORLD.rank == 0:
                if r == len(r_params):
                    PETSc.Sys.Print("\n Writing output to pvd file\n")
                    File(output_file + ".pvd").write(u_h, u_a)

    PETSc.Sys.Print("\nComputing convergence rates\n")
    l2_errors = np.array(l2_errors)
    rates = list(np.log2(l2_errors[:-1] / l2_errors[1:]))
    # Insert '---' in first slot, as there is rate to compute
    rates.insert(0, '---')
    degrees = [degree]*len(l2_errors)

    if COMM_WORLD.rank == 0:
        data = {"Mesh": r_params,
                "ResidualReductions": reductions,
                "L2Errors": l2_errors,
                "ConvRates": rates,
                "GMRESIterations": gmres_its,
                "SCPCIterations": sc_ksp_its,
                "NumDOFS": num_dofs,
                "NumCells": num_cells,
                "Degree": degrees}

        df = pd.DataFrame(data)
        result = "helmholtz_conv-d-%d.csv" % degree
        df.to_csv(result, index=False, mode="w")

if args.degree:
    run_convergence_test(degree=args.degree, write=args.write)
else:
    # If no degree provided, run full convergence test
    for degree in range(4, 8):
        run_convergence_test(degree=degree, write=args.write)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

'''
$ wget https://github.com/PheWAS/PheWAS/blob/master/data/phemap.rda
$ wget https://github.com/PheWAS/PheWAS/blob/master/data/pheinfo.rda
$ r
> load('phemap.rda')
> load('pheinfo.rda')
> write.csv(phemap, 'phemap.csv', row.names=F)
> write.csv(pheinfo, 'pheinfo.csv', row.names=F)

# found this link at <https://www.cms.gov/medicare/coding/ICD9providerdiagnosticcodes/codes.html>
$ wget https://www.cms.gov/Medicare/Coding/ICD9ProviderDiagnosticCodes/Downloads/ICD-9-CM-v32-master-descriptions.zip
$ unzip ICD-9-CM-v32-master-descriptions.zip

$ wget https://medschool.vanderbilt.edu/cpm/files/cpm/public_files/perl_phewas.zip
$ unzip perl_phewas.zip
$ cat code_translation.txt | tr "\r" "\n" > icd9s.tsv

# got ICD9_CodeCounts_20160323_LF.txt from group. it differs from others in some ways.

# maybe I should have just downloaded the last 6 icd9 versions and unioned them?
'''

import csv, json, itertools, string

icd9s_1 = list(csv.DictReader(open('icd9s.tsv'), delimiter='\t'))
string_for_icd9_1 = {}
for x in icd9s_1:
    string_for_icd9_1[x['CODE']] = x['STR_SHORT'].strip()

icd9s_2 = list(open("ICD-9-CM-v32-master-descriptions/CMS32_DESC_LONG_DX.txt", encoding="ISO-8859-1"))
string_for_icd9_2 = {}
for x in icd9s_2:
    icd9, desc = x.strip().split(' ', 1)
    icd9 = icd9[:3] + '.' + icd9[3:]
    string_for_icd9_2[icd9] = desc.strip()

icd9s_3 = list(csv.DictReader(open("ICD9_CodeCounts_20160323_LF.txt"), delimiter='\t'))
string_for_icd9_3 = {}
for x in icd9s_3:
    string_for_icd9_3[x['icd9']] = x['icd9_string'].strip()

phemap = list(csv.DictReader(open('phemap.csv')))
icd9s_for_phecode = {}
for x in phemap:
    icd9s_for_phecode.setdefault(x['phecode'], []).append(x['icd9'])

pheinfo = list(csv.DictReader(open("pheinfo.csv")))
info_for_phecode = {}
for x in pheinfo:
    info_for_phecode[x['phecode']] = {
        'desc': x['description'].strip(),
        'category': x['group'].strip(),
        'color': x['color'].strip(),
    }

def cmp(*xs):
    for n in range(1, 1+len(xs)):
        for c in itertools.combinations(range(len(xs)), n):
            print(''.join(string.ascii_letters[i] for i in c), end=':')
            print(len(set.intersection(*[set(xs[i]) for i in c])), end=' ')
    print('')
    for n in range(1, 1+len(xs)):
        for c in itertools.combinations(range(len(xs)), n):
            print(''.join(string.ascii_letters[i] for i in c), end='')
            comp = [i for i in range(len(xs)) if i not in c]
            if comp:
                print('-' + ''.join(string.ascii_letters[i] for i in comp), end='')
            print(':', end='')
            print(len(set.intersection(*[set(xs[i]) for i in c]).difference(*[xs[i] for i in comp])), end=' ')
    print('')
cmp(info_for_phecode, icd9s_for_phecode)
cmp(string_for_icd9_1, string_for_icd9_2, string_for_icd9_3)

for phecode in info_for_phecode:
    ii = []
    for icd9 in icd9s_for_phecode[phecode]:
        # 3 is from group, 2 is from govt, 1 is from vb
        desc = string_for_icd9_3.get(icd9, False) or string_for_icd9_2.get(icd9, False) or string_for_icd9_1.get(icd9, False) or '?'
        ii.append({'icd9': icd9, 'desc': desc})
    ii = sorted(ii, key=lambda x: x['icd9'])
    info_for_phecode[phecode]['icd9s'] = ii

with open('phecodes_icd9.json', 'w') as f:
    json.dump(info_for_phecode, f, sort_keys=True, indent=1)
<commit_msg>style<commit_after>#!/usr/bin/env python3

'''
$ wget https://github.com/PheWAS/PheWAS/blob/master/data/phemap.rda
$ wget https://github.com/PheWAS/PheWAS/blob/master/data/pheinfo.rda
$ r
> load('phemap.rda')
> load('pheinfo.rda')
> write.csv(phemap, 'phemap.csv', row.names=F)
> write.csv(pheinfo, 'pheinfo.csv', row.names=F)

# found this link at <https://www.cms.gov/medicare/coding/ICD9providerdiagnosticcodes/codes.html>
$ wget https://www.cms.gov/Medicare/Coding/ICD9ProviderDiagnosticCodes/Downloads/ICD-9-CM-v32-master-descriptions.zip
$ unzip ICD-9-CM-v32-master-descriptions.zip

$ wget https://medschool.vanderbilt.edu/cpm/files/cpm/public_files/perl_phewas.zip
$ unzip perl_phewas.zip
$ cat code_translation.txt | tr "\r" "\n" > icd9s.tsv

# got ICD9_CodeCounts_20160323_LF.txt from group. it differs from others in some ways.

# maybe I should have just downloaded the last 6 icd9 versions and unioned them?
'''

import csv
import json
import itertools
import string

icd9s_1 = list(csv.DictReader(open('icd9s.tsv'), delimiter='\t'))
string_for_icd9_1 = {}
for x in icd9s_1:
    string_for_icd9_1[x['CODE']] = x['STR_SHORT'].strip()

icd9s_2 = list(open("ICD-9-CM-v32-master-descriptions/CMS32_DESC_LONG_DX.txt", encoding="ISO-8859-1"))
string_for_icd9_2 = {}
for x in icd9s_2:
    icd9, desc = x.strip().split(' ', 1)
    icd9 = icd9[:3] + '.' + icd9[3:]
    string_for_icd9_2[icd9] = desc.strip()

icd9s_3 = list(csv.DictReader(open("ICD9_CodeCounts_20160323_LF.txt"), delimiter='\t'))
string_for_icd9_3 = {}
for x in icd9s_3:
    string_for_icd9_3[x['icd9']] = x['icd9_string'].strip()

phemap = list(csv.DictReader(open('phemap.csv')))
icd9s_for_phecode = {}
for x in phemap:
    icd9s_for_phecode.setdefault(x['phecode'], []).append(x['icd9'])

pheinfo = list(csv.DictReader(open("pheinfo.csv")))
info_for_phecode = {}
for x in pheinfo:
    info_for_phecode[x['phecode']] = {
        'desc': x['description'].strip(),
        'category': x['group'].strip(),
        'color': x['color'].strip(),
    }

def cmp(*xs):
    for n in range(1, 1+len(xs)):
        for c in itertools.combinations(range(len(xs)), n):
            print(''.join(string.ascii_letters[i] for i in c), end=':')
            print(len(set.intersection(*[set(xs[i]) for i in c])), end=' ')
    print('')
    for n in range(1, 1+len(xs)):
        for c in itertools.combinations(range(len(xs)), n):
            print(''.join(string.ascii_letters[i] for i in c), end='')
            comp = [i for i in range(len(xs)) if i not in c]
            if comp:
                print('-' + ''.join(string.ascii_letters[i] for i in comp), end='')
            print(':', end='')
            print(len(set.intersection(*[set(xs[i]) for i in c]).difference(*[xs[i] for i in comp])), end=' ')
    print('')
cmp(info_for_phecode, icd9s_for_phecode)
cmp(string_for_icd9_1, string_for_icd9_2, string_for_icd9_3)

for phecode in info_for_phecode:
    ii = []
    for icd9 in icd9s_for_phecode[phecode]:
        # 3 is from group, 2 is from govt, 1 is from vb
        desc = string_for_icd9_3.get(icd9, False) or string_for_icd9_2.get(icd9, False) or string_for_icd9_1.get(icd9, False) or '?'
        ii.append({'icd9': icd9, 'desc': desc})
    ii = sorted(ii, key=lambda x: x['icd9'])
    info_for_phecode[phecode]['icd9s'] = ii

with open('phecodes_icd9.json', 'w') as f:
    json.dump(info_for_phecode, f, sort_keys=True, indent=1)
<|endoftext|>


<commit_before>#This file is part of the ISIS IBEX application.
#Copyright (C) 2012-2016 Science & Technology Facilities Council.
#All rights reserved.
#
#This program is distributed in the hope that it will be useful.
#This program and the accompanying materials are made available under the
#terms of the Eclipse Public License v1.0 which accompanies this distribution.
#EXCEPT AS EXPRESSLY SET FORTH IN THE ECLIPSE PUBLIC LICENSE V1.0, THE PROGRAM
#AND ACCOMPANYING MATERIALS ARE PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES
#OR CONDITIONS OF ANY KIND.  See the Eclipse Public License v1.0 for more details.
#
#You should have received a copy of the Eclipse Public License v1.0
#along with this program; if not, you can obtain a copy from
#https://www.eclipse.org/org/documents/epl-v10.php or
#http://opensource.org/licenses/eclipse-1.0.php

import time
from server_common.channel_access import ChannelAccess
from server_common.utilities import print_and_log

ALIAS_HEADER = """\
##
EVALUATION ORDER ALLOW, DENY

## serve blockserver internal variables, including Flag variables needed by blockserver process to restart gateway
%sCS:GATEWAY:BLOCKSERVER:.*    				    ALLOW	ANYBODY	    1
## allow anybody to generate gateway reports
%sCS:GATEWAY:BLOCKSERVER:report[1-9]Flag		ALLOW	ANYBODY		1
"""

class Gateway(object):
    """A class for interacting with the EPICS gateway that creates the aliases used for implementing blocks"""

    def __init__(self, prefix, block_prefix, pvlist_file, pv_prefix):
        """Constructor.

        Args:
            prefix (string): The prefix for the gateway
            block_prefix (string): The block prefix
            pvlist_file (string): Where to write the gateway file
            pv_prefix (string): Prefix for instrument PVs
        """
        self._prefix = prefix
        self._block_prefix = block_prefix
        self._pvlist_file = pvlist_file
        self._pv_prefix = pv_prefix

    def exists(self):
        """Checks the gateway exists by querying on of the PVs.

        Returns:
            bool : Whether the gateway is running and is accessible
        """
        val = ChannelAccess.caget(self._prefix + "pvtotal")
        if val is None:
            return False
        else:
            return True

    def _reload(self):
        print_and_log("Reloading gateway")
        try:
            # Have to wait after put as the gateway does not do completion callbacks (it is not an IOC)
            ChannelAccess.caput(self._prefix + "newAsFlag", 1, False)

            while ChannelAccess.caget(self._prefix + "newAsFlag") == 1:
                time.sleep(1)
            print_and_log("Gateway reloaded")
        except Exception as err:
            print_and_log("Problem with reloading the gateway %s" % err)

    def _generate_alias_file(self, blocks=None):
        # Generate blocks.pvlist for gateway
        f = open(self._pvlist_file, 'w')
        header = ALIAS_HEADER % (self._pv_prefix, self._pv_prefix)
        f.write(header)
        if blocks is not None:
            for name, value in blocks.iteritems():
                lines = self._generate_alias(value.name, value.pv, value.local)
                for l in lines:
                    f.write(l)
        # Add a blank line at the end!
        f.write("\n")
        f.close()

    def _generate_alias(self, blockname, pv, local):
        print("Creating block: %s for %s" % (blockname, pv))
        lines = list()
        if pv.endswith(".VAL"):
            # Strip off the .VAL
            pv = pv.rstrip(".VAL")
        if pv.endswith(":SP"):
            # The block points at a setpoint
            lines.append("## The block points at a :SP, so it needs an optional group as genie_python will append an additional :SP, but ignore :RC:\n")
            if local:
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\(:SP\)?\([.:].*\)    ALIAS    %s%s\\2\n' % (self._pv_prefix, self._block_prefix,
                                                                                blockname, self._pv_prefix, pv))
                lines.append('%s%s%s\(:SP\)?:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s\(:SP\)?    ALIAS    %s%s\n' %
                             (self._pv_prefix, self._block_prefix, blockname, self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\(:SP\)?\([.:].*\)    ALIAS    %s\\2\n' % (self._pv_prefix, self._block_prefix,
                                                                               blockname, pv))
                lines.append('%s%s%s\(:SP\)?:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s\(:SP\)?    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
        elif pv.endswith(".RBV"):
            # The block points at a readback value (most likely for a motor)
            lines.append("## The block points at a .RBV, so it needs entries for both reading the RBV and for the rest, but ignore :RC:\n")
            if local:
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s%s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                         self._pv_prefix, pv.rstrip(".RBV")))
                lines.append('%s%s%s:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s[.]VAL    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                  self._pv_prefix, pv))
                lines.append('%s%s%s    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                            self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                       pv.rstrip(".RBV")))
                lines.append('%s%s%s:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s[.]VAL    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
                lines.append('%s%s%s    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
        else:
            # Standard case
            lines.append("## Standard block with entries for matching :SP and :SP:RBV as well as .EGU, but ignore :RC:\n")
            if local:
                # Pattern match is for picking up any any SP or SP:RBV
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s%s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                         self._pv_prefix, pv))
                lines.append('%s%s%s:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                            self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any any SP or SP:RBV
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                       pv))
                lines.append('%s%s%s:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
        return lines

    def set_new_aliases(self, blocks):
        """Creates the aliases for the blocks and restarts the gateway.

        Args:
            blocks (OrderedDict): The blocks that belong to the configuration
        """
        self._generate_alias_file(blocks)
        self._reload()
<commit_msg>Extend to handle a block pointing at any field name<commit_after>#This file is part of the ISIS IBEX application.
#Copyright (C) 2012-2016 Science & Technology Facilities Council.
#All rights reserved.
#
#This program is distributed in the hope that it will be useful.
#This program and the accompanying materials are made available under the
#terms of the Eclipse Public License v1.0 which accompanies this distribution.
#EXCEPT AS EXPRESSLY SET FORTH IN THE ECLIPSE PUBLIC LICENSE V1.0, THE PROGRAM
#AND ACCOMPANYING MATERIALS ARE PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES
#OR CONDITIONS OF ANY KIND.  See the Eclipse Public License v1.0 for more details.
#
#You should have received a copy of the Eclipse Public License v1.0
#along with this program; if not, you can obtain a copy from
#https://www.eclipse.org/org/documents/epl-v10.php or
#http://opensource.org/licenses/eclipse-1.0.php

import time
from server_common.channel_access import ChannelAccess
from server_common.utilities import print_and_log

ALIAS_HEADER = """\
##
EVALUATION ORDER ALLOW, DENY

## serve blockserver internal variables, including Flag variables needed by blockserver process to restart gateway
%sCS:GATEWAY:BLOCKSERVER:.*    				    ALLOW	ANYBODY	    1
## allow anybody to generate gateway reports
%sCS:GATEWAY:BLOCKSERVER:report[1-9]Flag		ALLOW	ANYBODY		1
"""

class Gateway(object):
    """A class for interacting with the EPICS gateway that creates the aliases used for implementing blocks"""

    def __init__(self, prefix, block_prefix, pvlist_file, pv_prefix):
        """Constructor.

        Args:
            prefix (string): The prefix for the gateway
            block_prefix (string): The block prefix
            pvlist_file (string): Where to write the gateway file
            pv_prefix (string): Prefix for instrument PVs
        """
        self._prefix = prefix
        self._block_prefix = block_prefix
        self._pvlist_file = pvlist_file
        self._pv_prefix = pv_prefix

    def exists(self):
        """Checks the gateway exists by querying on of the PVs.

        Returns:
            bool : Whether the gateway is running and is accessible
        """
        val = ChannelAccess.caget(self._prefix + "pvtotal")
        if val is None:
            return False
        else:
            return True

    def _reload(self):
        print_and_log("Reloading gateway")
        try:
            # Have to wait after put as the gateway does not do completion callbacks (it is not an IOC)
            ChannelAccess.caput(self._prefix + "newAsFlag", 1, False)

            while ChannelAccess.caget(self._prefix + "newAsFlag") == 1:
                time.sleep(1)
            print_and_log("Gateway reloaded")
        except Exception as err:
            print_and_log("Problem with reloading the gateway %s" % err)

    def _generate_alias_file(self, blocks=None):
        # Generate blocks.pvlist for gateway
        f = open(self._pvlist_file, 'w')
        header = ALIAS_HEADER % (self._pv_prefix, self._pv_prefix)
        f.write(header)
        if blocks is not None:
            for name, value in blocks.iteritems():
                lines = self._generate_alias(value.name, value.pv, value.local)
                for l in lines:
                    f.write(l)
        # Add a blank line at the end!
        f.write("\n")
        f.close()

    def _generate_alias(self, blockname, pv, local):
        print("Creating block: %s for %s" % (blockname, pv))
        lines = list()
        if pv.endswith(".VAL"):
            # Strip off the .VAL
            pv = pv.rstrip(".VAL")
        # look for a field name in PV
        m = re.match(r'.*(\.[A-Z0-9]+)$', pv)
        pvsuffix = m.group(1)
        if pv.endswith(":SP"):
            # The block points at a setpoint
            lines.append("## The block points at a :SP, so it needs an optional group as genie_python will append an additional :SP, but ignore :RC:\n")
            if local:
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\(:SP\)?\([.:].*\)    ALIAS    %s%s\\2\n' % (self._pv_prefix, self._block_prefix,
                                                                                blockname, self._pv_prefix, pv))
                lines.append('%s%s%s\(:SP\)?:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s\(:SP\)?    ALIAS    %s%s\n' %
                             (self._pv_prefix, self._block_prefix, blockname, self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\(:SP\)?\([.:].*\)    ALIAS    %s\\2\n' % (self._pv_prefix, self._block_prefix,
                                                                               blockname, pv))
                lines.append('%s%s%s\(:SP\)?:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s\(:SP\)?    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
        elif pvsuffix is not None:
            # The block points at a readback value (most likely for a motor)
            lines.append("## The block points at a .RBV, so it needs entries for both reading the RBV and for the rest, but ignore :RC:\n")
            if local:
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s%s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                         self._pv_prefix, pv.rstrip(pvsuffix)))
                lines.append('%s%s%s:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s[.]VAL    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                  self._pv_prefix, pv))
                lines.append('%s%s%s    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                            self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                       pv.rstrip(pvsuffix)))
                lines.append('%s%s%s:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s[.]VAL    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
                lines.append('%s%s%s    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
        else:
            # Standard case
            lines.append("## Standard block with entries for matching :SP and :SP:RBV as well as .EGU, but ignore :RC:\n")
            if local:
                # Pattern match is for picking up any any SP or SP:RBV
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s%s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                         self._pv_prefix, pv))
                lines.append('%s%s%s:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                            self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any any SP or SP:RBV
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                       pv))
                lines.append('%s%s%s:RC:.*    DENY\n' % (self._pv_prefix, self._block_prefix, blockname))
                lines.append('%s%s%s    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
        return lines

    def set_new_aliases(self, blocks):
        """Creates the aliases for the blocks and restarts the gateway.

        Args:
            blocks (OrderedDict): The blocks that belong to the configuration
        """
        self._generate_alias_file(blocks)
        self._reload()
<|endoftext|>


<commit_before># Copyright (c) 2015 SONATA-NFV, Thales Communications & Security
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, Thales Communications & Security
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

# pylint: disable=missing-docstring
import os
import sys
import logging
import tempfile
import shutil
import re
import typing  # noqa pylint: disable=unused-import
import pytest  # type: ignore
import docker  # type: ignore

_LOGGER = logging.getLogger(__name__)

@pytest.fixture(scope="module")
def docker_client() -> docker.DockerClient:
    yield docker.from_env()

@pytest.fixture(scope="module")
# pylint: disable=redefined-outer-name
def son_cli_image(docker_client: docker.DockerClient) -> str:
    target = "registry.sonata-nfv.eu:5000/son-cli:latest"
    try:
        docker_client.images.get(target)
    except docker.errors.ImageNotFound:
        pytest.fail("The Docker image {0!s} was not found.".format(target))
    return target

TYPE_SON_CLI = typing.Callable[[int, typing.List[str]], bytes]

@pytest.fixture(scope="module")
# pylint: disable=redefined-outer-name
def _son_cli(docker_client: docker.DockerClient,
             son_cli_image: str) -> TYPE_SON_CLI:
    labels = ["com.sonata.analyze.integration.pytest"]
    tmp_dir = None
    with tempfile.TemporaryDirectory(dir="/tmp") as cur_path:
        tmp_dir = cur_path
    path = os.path.realpath(os.path.join(
        sys.modules[__name__].__file__, '..', 'fixtures'))
    shutil.copytree(path, tmp_dir)
    volumes = {tmp_dir: {"bind": "/root", "mode": "rw"}}

    def run_in_son_cli(timeout_sec: int, command: typing.List[str]) -> bytes:
        entrypoint = ["/usr/bin/timeout", "-s", "KILL", str(timeout_sec)]
        tmp = b""
        try:
            tmp = docker_client.containers.run(image=son_cli_image,
                                               command=command,
                                               entrypoint=entrypoint,
                                               labels=labels,
                                               remove=True,
                                               volumes=volumes,
                                               working_dir="/root",
                                               stderr=True)
        except (docker.errors.ContainerError, docker.errors.APIError) as cntex:
            pytest.fail("A son-cli command failed: {0!s}".format(cntex))
        return tmp
    return run_in_son_cli

@pytest.fixture(scope="module")
# pylint: disable=redefined-outer-name
def son_cli(_son_cli: TYPE_SON_CLI) -> TYPE_SON_CLI:
    tmp = _son_cli(30, ["son-workspace", "--init"])
    if not re.search("Creating workspace at /root/.son-workspace",
                     tmp.decode()):
        pytest.fail("Failed to create the workspace: {0!s}"
                    .format(tmp.decode()))
    return _son_cli

@pytest.fixture(scope="module")
# pylint: disable=redefined-outer-name
def packages(docker_client: docker.DockerClient):
    path = os.path.realpath(os.path.join(
        sys.modules[__name__].__file__, '..', 'fixtures'))
    docker_client.images.build(path=path,
                               tag="integration-sonata",
                               dockerfile="Dockerfile")
    _LOGGER.debug("Docker image built")
    yield None
    #

@pytest.mark.integration
@pytest.mark.usefixtures("packages")
def test_run() -> None:
    assert True
<commit_msg>Create a fixture to create and return the service package<commit_after># Copyright (c) 2015 SONATA-NFV, Thales Communications & Security
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, Thales Communications & Security
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

# pylint: disable=missing-docstring
import os
import sys
import logging
import tempfile
import shutil
import re
import typing  # noqa pylint: disable=unused-import
import pytest  # type: ignore
import docker  # type: ignore

_LOGGER = logging.getLogger(__name__)

@pytest.fixture(scope="module")
def docker_client() -> docker.DockerClient:
    yield docker.from_env()

@pytest.fixture(scope="module")
# pylint: disable=redefined-outer-name
def son_cli_image(docker_client: docker.DockerClient) -> str:
    target = "registry.sonata-nfv.eu:5000/son-cli:latest"
    try:
        docker_client.images.get(target)
    except docker.errors.ImageNotFound:
        pytest.fail("The Docker image {0!s} was not found.".format(target))
    return target

TYPE_SON_CLI = typing.Callable[[int, typing.List[str]], bytes]

@pytest.fixture(scope="module")
# pylint: disable=redefined-outer-name
def _son_cli(docker_client: docker.DockerClient,
             son_cli_image: str) -> TYPE_SON_CLI:
    labels = ["com.sonata.analyze.integration.pytest"]
    tmp_dir = None
    with tempfile.TemporaryDirectory(dir="/tmp") as cur_path:
        tmp_dir = cur_path
    path = os.path.realpath(os.path.join(
        sys.modules[__name__].__file__, '..', 'fixtures'))
    shutil.copytree(path, tmp_dir)
    volumes = {tmp_dir: {"bind": "/root", "mode": "rw"}}

    def run_in_son_cli(timeout_sec: int, command: typing.List[str]) -> bytes:
        entrypoint = ["/usr/bin/timeout", "-s", "KILL", str(timeout_sec)]
        tmp = b""
        try:
            tmp = docker_client.containers.run(image=son_cli_image,
                                               command=command,
                                               entrypoint=entrypoint,
                                               labels=labels,
                                               remove=True,
                                               volumes=volumes,
                                               working_dir="/root",
                                               stderr=True)
        except (docker.errors.ContainerError, docker.errors.APIError) as cntex:
            pytest.fail("A son-cli command failed: {0!s}".format(cntex))
        return tmp
    return run_in_son_cli

@pytest.fixture(scope="module")
# pylint: disable=redefined-outer-name
def son_cli(_son_cli: TYPE_SON_CLI) -> TYPE_SON_CLI:
    tmp = _son_cli(30, ["son-workspace", "--init"])
    if not re.search("Creating workspace at /root/.son-workspace",
                     tmp.decode()):
        pytest.fail("Failed to create the workspace: {0!s}"
                    .format(tmp.decode()))
    return _son_cli

@pytest.fixture(scope="module")
# pylint: disable=redefined-outer-name
def packages(docker_client: docker.DockerClient):
    path = os.path.realpath(os.path.join(
        sys.modules[__name__].__file__, '..', 'fixtures'))
    docker_client.images.build(path=path,
                               tag="integration-sonata",
                               dockerfile="Dockerfile")
    _LOGGER.debug("Docker image built")
    yield None
    #

@pytest.fixture(scope="module")
# pylint: disable=redefined-outer-name
def service_packages(son_cli: TYPE_SON_CLI) -> str:
    tmp = son_cli(60,
                  ["son-package", "--project", "sonata-integration"]).decode()
    check = [re.search("Package generated successfully", tmp),
             re.search("File: /root/eu.sonata-nfv.pack"
                       "age.sonata-empty-service.0.4.son", tmp)]
    if not all(check):
        pytest.fail("Failed to create the service package: {0!s}".format(tmp))
    return "/root/eu.sonata-nfv.package.sonata-empty-service.0.4.son"

@pytest.mark.integration
@pytest.mark.usefixtures("packages")
def test_run() -> None:
    assert True
<|endoftext|>


<commit_before>#!/usr/local/bin/python3
# Python Challenge - 25
# http://www.pythonchallenge.com/pc/hex/lake.html
# Username: butter; Password: fly
# Keyword:

import os
import wave
import matplotlib.pyplot as plt
import numpy as np

def main():
    '''
    Hint: imagine how they sound; picture of lake with puzzle pieces overlaid
    can you see the waves; name of image is lake1.jpg
    Download wave files 1-25: http://www.pythonchallenge.com/pc/hex/lake[N].wav
    Bash command (-o option writes to file vs. stdout, #1 refers to seq val
        to name the file, -U is proxy-user login info):

        curl -U butter:fly -o "./waves_chall_25/lake#1.wav"
        "http://www.pythonchallenge.com/pc/hex/lake[1-25].wav"
    '''

    source_dir = './waves_chall_25/'
    wave_files = ['lake{}.wav'.format(i) for i in range(1, 26)]
    return 0

if __name__ == '__main__':
    main()
<commit_msg>Look at wave properties of several files<commit_after>#!/Applications/anaconda/envs/imgPIL/bin
# Python Challenge - 25
# http://www.pythonchallenge.com/pc/hex/lake.html
# Username: butter; Password: fly
# Keyword:

'''
Uses Anaconda environment with Pillow for image processing
    - Python 3.7, numpy, and Pillow (PIL)
    - Run `source activate imgPIL`, `python chall_25.py`
'''

import wave

def main():
    '''
    Hint: imagine how they sound; picture of lake with puzzle pieces overlaid
    can you see the waves; name of image is lake1.jpg
    Download wave files 1-25: http://www.pythonchallenge.com/pc/hex/lake[N].wav
    Bash command (-o option writes to file vs. stdout, #1 refers to seq val
        to name the file, -U is proxy-user login info):

        curl -U butter:fly -o "./waves_chall_25/lake#1.wav"
        "http://www.pythonchallenge.com/pc/hex/lake[1-25].wav"
    '''

    source_dir = './waves_chall_25/'
    wave_files = ['lake{}.wav'.format(i) for i in range(1, 26)]

    # Look at wave file properties
    with wave.open(source_dir + wave_files[0], 'rb') as tmp:
        print('Frames: {}'.format(tmp.getnframes()))  # 10800
        print('Channels: {}'.format(tmp.getnchannels()))  # 1
        print('Samp width: {}'.format(tmp.getsampwidth()))  # 1
        print('Frame rate: {}'.format(tmp.getframerate()))  # 9600

    # Convert each wave file to an image

    return 0

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# deploy.py - test cases for the COTDeploy class and helpers
#
# January 2015, Glenn F. Matthews
# Copyright (c) 2013-2015 the COT project developers.
# See the COPYRIGHT.txt file at the top-level directory of this distribution
# and at https://github.com/glennmatthews/cot/blob/master/COPYRIGHT.txt.
#
# This file is part of the Common OVF Tool (COT) project.
# It is subject to the license terms in the LICENSE.txt file found in the
# top-level directory of this distribution and at
# https://github.com/glennmatthews/cot/blob/master/LICENSE.txt. No part
# of COT, including this file, may be copied, modified, propagated, or
# distributed except according to the terms contained in the LICENSE.txt file.

"""Unit test cases for the COT.deploy.COTDeploy class and helpers."""

import logging
import re

from COT.tests.ut import COT_UT
from COT.ui_shared import UI
from COT.deploy import COTDeploy, SerialConnection
from COT.data_validation import InvalidInputError, ValueUnsupportedError

logger = logging.getLogger(__name__)

class TestCOTDeploy(COT_UT):
    """Test cases for COTDeploy."""

    def setUp(self):
        """Test case setup function called automatically prior to each test."""
        super(TestCOTDeploy, self).setUp()
        self.instance = COTDeploy(UI())
        self.instance.package = self.input_ovf

    def test_not_ready_with_no_args(self):
        """Verify that ready_to_run() is False without all mandatory args."""
        ready, reason = self.instance.ready_to_run()
        self.assertEqual(ready, False)
        self.assertTrue(re.search("HYPERVISOR.*mandatory", reason))
        self.assertRaises(InvalidInputError, self.instance.run)

        self.instance.hypervisor = "esxi"
        self.instance.package = None
        ready, reason = self.instance.ready_to_run()
        self.assertEqual(ready, False)
        self.assertTrue(re.search("PACKAGE.*mandatory", reason))
        self.assertRaises(InvalidInputError, self.instance.run)

    def test_invalid_args(self):
        """Negative testing for various arguments."""
        with self.assertRaises(InvalidInputError):
            self.instance.hypervisor = "frobozz"
        with self.assertRaises(InvalidInputError):
            self.instance.configuration = ""
        with self.assertRaises(InvalidInputError):
            self.instance.configuration = "X"
        with self.assertRaises(InvalidInputError):
            self.instance.network_map = [""]
        with self.assertRaises(InvalidInputError):
            self.instance.network_map = ["foo"]
        with self.assertRaises(InvalidInputError):
            self.instance.network_map = ["=bar"]
        with self.assertRaises(InvalidInputError):
            self.instance.network_map = ["foo="]

    def test_run(self):
        """Test the generic run() implementation."""
        self.instance.hypervisor = "esxi"
        self.instance.serial_connection = ["tcp::22", "tcp::23", "tcp::24"]
        self.instance.run()

class TestSerialConnection(COT_UT):
    """Unit test cases for SerialConnection class."""

    def test_from_cli_string_invalid(self):
        """Negative tests for SerialConnection.from_cli_string() method."""
        # Parsing failure
        self.assertRaises(InvalidInputError,
                          SerialConnection.from_cli_string, ',bar')
        # Parsing succeeds but invalid kind
        self.assertRaises(ValueUnsupportedError,
                          SerialConnection.from_cli_string, 'foo:bar')
        # Parsing succeeds but invalid value
        self.assertRaises(InvalidInputError,
                          SerialConnection.from_cli_string, 'tcp:bar')
        # Parsing succeeds but incorrect options
        self.assertRaises(InvalidInputError,
                          SerialConnection.from_cli_string,
                          'file:/tmp/foo.txt')
        self.assertEqual(None, SerialConnection.from_cli_string("   "))

    def test_from_cli_string_valid(self):
        """Positive tests for SerialConnection.from_cli_string()."""
        c = SerialConnection.from_cli_string('/dev/ttyS0')
        self.assertEqual('device', c.kind)
        self.assertEqual('/dev/ttyS0', c.value)
        self.assertEqual({}, c.options)

        c = SerialConnection.from_cli_string(
            'file:/tmp/foo.txt,datastore=datastore1')
        self.assertEqual('file', c.kind)
        self.assertEqual('/tmp/foo.txt', c.value)
        self.assertEqual({'datastore': 'datastore1'}, c.options)

        c = SerialConnection.from_cli_string('tcp::22,server')
        self.assertEqual('tcp', c.kind)
        self.assertEqual(':22', c.value)
        self.assertEqual({'server': True}, c.options)

        c = SerialConnection.from_cli_string('telnet://1.1.1.1:1111')
        self.assertEqual('telnet', c.kind)
        self.assertEqual('1.1.1.1:1111', c.value)
        self.assertEqual({}, c.options)
<commit_msg>Explicit test for str(SerialConnection)<commit_after>#!/usr/bin/env python
#
# deploy.py - test cases for the COTDeploy class and helpers
#
# January 2015, Glenn F. Matthews
# Copyright (c) 2013-2015 the COT project developers.
# See the COPYRIGHT.txt file at the top-level directory of this distribution
# and at https://github.com/glennmatthews/cot/blob/master/COPYRIGHT.txt.
#
# This file is part of the Common OVF Tool (COT) project.
# It is subject to the license terms in the LICENSE.txt file found in the
# top-level directory of this distribution and at
# https://github.com/glennmatthews/cot/blob/master/LICENSE.txt. No part
# of COT, including this file, may be copied, modified, propagated, or
# distributed except according to the terms contained in the LICENSE.txt file.

"""Unit test cases for the COT.deploy.COTDeploy class and helpers."""

import logging
import re

from COT.tests.ut import COT_UT
from COT.ui_shared import UI
from COT.deploy import COTDeploy, SerialConnection
from COT.data_validation import InvalidInputError, ValueUnsupportedError

logger = logging.getLogger(__name__)

class TestCOTDeploy(COT_UT):
    """Test cases for COTDeploy."""

    def setUp(self):
        """Test case setup function called automatically prior to each test."""
        super(TestCOTDeploy, self).setUp()
        self.instance = COTDeploy(UI())
        self.instance.package = self.input_ovf

    def test_not_ready_with_no_args(self):
        """Verify that ready_to_run() is False without all mandatory args."""
        ready, reason = self.instance.ready_to_run()
        self.assertEqual(ready, False)
        self.assertTrue(re.search("HYPERVISOR.*mandatory", reason))
        self.assertRaises(InvalidInputError, self.instance.run)

        self.instance.hypervisor = "esxi"
        self.instance.package = None
        ready, reason = self.instance.ready_to_run()
        self.assertEqual(ready, False)
        self.assertTrue(re.search("PACKAGE.*mandatory", reason))
        self.assertRaises(InvalidInputError, self.instance.run)

    def test_invalid_args(self):
        """Negative testing for various arguments."""
        with self.assertRaises(InvalidInputError):
            self.instance.hypervisor = "frobozz"
        with self.assertRaises(InvalidInputError):
            self.instance.configuration = ""
        with self.assertRaises(InvalidInputError):
            self.instance.configuration = "X"
        with self.assertRaises(InvalidInputError):
            self.instance.network_map = [""]
        with self.assertRaises(InvalidInputError):
            self.instance.network_map = ["foo"]
        with self.assertRaises(InvalidInputError):
            self.instance.network_map = ["=bar"]
        with self.assertRaises(InvalidInputError):
            self.instance.network_map = ["foo="]

    def test_run(self):
        """Test the generic run() implementation."""
        self.instance.hypervisor = "esxi"
        self.instance.serial_connection = ["tcp::22", "tcp::23", "tcp::24"]
        self.instance.run()

class TestSerialConnection(COT_UT):
    """Unit test cases for SerialConnection class."""

    def test_from_cli_string_invalid(self):
        """Negative tests for SerialConnection.from_cli_string() method."""
        # Parsing failure
        self.assertRaises(InvalidInputError,
                          SerialConnection.from_cli_string, ',bar')
        # Parsing succeeds but invalid kind
        self.assertRaises(ValueUnsupportedError,
                          SerialConnection.from_cli_string, 'foo:bar')
        # Parsing succeeds but invalid value
        self.assertRaises(InvalidInputError,
                          SerialConnection.from_cli_string, 'tcp:bar')
        # Parsing succeeds but incorrect options
        self.assertRaises(InvalidInputError,
                          SerialConnection.from_cli_string,
                          'file:/tmp/foo.txt')
        self.assertEqual(None, SerialConnection.from_cli_string("   "))

    def test_from_cli_string_valid(self):
        """Positive tests for SerialConnection.from_cli_string()."""
        c = SerialConnection.from_cli_string('/dev/ttyS0')
        self.assertEqual('device', c.kind)
        self.assertEqual('/dev/ttyS0', c.value)
        self.assertEqual({}, c.options)
        self.assertEqual(str(c),
                         "<SerialConnection kind: device "
                         "value: /dev/ttyS0 options: {}>")

        c = SerialConnection.from_cli_string(
            'file:/tmp/foo.txt,datastore=datastore1')
        self.assertEqual('file', c.kind)
        self.assertEqual('/tmp/foo.txt', c.value)
        self.assertEqual({'datastore': 'datastore1'}, c.options)
        self.assertEqual(str(c),
                         "<SerialConnection kind: file "
                         "value: /tmp/foo.txt "
                         "options: {'datastore': 'datastore1'}>")

        c = SerialConnection.from_cli_string('tcp::22,server')
        self.assertEqual('tcp', c.kind)
        self.assertEqual(':22', c.value)
        self.assertEqual({'server': True}, c.options)
        self.assertEqual(str(c),
                         "<SerialConnection kind: tcp value: :22 "
                         "options: {'server': True}>")

        c = SerialConnection.from_cli_string('telnet://1.1.1.1:1111')
        self.assertEqual('telnet', c.kind)
        self.assertEqual('1.1.1.1:1111', c.value)
        self.assertEqual({}, c.options)
        self.assertEqual(str(c),
                         "<SerialConnection kind: telnet "
                         "value: 1.1.1.1:1111 options: {}>")
<|endoftext|>


<commit_before>
# coding: utf-8
import os
import numpy as np
import matplotlib.pyplot as plt
import h5py
import scipy
from PIL import Image
from scipy import ndimage
import pickle

# In[106]:

def sigmoid(x):
    return 1/(1+ np.exp(-x))
# In[122]:

def predict(W,b,X):
    """ Predicts output """

    m = X.shape[1]

    predictions = np.zeros((1,m))
    W = W.reshape(X.shape[0], 1)
    yHat = sigmoid(np.dot(W.T, X) + b)

    for i in range(yHat.shape[1]):
        if yHat[0,i] <= 0.5 :
            predictions[0,i] = 0
        else:
            predictions[0,i] = 1

    return predictions

model_dir = "models"
if not os.path.exists(model_dir):
    os.makedirs(model_dir)

def saveModel(params, name):
    f = open( model_dir + '/' + name + ".model", "wb" )
    pickle.dump( params,f )
    f.close()

def loadModel(name):
    f = open(model_dir + '/' + name + ".model", "rb")
    model = pickle.load(f)
    f.close()
    return model

def get_predction(data):
    size = 64
    channel = 3
    model = loadModel('cat__lr0_01__iter_1000')

    ## resize img
    new_img = data.resize((size,size), Image.ANTIALIAS)
    img_norm = np.array(new_img)/255
    img_input = img_norm.reshape(size*size*channel,1)
    yhat = np.squeeze(predict(model["W"], model["b"],img_input))
    return 1 if yhat == 1 else 0

<commit_msg>removed unused pkgs<commit_after># coding: utf-8
import os
import numpy as np
import scipy
from PIL import Image
import pickle

# In[106]:

def sigmoid(x):
    return 1/(1+ np.exp(-x))
# In[122]:

def predict(W,b,X):
    """ Predicts output """

    m = X.shape[1]

    predictions = np.zeros((1,m))
    W = W.reshape(X.shape[0], 1)
    yHat = sigmoid(np.dot(W.T, X) + b)

    for i in range(yHat.shape[1]):
        if yHat[0,i] <= 0.5 :
            predictions[0,i] = 0
        else:
            predictions[0,i] = 1

    return predictions

model_dir = "models"
if not os.path.exists(model_dir):
    os.makedirs(model_dir)

def saveModel(params, name):
    f = open( model_dir + '/' + name + ".model", "wb" )
    pickle.dump( params,f )
    f.close()

def loadModel(name):
    f = open(model_dir + '/' + name + ".model", "rb")
    model = pickle.load(f)
    f.close()
    return model

def get_predction(data):
    size = 64
    channel = 3
    model = loadModel('cat__lr0_01__iter_1000')

    ## resize img
    new_img = data.resize((size,size), Image.ANTIALIAS)
    img_norm = np.array(new_img)/255
    img_input = img_norm.reshape(size*size*channel,1)
    yhat = np.squeeze(predict(model["W"], model["b"],img_input))
    return 1 if yhat == 1 else 0

<|endoftext|>