<commit_before>#!/usr/bin/python3

# author: Karel Fiala
# email: fiala.karel@gmail.com

# version 0.2.0

"""
Pouziti:

from network import Network

chceme jen posilat:
    client = Network()

chceme posilat i prijimat (sednout si na port):
    server = Network("0.0.0.0", 8888)

vstupy a vystupy jsou pole (prevod z/do JSONu je automaticky)
"""

import socket
import json

# asi staci kdyz MTU pro 1 paket je max 1500
BUFF_SIZE = 1024

class Network(object):

    def __init__(self, ip = "", port = 0):

        self.createdns()

        self.server = False;
        self.addr = False;
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        if ip != "":
            self.sock.bind((ip, port)) # jen server
            self.server = True;
        pass

    def createdns(self):
        # define IP -- DNS too slow and fails often
        self.host = dict()
        self.host["main.haut.local"] = "192.168.1.51"
        self.host["dev.haut.local"] = "192.168.1.90"
        self.host["webserver.haut.local"] = "127.0.0.1"

    def getip(self, host):
        return self.host[host]

    def gethost(self, ip):
        for value, key in self.host.items():
            if key == ip:
                return value

    def send(self, message, host, port):
        try:
            self.sock.sendto(bytes(json.dumps(message), "utf-8"), (self.host[host] , port))
        except ValueError:
            print("ValueError happen -- continuing")
            pass        # chyba JSONu
        # TODO -- doplnit osetreni chyb socketu?

    def recv(self):
        if self.server:
            data, self.addr = self.sock.recvfrom(BUFF_SIZE)
            try:
                return json.loads(data.decode("utf-8"))
            except ValueError:
                return False    # neprisel nam JSON (vracime False)
        else:
            return False

    def recv_from(self):
        # return last ip
        return self.addr
<commit_msg>server+webserver moved to rpi2b<commit_after>#!/usr/bin/python3

# author: Karel Fiala
# email: fiala.karel@gmail.com

# version 0.2.0

"""
Pouziti:

from network import Network

chceme jen posilat:
    client = Network()

chceme posilat i prijimat (sednout si na port):
    server = Network("0.0.0.0", 8888)

vstupy a vystupy jsou pole (prevod z/do JSONu je automaticky)
"""

import socket
import json

# asi staci kdyz MTU pro 1 paket je max 1500
BUFF_SIZE = 1024

class Network(object):

    def __init__(self, ip = "", port = 0):

        self.createdns()

        self.server = False;
        self.addr = False;
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        if ip != "":
            self.sock.bind((ip, port)) # jen server
            self.server = True;
        pass

    def createdns(self):
        # define IP -- DNS too slow and fails often
        self.host = dict()
        self.host["main.haut.local"] = "192.168.1.74"
        self.host["dev.haut.local"] = "192.168.1.90"
        self.host["webserver.haut.local"] = "127.0.0.1"

    def getip(self, host):
        return self.host[host]

    def gethost(self, ip):
        for value, key in self.host.items():
            if key == ip:
                return value

    def send(self, message, host, port):
        try:
            self.sock.sendto(bytes(json.dumps(message), "utf-8"), (self.host[host] , port))
        except ValueError:
            print("ValueError happen -- continuing")
            pass        # chyba JSONu
        # TODO -- doplnit osetreni chyb socketu?

    def recv(self):
        if self.server:
            data, self.addr = self.sock.recvfrom(BUFF_SIZE)
            try:
                return json.loads(data.decode("utf-8"))
            except ValueError:
                return False    # neprisel nam JSON (vracime False)
        else:
            return False

    def recv_from(self):
        # return last ip
        return self.addr
<|endoftext|>


<commit_before>import logging.handlers
import re
import traceback
from datetime import datetime
from collections import defaultdict

import praw

import database
import globals
import reddit
import strings
import utility

log = logging.getLogger("bot")

def MessageLineUpdateSubscribe(line, author, created):
	results = defaultdict(list)
	if line.startswith("updateme") or line.startswith("subscribeme") or line.startswith("http"):
		users = re.findall('(?: /?u/)([\w-]*)', line)
		subs = re.findall('(?: /?r/)(\w*)', line)
		links = re.findall('(?:reddit.com/r/\w*/comments/)(\w*)', line)
		filters = re.findall('(?:filter=)(\S*)', line)

		if len(links) != 0:
			try:
				submission = reddit.getSubmission(links[0])
				users.append(str(submission.author))
				subs.append(str(submission.subreddit))
			except Exception as err:
				log.debug("Exception parsing link")

		if len(users) != 0 and len(subs) != 0 and not (len(users) > 1 and len(subs) > 1):
			if line.startswith("updateme"):
				subscriptionTypeSingle = True
			elif line.startswith("subscribeme"):
				subscriptionTypeSingle = False
			else:
				subscriptionTypeSingle = not database.subredditDefaultSubscribe(subs[0].lower())

			if len(filters):
				filter = filters[0]
			else:
				filter = None
			if len(users) > 1:
				for user in users:
					result, data = utility.addUpdateSubscription(author, user, subs[0], created, subscriptionTypeSingle, filter)
					results[result].append(data)
			elif len(subs) > 1:
				for sub in subs:
					result, data = utility.addUpdateSubscription(author, users[0], sub, created, subscriptionTypeSingle, filter)
					results[result].append(data)
			else:
				result, data = utility.addUpdateSubscription(author, users[0], subs[0], created, subscriptionTypeSingle, filter)
				results[result].append(data)

	return results

def MessageLineRemove(line, author):
	results = defaultdict(list)
	if line.startswith("removeall"):
		log.info("Removing all subscriptions for /u/"+author)
		results['removed'].extend(database.getMySubscriptions(author))
		database.removeAllSubscriptions(author)

	elif line.startswith("remove"):
		users = re.findall('(?:/?u/)([\w-]*)', line)
		subs = re.findall('(?:/?r/)(\w*)', line)

		if len(users) != 0 and len(subs) != 0 and not (len(users) > 1 and len(subs) > 1):
			if len(users) > 1:
				for user in users:
					result, data = utility.removeSubscription(author, user, subs[0])
					results[result].append(data)
			elif len(subs) > 1:
				for sub in subs:
					result, data = utility.removeSubscription(author, users[0], sub)
					results[result].append(data)
			else:
				result, data = utility.removeSubscription(author, users[0], subs[0])
				results[result].append(data)

	return results

def MessageLineList(line, author):
	results = defaultdict(list)
	if line.startswith("mysubscriptions") or line.startswith("myupdates"):
		log.info("Listing subscriptions for /u/"+author)
		results['list'] = True

	return results

def MessageLineDelete(line, author):
	results = defaultdict(list)
	if line.startswith("deletecomment"):
		threadID = re.findall('(?: t3_)(\w*)', line)

		if len(threadID) == 0: return results

		commentID = database.deleteComment(threadID[0], author)
		if commentID:
			log.info("Deleting comment with ID %s/%s", threadID[0], commentID)
			if reddit.deleteComment(id=commentID):
				results['commentsDeleted'].append(threadID[0])
			else:
				log.warning("Could not delete comment with ID %s/%s", threadID[0], commentID)

	return results

def MessageLineAddSubreddit(line):
	results = defaultdict(list)
	if line.startswith("addsubreddit"):
		subs = re.findall('(?:/?r/)(\w*)', line)
		filters = re.findall('(?:filter=)(\S*)', line)
		if len(filters):
			filter = filters[0]
			log.debug("Found filter in addsubreddit: "+filter)
		else:
			filter = None

		for sub in subs:
			log.info("Whitelisting subreddit /r/"+sub)
			deniedRequests = database.getDeniedSubscriptions(sub.lower())

			for user in deniedRequests:
				log.info("Messaging /u/%s that their subscriptions in /r/%s have been activated", user, sub)
				strList = strings.activatingSubredditMessage(sub.lower(), deniedRequests[user])
				strList.append("\n\n*****\n\n")
				strList.append(strings.footer)
				if not reddit.sendMessage(user, strings.messageSubject(user), ''.join(strList)):
					log.warning("Could not send message to /u/%s when activating subreddit", user)

			results['subredditsAdded'].append({'subreddit': sub, 'subscribers': len(deniedRequests)})

			database.activateSubreddit(sub, line.startswith("addsubredditsub"), filter)

	return results

def MessageLineSubredditPM(line):
	results = defaultdict(list)
	if line.startswith("subredditpm"):
		subs = re.findall('(?:/?r/)(\w*)', line)
		if line.startswith("subredditpmtrue"):
			alwaysPM = True
		elif line.startswith("subredditpmfalse"):
			alwaysPM = False
		else:
			return results

		for sub in subs:
			log.info("Setting subreddit /r/"+sub+" to "+("don't " if not alwaysPM else "")+"alwaysPM")
			database.setAlwaysPMForSubreddit(sub.lower(), alwaysPM)
			results['alwaysPM'].append({'subreddit': sub, 'alwaysPM': alwaysPM})

	return results

def MessageLineMute(line, author, hasAdmin):
	results = defaultdict(list)
	if line.startswith("leavemealone") or line.startswith("talktome"):
		addBlacklist = True if line.startswith("leavemealone") else False
		subs = re.findall('(?:/?r/)(\w*)', line)
		users = re.findall('(?:/?u/)([\w-]*)', line)

		if len(subs) or len(users):
			if hasAdmin:
				for sub in subs:
					log.info(("Blacklisting" if addBlacklist else "Removing blacklist for")+" subreddit /r/"+sub)
					result = database.blacklist(sub, True, addBlacklist)
					results['blacklist'].append({'name': sub, 'isSubreddit': True, 'added': addBlacklist, 'result': result})

				for user in users:
					log.info(("Blacklisting" if addBlacklist else "Removing blacklist for")+" user /u/"+user)
					result = database.blacklist(user, False, addBlacklist)
					results['blacklist'].append({'name': user, 'isSubreddit': False, 'added': addBlacklist, 'result': result})
			else:
				log.info("User /u/"+author+" tried to blacklist")
				results['blacklistNot'] = True
		else:
			log.info(("Blacklisting" if addBlacklist else "Removing blacklist for")+" user /u/"+author)
			result = database.blacklist(author, False, addBlacklist)
			results['blacklist'].append({'name': author, 'isSubreddit': False, 'added': addBlacklist, 'result': result})

	return results

def MessageLinePrompt(line, author, hasAdmin):
	results = defaultdict(list)
	if line.startswith("prompt") or line.startswith("dontprompt"):
		addPrompt = True if line.startswith("prompt") else False
		subs = re.findall('(?:/?r/)(\w*)', line)
		users = re.findall('(?:/?u/)([\w-]*)', line)

		skip = False
		if len(subs) and len(users) and hasAdmin:
			sub = subs[0].lower()
			user = users[0].lower()
		elif len(subs):
			sub = subs[0].lower()
			user = author
		else:
			skip = True

		if not skip and not database.isSubredditWhitelisted(sub):
			log.info("Could not add prompt for /u/"+user+" in /r/"+sub+", not whitelisted")
			results["couldnotadd"].append({'subreddit': sub})
			skip = True
			utility.checkDeniedRequests(sub)

		if not skip:
			if database.isPrompt(user, sub):
				if not addPrompt:
					log.info("Removing prompt for /u/"+user+" in /r/"+sub)
					database.removePrompt(user, sub)
					results['prompt'].append({'name': user, 'subreddit': sub, 'added': False, 'exists': False})
				else:
					log.info("Prompt doesn't exist for /u/"+user+" in /r/"+sub)
					results['prompt'].append({'name': user, 'subreddit': sub, 'added': False, 'exists': True})
			else:
				if addPrompt:
					log.info("Adding prompt for /u/"+user+" in /r/"+sub)
					database.addPrompt(user, sub)
					results['prompt'].append({'name': user, 'subreddit': sub, 'added': True, 'exists': False})
				else:
					log.info("Prompt already exists for /u/"+user+" in /r/"+sub)
					results['prompt'].append({'name': user, 'subreddit': sub, 'added': False, 'exists': True})

	return results

def processMessages():
	messagesProcessed = 0
	try:
		for message in reddit.getMessages():
			# checks to see as some comments might be replys and non PMs
			if isinstance(message, praw.models.Message):
				messagesProcessed += 1
				replies = defaultdict(list)
				msgAuthor = str(message.author).lower()
				log.info("Parsing message from /u/{} : {}".format(msgAuthor, message.id))
				hasAdmin = msgAuthor == globals.OWNER_NAME.lower()

				for line in message.body.lower().splitlines():
					utility.combineDictLists(replies, MessageLineUpdateSubscribe(line, msgAuthor, datetime.utcfromtimestamp(message.created_utc)))
					utility.combineDictLists(replies, MessageLineRemove(line, msgAuthor))
					utility.combineDictLists(replies, MessageLineList(line, msgAuthor))
					utility.combineDictLists(replies, MessageLineDelete(line, msgAuthor))
					utility.combineDictLists(replies, MessageLineMute(line, msgAuthor, hasAdmin))
					utility.combineDictLists(replies, MessageLinePrompt(line, msgAuthor, hasAdmin))
					if hasAdmin:
						utility.combineDictLists(replies, MessageLineAddSubreddit(line))
						utility.combineDictLists(replies, MessageLineSubredditPM(line))

				sections = [
					{'key': "added", 'function': strings.confirmationSection},
					{'key': "updated", 'function': strings.updatedSubscriptionSection},
					{'key': "exist", 'function': strings.alreadySubscribedSection},
					{'key': "removed", 'function': strings.removeUpdatesConfirmationSection},
					{'key': "commentsDeleted", 'function': strings.deletedCommentSection},
					{'key': "couldnotadd", 'function': strings.couldNotSubscribeSection},
					{'key': "subredditsAdded", 'function': strings.subredditActivatedMessage},
					{'key': "alwaysPM", 'function': strings.subredditAlwaysPMMessage},
					{'key': "blacklist", 'function': strings.blacklistSection},
					{'key': "blacklistNot", 'function': strings.blacklistNotSection},
					{'key': "prompt", 'function': strings.promptSection},
				]

				strList = []
				for section in sections:
					if section['key'] in replies:
						strList.extend(section['function'](replies[section['key']]))
						strList.append("\n\n*****\n\n")

				# this is special cased since we need to pull in the subscriptions now, rather than during line processing
				if 'list' in replies:
					strList.extend(strings.yourUpdatesSection(database.getMySubscriptions(msgAuthor)))
					strList.append("\n\n*****\n\n")

				if len(strList) == 0:
					log.info("Nothing found in message")
					strList.append(strings.couldNotUnderstandSection)
					strList.append("\n\n*****\n\n")

				strList.append(strings.footer)

				reddit.markMessageRead(message)

				log.debug("Sending message to /u/"+str(message.author))
				if not reddit.replyMessage(message, ''.join(strList)):
					log.warning("Exception sending confirmation message")
			else:
				body = message.body.lower()
				if globals.SUBSCRIPTION_NAME in body or globals.UPDATE_NAME in body:
					log.debug("Marking comment as read: {}".format(message.id))
					reddit.markMessageRead(message)
	except Exception as err:
		log.warning("Exception reading messages")
		log.warning(traceback.format_exc())

	return messagesProcessed
<commit_msg>Replace non-breaking spaces<commit_after>import logging.handlers
import re
import traceback
from datetime import datetime
from collections import defaultdict

import praw

import database
import globals
import reddit
import strings
import utility

log = logging.getLogger("bot")

def MessageLineUpdateSubscribe(line, author, created):
	results = defaultdict(list)
	line = line.replace("\u00A0"," ")
	if line.startswith("updateme") or line.startswith("subscribeme") or line.startswith("http"):
		users = re.findall('(?: /?u/)([\w-]*)', line)
		subs = re.findall('(?: /?r/)(\w*)', line)
		links = re.findall('(?:reddit.com/r/\w*/comments/)(\w*)', line)
		filters = re.findall('(?:filter=)(\S*)', line)

		if len(links) != 0:
			try:
				submission = reddit.getSubmission(links[0])
				users.append(str(submission.author))
				subs.append(str(submission.subreddit))
			except Exception as err:
				log.debug("Exception parsing link")

		if len(users) != 0 and len(subs) != 0 and not (len(users) > 1 and len(subs) > 1):
			if line.startswith("updateme"):
				subscriptionTypeSingle = True
			elif line.startswith("subscribeme"):
				subscriptionTypeSingle = False
			else:
				subscriptionTypeSingle = not database.subredditDefaultSubscribe(subs[0].lower())

			if len(filters):
				filter = filters[0]
			else:
				filter = None
			if len(users) > 1:
				for user in users:
					result, data = utility.addUpdateSubscription(author, user, subs[0], created, subscriptionTypeSingle, filter)
					results[result].append(data)
			elif len(subs) > 1:
				for sub in subs:
					result, data = utility.addUpdateSubscription(author, users[0], sub, created, subscriptionTypeSingle, filter)
					results[result].append(data)
			else:
				result, data = utility.addUpdateSubscription(author, users[0], subs[0], created, subscriptionTypeSingle, filter)
				results[result].append(data)

	return results

def MessageLineRemove(line, author):
	results = defaultdict(list)
	if line.startswith("removeall"):
		log.info("Removing all subscriptions for /u/"+author)
		results['removed'].extend(database.getMySubscriptions(author))
		database.removeAllSubscriptions(author)

	elif line.startswith("remove"):
		users = re.findall('(?:/?u/)([\w-]*)', line)
		subs = re.findall('(?:/?r/)(\w*)', line)

		if len(users) != 0 and len(subs) != 0 and not (len(users) > 1 and len(subs) > 1):
			if len(users) > 1:
				for user in users:
					result, data = utility.removeSubscription(author, user, subs[0])
					results[result].append(data)
			elif len(subs) > 1:
				for sub in subs:
					result, data = utility.removeSubscription(author, users[0], sub)
					results[result].append(data)
			else:
				result, data = utility.removeSubscription(author, users[0], subs[0])
				results[result].append(data)

	return results

def MessageLineList(line, author):
	results = defaultdict(list)
	if line.startswith("mysubscriptions") or line.startswith("myupdates"):
		log.info("Listing subscriptions for /u/"+author)
		results['list'] = True

	return results

def MessageLineDelete(line, author):
	results = defaultdict(list)
	if line.startswith("deletecomment"):
		threadID = re.findall('(?: t3_)(\w*)', line)

		if len(threadID) == 0: return results

		commentID = database.deleteComment(threadID[0], author)
		if commentID:
			log.info("Deleting comment with ID %s/%s", threadID[0], commentID)
			if reddit.deleteComment(id=commentID):
				results['commentsDeleted'].append(threadID[0])
			else:
				log.warning("Could not delete comment with ID %s/%s", threadID[0], commentID)

	return results

def MessageLineAddSubreddit(line):
	results = defaultdict(list)
	if line.startswith("addsubreddit"):
		subs = re.findall('(?:/?r/)(\w*)', line)
		filters = re.findall('(?:filter=)(\S*)', line)
		if len(filters):
			filter = filters[0]
			log.debug("Found filter in addsubreddit: "+filter)
		else:
			filter = None

		for sub in subs:
			log.info("Whitelisting subreddit /r/"+sub)
			deniedRequests = database.getDeniedSubscriptions(sub.lower())

			for user in deniedRequests:
				log.info("Messaging /u/%s that their subscriptions in /r/%s have been activated", user, sub)
				strList = strings.activatingSubredditMessage(sub.lower(), deniedRequests[user])
				strList.append("\n\n*****\n\n")
				strList.append(strings.footer)
				if not reddit.sendMessage(user, strings.messageSubject(user), ''.join(strList)):
					log.warning("Could not send message to /u/%s when activating subreddit", user)

			results['subredditsAdded'].append({'subreddit': sub, 'subscribers': len(deniedRequests)})

			database.activateSubreddit(sub, line.startswith("addsubredditsub"), filter)

	return results

def MessageLineSubredditPM(line):
	results = defaultdict(list)
	if line.startswith("subredditpm"):
		subs = re.findall('(?:/?r/)(\w*)', line)
		if line.startswith("subredditpmtrue"):
			alwaysPM = True
		elif line.startswith("subredditpmfalse"):
			alwaysPM = False
		else:
			return results

		for sub in subs:
			log.info("Setting subreddit /r/"+sub+" to "+("don't " if not alwaysPM else "")+"alwaysPM")
			database.setAlwaysPMForSubreddit(sub.lower(), alwaysPM)
			results['alwaysPM'].append({'subreddit': sub, 'alwaysPM': alwaysPM})

	return results

def MessageLineMute(line, author, hasAdmin):
	results = defaultdict(list)
	if line.startswith("leavemealone") or line.startswith("talktome"):
		addBlacklist = True if line.startswith("leavemealone") else False
		subs = re.findall('(?:/?r/)(\w*)', line)
		users = re.findall('(?:/?u/)([\w-]*)', line)

		if len(subs) or len(users):
			if hasAdmin:
				for sub in subs:
					log.info(("Blacklisting" if addBlacklist else "Removing blacklist for")+" subreddit /r/"+sub)
					result = database.blacklist(sub, True, addBlacklist)
					results['blacklist'].append({'name': sub, 'isSubreddit': True, 'added': addBlacklist, 'result': result})

				for user in users:
					log.info(("Blacklisting" if addBlacklist else "Removing blacklist for")+" user /u/"+user)
					result = database.blacklist(user, False, addBlacklist)
					results['blacklist'].append({'name': user, 'isSubreddit': False, 'added': addBlacklist, 'result': result})
			else:
				log.info("User /u/"+author+" tried to blacklist")
				results['blacklistNot'] = True
		else:
			log.info(("Blacklisting" if addBlacklist else "Removing blacklist for")+" user /u/"+author)
			result = database.blacklist(author, False, addBlacklist)
			results['blacklist'].append({'name': author, 'isSubreddit': False, 'added': addBlacklist, 'result': result})

	return results

def MessageLinePrompt(line, author, hasAdmin):
	results = defaultdict(list)
	if line.startswith("prompt") or line.startswith("dontprompt"):
		addPrompt = True if line.startswith("prompt") else False
		subs = re.findall('(?:/?r/)(\w*)', line)
		users = re.findall('(?:/?u/)([\w-]*)', line)

		skip = False
		if len(subs) and len(users) and hasAdmin:
			sub = subs[0].lower()
			user = users[0].lower()
		elif len(subs):
			sub = subs[0].lower()
			user = author
		else:
			skip = True

		if not skip and not database.isSubredditWhitelisted(sub):
			log.info("Could not add prompt for /u/"+user+" in /r/"+sub+", not whitelisted")
			results["couldnotadd"].append({'subreddit': sub})
			skip = True
			utility.checkDeniedRequests(sub)

		if not skip:
			if database.isPrompt(user, sub):
				if not addPrompt:
					log.info("Removing prompt for /u/"+user+" in /r/"+sub)
					database.removePrompt(user, sub)
					results['prompt'].append({'name': user, 'subreddit': sub, 'added': False, 'exists': False})
				else:
					log.info("Prompt doesn't exist for /u/"+user+" in /r/"+sub)
					results['prompt'].append({'name': user, 'subreddit': sub, 'added': False, 'exists': True})
			else:
				if addPrompt:
					log.info("Adding prompt for /u/"+user+" in /r/"+sub)
					database.addPrompt(user, sub)
					results['prompt'].append({'name': user, 'subreddit': sub, 'added': True, 'exists': False})
				else:
					log.info("Prompt already exists for /u/"+user+" in /r/"+sub)
					results['prompt'].append({'name': user, 'subreddit': sub, 'added': False, 'exists': True})

	return results

def processMessages():
	messagesProcessed = 0
	try:
		for message in reddit.getMessages():
			# checks to see as some comments might be replys and non PMs
			if isinstance(message, praw.models.Message):
				messagesProcessed += 1
				replies = defaultdict(list)
				msgAuthor = str(message.author).lower()
				log.info("Parsing message from /u/{} : {}".format(msgAuthor, message.id))
				hasAdmin = msgAuthor == globals.OWNER_NAME.lower()

				for line in message.body.lower().splitlines():
					utility.combineDictLists(replies, MessageLineUpdateSubscribe(line, msgAuthor, datetime.utcfromtimestamp(message.created_utc)))
					utility.combineDictLists(replies, MessageLineRemove(line, msgAuthor))
					utility.combineDictLists(replies, MessageLineList(line, msgAuthor))
					utility.combineDictLists(replies, MessageLineDelete(line, msgAuthor))
					utility.combineDictLists(replies, MessageLineMute(line, msgAuthor, hasAdmin))
					utility.combineDictLists(replies, MessageLinePrompt(line, msgAuthor, hasAdmin))
					if hasAdmin:
						utility.combineDictLists(replies, MessageLineAddSubreddit(line))
						utility.combineDictLists(replies, MessageLineSubredditPM(line))

				sections = [
					{'key': "added", 'function': strings.confirmationSection},
					{'key': "updated", 'function': strings.updatedSubscriptionSection},
					{'key': "exist", 'function': strings.alreadySubscribedSection},
					{'key': "removed", 'function': strings.removeUpdatesConfirmationSection},
					{'key': "commentsDeleted", 'function': strings.deletedCommentSection},
					{'key': "couldnotadd", 'function': strings.couldNotSubscribeSection},
					{'key': "subredditsAdded", 'function': strings.subredditActivatedMessage},
					{'key': "alwaysPM", 'function': strings.subredditAlwaysPMMessage},
					{'key': "blacklist", 'function': strings.blacklistSection},
					{'key': "blacklistNot", 'function': strings.blacklistNotSection},
					{'key': "prompt", 'function': strings.promptSection},
				]

				strList = []
				for section in sections:
					if section['key'] in replies:
						strList.extend(section['function'](replies[section['key']]))
						strList.append("\n\n*****\n\n")

				# this is special cased since we need to pull in the subscriptions now, rather than during line processing
				if 'list' in replies:
					strList.extend(strings.yourUpdatesSection(database.getMySubscriptions(msgAuthor)))
					strList.append("\n\n*****\n\n")

				if len(strList) == 0:
					log.info("Nothing found in message")
					strList.append(strings.couldNotUnderstandSection)
					strList.append("\n\n*****\n\n")

				strList.append(strings.footer)

				reddit.markMessageRead(message)

				log.debug("Sending message to /u/"+str(message.author))
				if not reddit.replyMessage(message, ''.join(strList)):
					log.warning("Exception sending confirmation message")
			else:
				body = message.body.lower()
				if globals.SUBSCRIPTION_NAME in body or globals.UPDATE_NAME in body:
					log.debug("Marking comment as read: {}".format(message.id))
					reddit.markMessageRead(message)
	except Exception as err:
		log.warning("Exception reading messages")
		log.warning(traceback.format_exc())

	return messagesProcessed
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

import textwrap

class OptionDescriptor(object):
    @property
    def dataname(self):
        return "_" + self.name

    def __get__(self, instance, owner):
        if instance is None:
            return self
        return getattr(instance, self.dataname, self.default)

    def cli_argument_name(self):
        name = {"bindir": "home"}.get(self.name, self.name)
        return "--" + name.replace("_", "-")

    def cli_argument_spec(self):
        return dict(help="Julia's ``{}`` option.".format(self.cli_argument_name()))
        # TODO: parse help from `options_docs`.

class String(OptionDescriptor):
    def __init__(self, name, default=None):
        self.name = name
        self.default = default

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError(self.name)
        if value == self.default or isinstance(value, str):
            setattr(instance, self.dataname, value)
        else:
            raise ValueError(
                "Option {!r} only accepts `str`. Got: {!r}".format(self.name, value)
            )

    def _domain(self):  # used in test
        return str

class Choices(OptionDescriptor):
    def __init__(self, name, choicemap, default=None):
        self.name = name
        self.choicemap = choicemap
        self.default = default

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError(self.name)
        if value == self.default:
            setattr(instance, self.dataname, value)
        elif value in self.choicemap:
            setattr(instance, self.dataname, self.choicemap[value])
        else:
            raise ValueError(
                "Option {!r} does not accept {!r}".format(self.name, value)
            )

    def _domain(self):  # used in test
        return set(self.choicemap)

    def cli_argument_spec(self):
        return dict(
            super(Choices, self).cli_argument_spec(),
            choices=list(self.choicemap.values()),
        )

def yes_no_etc(*etc):
    choicemap = {True: "yes", False: "no", "yes": "yes", "no": "no"}
    for v in etc:
        choicemap[v] = v
    return choicemap

options_docs = """
bindir: str
    Set location of `julia` executable relative to which we find
    system image (``sys.so``).  It is inferred from `runtime` if not
    given.  Equivalent to ``--home`` of the Julia CLI.

check_bounds: {True, False, 'yes', 'no'}
    Emit bounds checks always or never (ignoring declarations).
    `True` and `False` are synonym of ``'yes'`` and ``'no'``, respectively.
    This applies to all other options.

compile: {True, False, 'yes', 'no', 'all', 'min'}
    Enable or disable JIT compiler, or request exhaustive compilation.

compiled_modules: {True, False, 'yes', 'no'}
    Enable or disable incremental precompilation of modules.

depwarn: {True, False, 'yes', 'no', 'error'}
    Enable or disable syntax and method deprecation warnings ("error"
    turns warnings into errors).

inline: {True, False, 'yes', 'no'}
    Control whether inlining is permitted, including overriding
    @inline declarations.

optimize: {0, 1, 2, 3}
    Set the optimization level (default level is 2 if unspecified or 3
    if used without a level).

sysimage: str
    Start up with the given system image file.

warn_overwrite: {True, False, 'yes', 'no'}
    Enable or disable method overwrite warnings.

min_optlevel: {0, 1, 2, 3}
    Lower bound on the optimization level.

threads: int or "auto"
    How many threads to use.
"""

class JuliaOptions(object):
    """
    Julia options validator.

    Attributes
    ----------
    """

    __doc__ = textwrap.dedent(__doc__) + options_docs

    # `options_docs` defined above must be updated when changing the
    # list of options supported by `JuliaOptions`. `test_options_docs`
    # tests that `options_docs` matches with the definition of
    # `JuliaOptions`

    sysimage = String("sysimage")
    bindir = String("bindir")
    compiled_modules = Choices("compiled_modules", yes_no_etc())
    compile = Choices("compile", yes_no_etc("all", "min"))
    depwarn = Choices("depwarn", yes_no_etc("error"))
    warn_overwrite = Choices("warn_overwrite", yes_no_etc())
    min_optlevel = Choices("min_optlevel", dict(zip(range(4), map(str, range(4)))))
    optimize = Choices("optimize", dict(zip(range(4), map(str, range(4)))))
    inline = Choices("inline", yes_no_etc())
    check_bounds = Choices("check_bounds", yes_no_etc())
    threads = Choices("threads", dict(zip(range(12), map(str, range(12)))))

    def __init__(self, **kwargs):
        unsupported = []
        for (name, value) in kwargs.items():
            if self.is_supported(name):
                setattr(self, name, value)
            else:
                unsupported.append(name)
        if unsupported:
            raise TypeError(
                "Unsupported Julia option(s): {}".format(", ".join(unsupported))
            )

    @classmethod
    def is_supported(cls, name):
        return isinstance(getattr(cls, name, None), OptionDescriptor)

    def is_specified(self, name):
        desc = getattr(self.__class__, name)
        if isinstance(desc, OptionDescriptor):
            return getattr(self, name) != desc.default
        return False

    def specified(self):
        for name in dir(self.__class__):
            if self.is_specified(name):
                yield getattr(self.__class__, name), getattr(self, name)

    def as_args(self):
        args = []
        for (desc, value) in self.specified():
            if len(desc.cli_argument_name()) == 1:
                args.append(desc.cli_argument_name() + str(value))
            else:
                args.append(desc.cli_argument_name()+"="+str(value))
        return args

    @classmethod
    def supported_options(cls):
        for name in dir(cls):
            if cls.is_supported(name):
                yield getattr(cls, name)

def parse_jl_options(options):
    """
    Parse --home and --sysimage options.

    Examples
    --------
    >>> ns = parse_jl_options(["--home", "PATH/TO/HOME"])
    >>> ns
    Namespace(home='PATH/TO/HOME', sysimage=None)
    >>> ns.home
    'PATH/TO/HOME'
    >>> parse_jl_options([])
    Namespace(home=None, sysimage=None)
    >>> parse_jl_options(["-HHOME", "--sysimage=PATH/TO/sys.so"])
    Namespace(home='HOME', sysimage='PATH/TO/sys.so')
    """
    import argparse

    def exit(*_):
        raise Exception("`exit` must not be called")

    def error(message):
        raise RuntimeError(message)

    parser = argparse.ArgumentParser()
    parser.add_argument("--home", "-H")
    parser.add_argument("--sysimage", "-J")

    parser.exit = exit
    parser.error = error
    ns, _ = parser.parse_known_args(options)
    return ns
<commit_msg>Allow arbitrary int for threads<commit_after>from __future__ import absolute_import, print_function

import textwrap

class OptionDescriptor(object):
    @property
    def dataname(self):
        return "_" + self.name

    def __get__(self, instance, owner):
        if instance is None:
            return self
        return getattr(instance, self.dataname, self.default)

    def cli_argument_name(self):
        name = {"bindir": "home"}.get(self.name, self.name)
        return "--" + name.replace("_", "-")

    def cli_argument_spec(self):
        return dict(help="Julia's ``{}`` option.".format(self.cli_argument_name()))
        # TODO: parse help from `options_docs`.

class String(OptionDescriptor):
    def __init__(self, name, default=None):
        self.name = name
        self.default = default

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError(self.name)
        if value == self.default or isinstance(value, str):
            setattr(instance, self.dataname, value)
        else:
            raise ValueError(
                "Option {!r} only accepts `str`. Got: {!r}".format(self.name, value)
            )

    def _domain(self):  # used in test
        return str

class IntEtc(OptionDescriptor):
    def __init__(self, name, *, etc={}):
        self.name = name
        self.default = etc

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError(self.name)
        elif value in {None, *self.default} or isinstance(value, int):
            setattr(instance, self.dataname, value)
        else:
            part = f" or {self.default}" if self.default else ""
            raise ValueError(
                f"Option {self.name} only accepts integers{part}. Got: {value}"
            )

    def _domain(self):
        return {int, *self.default}

class Choices(OptionDescriptor):
    def __init__(self, name, choicemap, default=None):
        self.name = name
        self.choicemap = choicemap
        self.default = default

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError(self.name)
        if value == self.default:
            setattr(instance, self.dataname, value)
        elif value in self.choicemap:
            setattr(instance, self.dataname, self.choicemap[value])
        else:
            raise ValueError(
                "Option {!r} does not accept {!r}".format(self.name, value)
            )

    def _domain(self):  # used in test
        return set(self.choicemap)

    def cli_argument_spec(self):
        return dict(
            super(Choices, self).cli_argument_spec(),
            choices=list(self.choicemap.values()),
        )

def yes_no_etc(*etc):
    choicemap = {True: "yes", False: "no", "yes": "yes", "no": "no"}
    for v in etc:
        choicemap[v] = v
    return choicemap

options_docs = """
bindir: str
    Set location of `julia` executable relative to which we find
    system image (``sys.so``).  It is inferred from `runtime` if not
    given.  Equivalent to ``--home`` of the Julia CLI.

check_bounds: {True, False, 'yes', 'no'}
    Emit bounds checks always or never (ignoring declarations).
    `True` and `False` are synonym of ``'yes'`` and ``'no'``, respectively.
    This applies to all other options.

compile: {True, False, 'yes', 'no', 'all', 'min'}
    Enable or disable JIT compiler, or request exhaustive compilation.

compiled_modules: {True, False, 'yes', 'no'}
    Enable or disable incremental precompilation of modules.

depwarn: {True, False, 'yes', 'no', 'error'}
    Enable or disable syntax and method deprecation warnings ("error"
    turns warnings into errors).

inline: {True, False, 'yes', 'no'}
    Control whether inlining is permitted, including overriding
    @inline declarations.

optimize: {0, 1, 2, 3}
    Set the optimization level (default level is 2 if unspecified or 3
    if used without a level).

sysimage: str
    Start up with the given system image file.

warn_overwrite: {True, False, 'yes', 'no'}
    Enable or disable method overwrite warnings.

min_optlevel: {0, 1, 2, 3}
    Lower bound on the optimization level.

threads: {int, 'auto'}
    How many threads to use.
"""

class JuliaOptions(object):
    """
    Julia options validator.

    Attributes
    ----------
    """

    __doc__ = textwrap.dedent(__doc__) + options_docs

    # `options_docs` defined above must be updated when changing the
    # list of options supported by `JuliaOptions`. `test_options_docs`
    # tests that `options_docs` matches with the definition of
    # `JuliaOptions`

    sysimage = String("sysimage")
    bindir = String("bindir")
    compiled_modules = Choices("compiled_modules", yes_no_etc())
    compile = Choices("compile", yes_no_etc("all", "min"))
    depwarn = Choices("depwarn", yes_no_etc("error"))
    warn_overwrite = Choices("warn_overwrite", yes_no_etc())
    min_optlevel = Choices("min_optlevel", dict(zip(range(4), map(str, range(4)))))
    optimize = Choices("optimize", dict(zip(range(4), map(str, range(4)))))
    inline = Choices("inline", yes_no_etc())
    check_bounds = Choices("check_bounds", yes_no_etc())
    threads = IntEtc("threads", etc={"auto"})

    def __init__(self, **kwargs):
        unsupported = []
        for (name, value) in kwargs.items():
            if self.is_supported(name):
                setattr(self, name, value)
            else:
                unsupported.append(name)
        if unsupported:
            raise TypeError(
                "Unsupported Julia option(s): {}".format(", ".join(unsupported))
            )

    @classmethod
    def is_supported(cls, name):
        return isinstance(getattr(cls, name, None), OptionDescriptor)

    def is_specified(self, name):
        desc = getattr(self.__class__, name)
        if isinstance(desc, OptionDescriptor):
            return getattr(self, name) != desc.default
        return False

    def specified(self):
        for name in dir(self.__class__):
            if self.is_specified(name):
                yield getattr(self.__class__, name), getattr(self, name)

    def as_args(self):
        args = []
        for (desc, value) in self.specified():
            if len(desc.cli_argument_name()) == 1:
                args.append(desc.cli_argument_name() + str(value))
            else:
                args.append(desc.cli_argument_name() + "=" + str(value))
        return args

    @classmethod
    def supported_options(cls):
        for name in dir(cls):
            if cls.is_supported(name):
                yield getattr(cls, name)

def parse_jl_options(options):
    """
    Parse --home and --sysimage options.

    Examples
    --------
    >>> ns = parse_jl_options(["--home", "PATH/TO/HOME"])
    >>> ns
    Namespace(home='PATH/TO/HOME', sysimage=None)
    >>> ns.home
    'PATH/TO/HOME'
    >>> parse_jl_options([])
    Namespace(home=None, sysimage=None)
    >>> parse_jl_options(["-HHOME", "--sysimage=PATH/TO/sys.so"])
    Namespace(home='HOME', sysimage='PATH/TO/sys.so')
    """
    import argparse

    def exit(*_):
        raise Exception("`exit` must not be called")

    def error(message):
        raise RuntimeError(message)

    parser = argparse.ArgumentParser()
    parser.add_argument("--home", "-H")
    parser.add_argument("--sysimage", "-J")

    parser.exit = exit
    parser.error = error
    ns, _ = parser.parse_known_args(options)
    return ns
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Print the last time a reviewer(bot) left a comment."""

import argparse
import calendar
import collections
import datetime
import json
import sys
import yaml

import requests

try:
    # Disable InsecurePlatformWarning warnings as documented here
    # https://github.com/kennethreitz/requests/issues/2214
    from requests.packages.urllib3.exceptions import InsecurePlatformWarning
    from requests.packages.urllib3.exceptions import InsecureRequestWarning
    requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
except ImportError:
    # If there's an import error, then urllib3 may be packaged
    # separately, so apply it there too
    import urllib3
    from urllib3.exceptions import InsecurePlatformWarning
    from urllib3.exceptions import InsecureRequestWarning
    urllib3.disable_warnings(InsecurePlatformWarning)
    urllib3.disable_warnings(InsecureRequestWarning)

TIME_FORMAT = "%Y-%m-%d %H:%M:%S"

class Comment(object):
    date = None
    number = None
    subject = None
    now = None
    gerrit_url = None

    def __init__(self, date, number, subject, message):
        super(Comment, self).__init__()
        self.date = date
        self.number = number
        self.subject = subject
        self.message = message
        self.now = datetime.datetime.utcnow().replace(microsecond=0)

    def __str__(self):
        return ("%s (%s old) %s/%s '%s' " % (
            self.date.strftime(TIME_FORMAT),
            self.age(),
            self.gerrit_url,
            self.number, self.subject))

    def age(self):
        return self.now - self.date

    def __le__(self, other):
        # self < other
        return self.date < other.date

    def __repr__(self):
        # for sorting
        return repr((self.date, self.number))

def get_comments(change, name):
    """Generator that returns all comments by name on a given change."""
    body = None
    for message in change['messages']:
        if 'author' in message and message['author']['name'] == name:
            if (message['message'].startswith("Uploaded patch set") and
               len(message['message'].split()) is 4):
                # comment is auto created from posting a new patch
                continue
            date = message['date']
            body = message['message']
            # https://review.openstack.org/Documentation/rest-api.html#timestamp
            # drop nanoseconds
            date = date.split('.')[0]
            date = datetime.datetime.strptime(date, TIME_FORMAT)
            yield date, body

def query_gerrit(gerrit_url, name, count, project, verify=True):
    # Include review messages in query
    search = "reviewer:\"%s\"" % name
    if project:
        search = search + (" AND project:\"%s\"" % project)
    query = ("%s/changes/?q=%s&"
             "o=MESSAGES" % (gerrit_url, search))
    r = requests.get(query, verify=verify)
    try:
        changes = json.loads(r.text[4:])
    except ValueError:
        print "query: '%s' failed with:\n%s" % (query, r.text)
        sys.exit(1)

    comments = []
    for change in changes:
        for date, message in get_comments(change, name):
            if date is None:
                # no comments from reviewer yet. This can happen since
                # 'Uploaded patch set X.' is considered a comment.
                continue
            comments.append(Comment(date, change['_number'],
                                    change['subject'], message))

    return sorted(comments, key=lambda comment: comment.date,
                  reverse=True)[0:count]

def vote(comment, success, failure, log=False):
    for line in comment.message.splitlines():
        if line.startswith("* ") or line.startswith("- "):
            job = line.split(' ')[1]
            if " : SUCCESS" in line:
                success[job] += 1
                if log:
                    print line
            if " : FAILURE" in line:
                failure[job] += 1
                if log:
                    print line

def generate_report(gerrit_url, name, count, project, verify):
    result = {'name': name, 'project': project}
    success = collections.defaultdict(int)
    failure = collections.defaultdict(int)

    comments = query_gerrit(gerrit_url, name, count, project, verify)

    if len(comments) == 0:
        print "didn't find anything"
        return None

    print "last seen: %s (%s old)" % (comments[0].date, comments[0].age())
    result['last'] = epoch(comments[0].date)

    for comment in comments:
        vote(comment, success, failure)

    total = sum(success.values()) + sum(failure.values())
    if total > 0:
        success_rate = str(int(sum(success.values()) /
                               float(total) * 100)) + "%"
        result['rate'] = success_rate
        print "success rate: %s" % success_rate
    return result

def print_last_comments(gerrit_url, name, count, print_message, project,
                        votes, verify):
    success = collections.defaultdict(int)
    failure = collections.defaultdict(int)

    comments = query_gerrit(gerrit_url, name, count, project, verify)

    message = "last %s comments from '%s'" % (count, name)
    if project:
        message += " on project '%s'" % project
    print message
    # sort by time
    for i, comment in enumerate(comments):
        print "[%d] %s" % (i, comment)
        if print_message:
            print "message: \"%s\"" % comment.message
            print
        if votes:
            vote(comment, success, failure, log=True)

    if votes:
        print "success count by job:"
        for job in success.iterkeys():
            print "* %s: %d" % (job, success[job])
        print "failure count by job:"
        for job in failure.iterkeys():
            print "* %s: %d" % (job, failure[job])

def epoch(timestamp):
    return int(calendar.timegm(timestamp.timetuple()))

def main():
    parser = argparse.ArgumentParser(description='list most recent comment by '
                                     'reviewer')
    parser.add_argument('-n', '--name',
                        default="Elastic Recheck",
                        help='unique gerrit name of the reviewer')
    parser.add_argument('-c', '--count',
                        default=10,
                        type=int,
                        help='Max number of results to return')
    parser.add_argument('-f', '--file',
                        default=None,
                        help='yaml file containing list of names to search on'
                             'project: name'
                             ' (overwrites -p and -n)')
    parser.add_argument('-m', '--message',
                        action='store_true',
                        help='print comment message')
    parser.add_argument('-v', '--votes',
                        action='store_true',
                        help=('Look in comments for CI Jobs and detect '
                              'SUCCESS/FAILURE'))
    parser.add_argument('--json',
                        nargs='?',
                        const='lastcomment.json',
                        help=("Generate report to be stored in the json file "
                              "specified here. Ignores -v and -m "
                              "(default: 'lastcomment.json')"))
    parser.add_argument('-p', '--project',
                        help='only list hits for a specific project')
    parser.add_argument('-g', '--gerrit-url',
                        default='https://review.openstack.org/',
                        help='Gerrit server http/https url')
    parser.add_argument('--no-verify',
                        action='store_false',
                        help='Ignore gerrit server certificate validity')

    args = parser.parse_args()
    names = {args.project: [args.name]}
    if args.file:
        with open(args.file) as f:
            names = yaml.load(f)

    if args.json:
        print "generating report %s" % args.json
        print "report is over last %s comments" % args.count
        report = {}
        timestamp = epoch(datetime.datetime.utcnow())
        report['timestamp'] = timestamp
        report['rows'] = []

    for project in names:
        print 'Checking project: %s' % project
        for name in names[project]:
            print 'Checking name: %s' % name
            try:
                if args.json:
                    report['rows'].append(generate_report(args.gerrit_url,
                        name, args.count, project, args.no_verify))
                else:
                    print_last_comments(args.gerrit_url, name, args.count,
                                        args.message, project, args.votes,
                                        args.no_verify)
            except Exception as e:
                print e
                pass

    if args.json:
        with open(args.json, 'w') as f:
            json.dump(report, f)

if __name__ == "__main__":
    main()
<commit_msg>Look up gerrit account(s) before running queries<commit_after>#!/usr/bin/env python

"""Print the last time a reviewer(bot) left a comment."""

import argparse
import calendar
import collections
import datetime
import json
import sys
import urllib
import yaml

import requests

try:
    # Disable InsecurePlatformWarning warnings as documented here
    # https://github.com/kennethreitz/requests/issues/2214
    from requests.packages.urllib3.exceptions import InsecurePlatformWarning
    from requests.packages.urllib3.exceptions import InsecureRequestWarning
    requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
except ImportError:
    # If there's an import error, then urllib3 may be packaged
    # separately, so apply it there too
    import urllib3
    from urllib3.exceptions import InsecurePlatformWarning
    from urllib3.exceptions import InsecureRequestWarning
    urllib3.disable_warnings(InsecurePlatformWarning)
    urllib3.disable_warnings(InsecureRequestWarning)

TIME_FORMAT = "%Y-%m-%d %H:%M:%S"

class Account(object):
    _account_id = None
    name = None
    email = None
    username = None

    def __init__(self, account_info={}):
        super(Account, self).__init__()
        self._account_id = account_info.get('_account_id', 0)
        self.name = account_info.get('name', None)
        self.email = account_info.get('email', None)
        self.username = account_info.get('username', None)

    def __str__(self):
        a = []
        if self.name:
            a.append("'%s'" % self.name)
        if self.username:
            a.append(self.username)
        if self.email:
            a.append("<%s>" % self.email)
        if a:
            return "ID %s (%s)" % (self._account_id, ", ".join(a))
        else:
            return "ID %s" % self._account_id

    def __le__(self, other):
        # self < other
        return self._account_id < other._account_id

class Comment(object):
    date = None
    number = None
    subject = None
    now = None
    gerrit_url = None

    def __init__(self, date, number, subject, message):
        super(Comment, self).__init__()
        self.date = date
        self.number = number
        self.subject = subject
        self.message = message
        self.now = datetime.datetime.utcnow().replace(microsecond=0)

    def __str__(self):
        return ("%s (%s old) %s/%s '%s' " % (
            self.date.strftime(TIME_FORMAT),
            self.age(),
            self.gerrit_url,
            self.number, self.subject))

    def age(self):
        return self.now - self.date

    def __le__(self, other):
        # self < other
        return self.date < other.date

    def __repr__(self):
        # for sorting
        return repr((self.date, self.number))

def get_comments(change, account):
    """Generator that returns all comments by account on a given change."""
    body = None
    for message in change['messages']:
        if ('author' in message and
            '_account_id' in message['author'] and
            message['author']['_account_id'] == account._account_id):

            if (message['message'].startswith("Uploaded patch set") and
                len(message['message'].split()) is 4):
                # comment is auto created from posting a new patch
                continue
            date = message['date']
            body = message['message']
            # https://review.openstack.org/Documentation/rest-api.html#timestamp
            # drop nanoseconds
            date = date.split('.')[0]
            date = datetime.datetime.strptime(date, TIME_FORMAT)
            yield date, body

def query_gerrit(gerrit_url, account, count, project, verify=True):
    # Include review messages in query
    search = "reviewer:{%s}" % account._account_id
    if project:
        search = search + (" AND project:{%s}" % project)
    query = ("%s/changes/?q=%s&"
             "o=MESSAGES&pp=0" % (gerrit_url, urllib.quote_plus(search)))
    r = requests.get(query, verify=verify)
    try:
        changes = json.loads(r.text[4:])
    except ValueError:
        print "query: '%s' failed with:\n%s" % (query, r.text)
        sys.exit(1)

    comments = []
    for change in changes:
        for date, message in get_comments(change, account):
            if date is None:
                # no comments from reviewer yet. This can happen since
                # 'Uploaded patch set X.' is considered a comment.
                continue
            comments.append(Comment(date, change['_number'],
                                    change['subject'], message))

    return sorted(comments, key=lambda comment: comment.date,
                  reverse=True)[0:count]

def lookup_account(gerrit_url, account_id, verify=True):
    """Look up account information.

    An account "ID" can be any uniquely identifying account information. See the
    API documentation for more information:

    https://review.openstack.org/Documentation/rest-api-accounts.html#account-id
    """

    query = "%s/accounts/%s?pp=0" % (gerrit_url, urllib.quote_plus(account_id))
    r = requests.get(query, verify=verify)
    try:
        return Account(json.loads(r.text[4:]))
    except ValueError:
        print "account lookup for '%s' failed with:\n%s" % (account_id, r.text)
        sys.exit(1)

def vote(comment, success, failure, log=False):
    for line in comment.message.splitlines():
        if line.startswith("* ") or line.startswith("- "):
            job = line.split(' ')[1]
            if " : SUCCESS" in line:
                success[job] += 1
                if log:
                    print line
            if " : FAILURE" in line:
                failure[job] += 1
                if log:
                    print line

def generate_report(gerrit_url, account, count, project, verify):
    result = {'account': account.__dict__, 'project': project}
    success = collections.defaultdict(int)
    failure = collections.defaultdict(int)

    comments = query_gerrit(gerrit_url, account, count, project, verify)

    if len(comments) == 0:
        print "didn't find anything"
        return None

    print "last seen: %s (%s old)" % (comments[0].date, comments[0].age())
    result['last'] = epoch(comments[0].date)

    for comment in comments:
        vote(comment, success, failure)

    total = sum(success.values()) + sum(failure.values())
    if total > 0:
        success_rate = str(int(sum(success.values()) /
                               float(total) * 100)) + "%"
        result['rate'] = success_rate
        print "success rate: %s" % success_rate
    return result

def print_last_comments(gerrit_url, account, count, print_message, project,
                        votes, verify):
    success = collections.defaultdict(int)
    failure = collections.defaultdict(int)

    comments = query_gerrit(gerrit_url, account, count, project, verify)

    message = "last %s comments from '%s'" % (count, account.name)
    if project:
        message += " on project '%s'" % project
    print message
    # sort by time
    for i, comment in enumerate(comments):
        print "[%d] %s" % (i, comment)
        if print_message:
            print "message: \"%s\"" % comment.message
            print
        if votes:
            vote(comment, success, failure, log=True)

    if votes:
        print "success count by job:"
        for job in success.iterkeys():
            print "* %s: %d" % (job, success[job])
        print "failure count by job:"
        for job in failure.iterkeys():
            print "* %s: %d" % (job, failure[job])

def epoch(timestamp):
    return int(calendar.timegm(timestamp.timetuple()))

def main():
    parser = argparse.ArgumentParser(description='list most recent comment by '
                                     'reviewer')
    parser.add_argument('-n', '--name',
                        default="Elastic Recheck",
                        help='unique gerrit name of the reviewer')
    parser.add_argument('-c', '--count',
                        default=10,
                        type=int,
                        help='Max number of results to return')
    parser.add_argument('-f', '--file',
                        default=None,
                        help='yaml file containing list of names to search on'
                             'project: name'
                             ' (overwrites -p and -n)')
    parser.add_argument('-m', '--message',
                        action='store_true',
                        help='print comment message')
    parser.add_argument('-v', '--votes',
                        action='store_true',
                        help=('Look in comments for CI Jobs and detect '
                              'SUCCESS/FAILURE'))
    parser.add_argument('--json',
                        nargs='?',
                        const='lastcomment.json',
                        help=("Generate report to be stored in the json file "
                              "specified here. Ignores -v and -m "
                              "(default: 'lastcomment.json')"))
    parser.add_argument('-p', '--project',
                        help='only list hits for a specific project')
    parser.add_argument('-g', '--gerrit-url',
                        default='https://review.openstack.org/',
                        help='Gerrit server http/https url')
    parser.add_argument('--no-verify',
                        action='store_false',
                        help='Ignore gerrit server certificate validity')

    args = parser.parse_args()
    names = {args.project: [args.name]}
    accounts = {}
    if args.file:
        with open(args.file) as f:
            names = yaml.load(f)

    for project in names:
        for id in names[project]:
            if id in accounts:
                continue
            accounts[id] = lookup_account(args.gerrit_url, id, args.no_verify)

    if args.json:
        print "generating report %s" % args.json
        print "report is over last %s comments" % args.count
        report = {}
        timestamp = epoch(datetime.datetime.utcnow())
        report['timestamp'] = timestamp
        report['rows'] = []

    for project in names:
        print 'Checking project: %s' % project
        for name in names[project]:
            account = accounts[name]
            print 'Checking account: %s' % account
            try:
                if args.json:
                    report['rows'].append(generate_report(args.gerrit_url,
                        account, args.count, project, args.no_verify))
                else:
                    print_last_comments(args.gerrit_url, account, args.count,
                                        args.message, project, args.votes,
                                        args.no_verify)
            except Exception as e:
                print e
                pass

    if args.json:
        with open(args.json, 'w') as f:
            json.dump(report, f)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># GrowthCurve.py
# Bacteria growth curve class to extract parameters
# Use with Phenotype MicroArray analysis pipeline
#
# Author: Daniel A Cuevas
# Created on 21 Nov 2013
# Updated on 03 Apr 2017

from __future__ import absolute_import, division, print_function
import PMUtil as util
import pylab as py
import scipy.optimize as optimize
from numpy import trapz

class GrowthCurve:
    """Bacteria growth curve class"""
    def __init__(self, data, sample, rep, well, newGrowth=0):
        # data format
        #   Pandas GroupBy group object
        #   Indices: sample rep well time
        #   Value: OD reading

        self.rawcurve = data.values  # OD values
        self.time = py.array(data.index.get_level_values("time"))
        # Lowest y0 is chosen
        # Instances of condensation issue at beginning of experiment can cause
        # high OD values
        self.y0 = py.amin(self.rawcurve[0:3])
        self.asymptote = self.__calcAsymptote()
        self.maxGrowthRate, self.mgrTime = self.__calcMGR()
        self.y0, self.asymptote, self.maxGrowthRate, self.lag = (
            self.__calcParameters(
                (self.y0, self.asymptote, self.maxGrowthRate, 0.01),
                self.time, self.rawcurve, sample, rep, well)
        )

        self.dataLogistic = logistic(self.time,
                                     self.y0,
                                     self.asymptote,
                                     self.maxGrowthRate,
                                     self.lag)
        if newGrowth == 2:
            self.growthLevel = calcNewGrowth2(self.dataLogistic,
                                              self.asymptote,
                                              self.y0)
        elif newGrowth == 1:
            self.growthLevel = calcNewGrowth(self.dataLogistic,
                                             self.asymptote,
                                             self.y0)
        else:
            self.growthLevel = calcGrowth(self.dataLogistic, self.asymptote)
        self.glScaled = calcGrowth2(self.dataLogistic, self.asymptote)
        self.expGrowth = calcExpGrowth(self.maxGrowthRate, self.asymptote)

        self.auc_raw = calcAUCData(self.rawcurve, self.time)
        self.auc_rshift = calcShiftAUC(self.auc_raw, self.y0, self.time[-1])
        self.auc_log = calcAUC(self.rawcurve, self.y0, self.lag,
                               self.maxGrowthRate, self.asymptote, self.time)
        self.auc_lshift = calcShiftAUC(self.auc_log, self.y0, self.time[-1])

        self.growthClass = growthClass(self.growthLevel)
        self.sse = sum((self.dataLogistic - self.rawcurve) ** 2)
        self.mse = self.sse / len(self.time)

    def __calcParameters(self, y0, t, raw, sample, rep, well):
        """Perform curve-fitting optimization to obtain parameters"""
        # Calculate upper bounds
        # y0[0] = start OD
        # y0[1] = A
        # y0[2] = MGR
        # y0[3] = lag (set at 0.01)
        a_ub = y0[1] + (y0[0] / 2)  # A bounded by estimated A plus half
        mgr_ub = y0[2] + (y0[2] * 0.10)  # MGR bounded by 110% estimated MGR
        lag_ub = t[-1]  # Lag bounded by final time
        try:
            results = optimize.minimize(self.__logisticSSE, y0, args=(t, raw),
                                        bounds=((0, None),
                                                (0.001, a_ub),
                                                (0, mgr_ub),
                                                (0, lag_ub)),
                                        method="L-BFGS-B")
        except RuntimeError as e:
            util.printStatus(e)
            util.printStatus(self.rawcurve)
            util.exitScript()

        if not results.success:
            util.printStatus("*" * 55)
            util.printStatus("CurveFit Unsuccessful")
            util.printStatus(results.message)
            util.printStatus("*" * 55)

        # Check if max growth rate is reasonable compared to asymptote
        # Curve should not be able to reach asymptote in less than 0.1 hrs
        # max growth rate * 0.1 should not be > A
        if results.x[2] * 0.1 > results.x[1]:
            # If this is true, set max growth rate to A / 1hr
            util.printStatus("{}\t{}\t{}".format(sample, rep, well))
            util.printStatus("Max growth rate ({:.3f}) * 0.1hr > Asymptote "
                             "({:.3f}). Setting to MGR to  A / 1hr".format(
                             results.x[2], results.x[1]))
            results.x[2] = results.x[1]

        return results.x

    def __logisticSSE(self, params, t, y):
        y0, a, mgr, l = params
        return py.sum((logistic(t, y0, a, mgr, l) - y) ** 2)

    def __calcAsymptote(self):
        """Obtain the initial guess of the highest OD reading"""
        # Calculate asymptote using a sliding window of 3 data points
        stop = len(self.time) - 3
        maxA = -1
        for idx in range(1, stop):
            av = py.mean(self.rawcurve[idx:idx + 3])
            if av > maxA:
                maxA = av
        return maxA

    def __calcMGR(self):
        """Obtain the initial guess of the max growth"""
        # Calculate max growth rate using a sliding window of 4 data points
        stop = len(self.time) - 3
        maxGR = 0
        t = 0
        for idx in range(0, stop):
            # Growth rate calculation:
            # (log(i+3) - log(i)) / (time(i+3) - time(i))

            # Be sure to check for zeros
            ya = self.rawcurve[idx]
            yb = self.rawcurve[idx + 2]
            if ya <= 0:
                ya = 0.001
            if yb <= 0:
                yb = 0.001
            gr = ((py.log(yb) - py.log(ya)) /
                  (self.time[idx + 2] - self.time[idx]))
            if idx == 0 or gr > maxGR:
                maxGR = gr
                t = self.time[idx + 1]  # Midpoint time value

        return maxGR, t

def calcGrowth(logistic, asym):
    """
    Calculate growth level using an adjusted harmonic mean
    using a logistic model and its asymptote
    """
    return len(logistic) / py.sum((1 / (logistic + asym)))

def calcNewGrowth(logistic, asym, y0):
    """
    Calculate growth level using an adjusted harmonic mean
    using a logistic model, its asymptote, and its starting OD value
    """
    diff = asym - y0
    return len(logistic) / py.sum((1 / (logistic + diff)))

def calcNewGrowth2(logistic, asym, y0):
    """
    Calculate growth level using an adjusted harmonic mean
    using a logistic model, its asymptote, and its starting OD value
    """
    amplitude = asym - y0
    diff = logistic - y0
    return len(logistic) / py.sum((1 / (amplitude + diff)))

def calcGrowth2(logistic, asym):
    """
    Calculate growth level using an adjusted harmonic mean
    using a logistic model and its asymptote
    """
    return (len(logistic) / (asym * py.sum((1 / (logistic + asym))))) - 1

def calcExpGrowth(mgr, asym):
    """Calculate exponential growth value 'r'"""
    return 4 * mgr / asym

def calcAUC(data, y0, lag, mgr, asym, time):
    """
    Calculate the area under the curve of the logistic function
    using its integrated formula
    [ A( [A-y0] log[ exp( [4m(l-t)/A]+2 )+1 ]) / 4m ] + At
    """

    # First check that max growth rate is not zero
    # If so, calculate using the data instead of the equation
    if mgr == 0:
        auc = calcAUCData(data, time)
    else:
        timeS = time[0]
        timeE = time[-1]
        t1 = asym - y0
        #try:
        t2_s = py.log(py.exp((4 * mgr * (lag - timeS) / asym) + 2) + 1)
        t2_e = py.log(py.exp((4 * mgr * (lag - timeE) / asym) + 2) + 1)
        #except RuntimeWarning as rw:
            # Exponent is too large, setting to 10^3
        #    newexp = 1000
        #    t2_s = py.log(newexp + 1)
        #    t2_e = py.log(newexp + 1)
        t3 = 4 * mgr
        t4_s = asym * timeS
        t4_e = asym * timeE

        start = (asym * (t1 * t2_s) / t3) + t4_s
        end = (asym * (t1 * t2_e) / t3) + t4_e
        auc = end - start

    if py.absolute(auc) == float('Inf'):
        x = py.diff(time)
        auc = py.sum(x * data[1:])
    return auc

def calcAUCData(data, time):
    """Calculate the area under the curve based on data values"""
    ld = len(data)
    lt = len(time)
    if ld != lt:
        raise ValueError("len(data) {} != len(time) {}".format(ld, lt))
    try:
        return trapz(data, time)
    except Exception:
        raise

def calcShiftAUC(auc, y0, tF):
    """
    Calculate the area under the curve minus
    the area below the starting OD
    """
    return auc - (y0 * tF)

def growthClass(gLevel):
    """Determine growth class based on growth level"""
    if gLevel >= 0.75:
        return "+++"
    elif gLevel >= 0.50:
        return "++"
    elif gLevel >= 0.35:
        return "+"
    elif gLevel >= 0.25:
        return "-"
    else:
        return "--"

def logistic(t, y0, a, mgr, l):
    """Logistic modeling"""
    startOD = y0
    exponent = ((4 * mgr / a) * (l - t)) + 2
    try:
        denom = 1 + py.exp(exponent)
        lg = startOD + ((a - startOD) / denom)
    except RuntimeWarning as rw:
        util.printStatus("*" * 55)
        util.printStatus("RuntimeWarning: {}".format(rw))
        util.printStatus("   Exponent value for logistic "
                         "equation is {:.3f}".format(exponent[0]))
        util.printStatus("   This produces a large value in the denominator "
                         "of the logistic equation, probably due to a small "
                         "asymptote value and large max growth rate")
        util.printStatus("   Now setting denominator to value of 10^3")
        util.printStatus("   Predicted parameters:")
        util.printStatus("      y0: {:.3f}".format(y0))
        util.printStatus("      Lag: {:.3f}".format(l))
        util.printStatus("      MGR: {:.3f}".format(mgr))
        util.printStatus("      A: {:.3f}".format(a))
        util.printStatus("*" * 55)
        newdenom = []
        for d in denom:
            if d > 500:
                newdenom.append(500)
            else:
                newdenom.append(d)
        denom = py.array(newdenom)
        lg = startOD + ((a - startOD) / denom)

    return lg
<commit_msg>Exponent check. - (Fix) Bug fix during exponent check for out of bounds error.<commit_after># GrowthCurve.py
# Bacteria growth curve class to extract parameters
# Use with Phenotype MicroArray analysis pipeline
#
# Author: Daniel A Cuevas
# Created on 21 Nov 2013
# Updated on 03 Apr 2017

from __future__ import absolute_import, division, print_function
import PMUtil as util
import pylab as py
import scipy.optimize as optimize
from numpy import trapz

class GrowthCurve:
    """Bacteria growth curve class"""
    def __init__(self, data, sample, rep, well, newGrowth=0):
        # data format
        #   Pandas GroupBy group object
        #   Indices: sample rep well time
        #   Value: OD reading

        self.rawcurve = data.values  # OD values
        self.time = py.array(data.index.get_level_values("time"))
        # Lowest y0 is chosen
        # Instances of condensation issue at beginning of experiment can cause
        # high OD values
        self.y0 = py.amin(self.rawcurve[0:3])
        self.asymptote = self.__calcAsymptote()
        self.maxGrowthRate, self.mgrTime = self.__calcMGR()
        self.y0, self.asymptote, self.maxGrowthRate, self.lag = (
            self.__calcParameters(
                (self.y0, self.asymptote, self.maxGrowthRate, 0.01),
                self.time, self.rawcurve, sample, rep, well)
        )

        self.dataLogistic = logistic(self.time,
                                     self.y0,
                                     self.asymptote,
                                     self.maxGrowthRate,
                                     self.lag)
        if newGrowth == 2:
            self.growthLevel = calcNewGrowth2(self.dataLogistic,
                                              self.asymptote,
                                              self.y0)
        elif newGrowth == 1:
            self.growthLevel = calcNewGrowth(self.dataLogistic,
                                             self.asymptote,
                                             self.y0)
        else:
            self.growthLevel = calcGrowth(self.dataLogistic, self.asymptote)
        self.glScaled = calcGrowth2(self.dataLogistic, self.asymptote)
        self.expGrowth = calcExpGrowth(self.maxGrowthRate, self.asymptote)

        self.auc_raw = calcAUCData(self.rawcurve, self.time)
        self.auc_rshift = calcShiftAUC(self.auc_raw, self.y0, self.time[-1])
        self.auc_log = calcAUC(self.rawcurve, self.y0, self.lag,
                               self.maxGrowthRate, self.asymptote, self.time)
        self.auc_lshift = calcShiftAUC(self.auc_log, self.y0, self.time[-1])

        self.growthClass = growthClass(self.growthLevel)
        self.sse = sum((self.dataLogistic - self.rawcurve) ** 2)
        self.mse = self.sse / len(self.time)

    def __calcParameters(self, y0, t, raw, sample, rep, well):
        """Perform curve-fitting optimization to obtain parameters"""
        # Calculate upper bounds
        # y0[0] = start OD
        # y0[1] = A
        # y0[2] = MGR
        # y0[3] = lag (set at 0.01)
        a_ub = y0[1] + (y0[0] / 2)  # A bounded by estimated A plus half
        mgr_ub = y0[2] + (y0[2] * 0.10)  # MGR bounded by 110% estimated MGR
        lag_ub = t[-1]  # Lag bounded by final time
        try:
            results = optimize.minimize(self.__logisticSSE, y0, args=(t, raw),
                                        bounds=((0, None),
                                                (0.001, a_ub),
                                                (0, mgr_ub),
                                                (0, lag_ub)),
                                        method="L-BFGS-B")
        except RuntimeError as e:
            util.printStatus(e)
            util.printStatus(self.rawcurve)
            util.exitScript()

        if not results.success:
            util.printStatus("*" * 55)
            util.printStatus("CurveFit Unsuccessful")
            util.printStatus(results.message)
            util.printStatus("*" * 55)

        # Check if max growth rate is reasonable compared to asymptote
        # Curve should not be able to reach asymptote in less than 0.1 hrs
        # max growth rate * 0.1 should not be > A
        if results.x[2] * 0.1 > results.x[1]:
            # If this is true, set max growth rate to A / 1hr
            util.printStatus("{}\t{}\t{}".format(sample, rep, well))
            util.printStatus("Max growth rate ({:.3f}) * 0.1hr > Asymptote "
                             "({:.3f}). Setting to MGR to  A / 1hr".format(
                             results.x[2], results.x[1]))
            results.x[2] = results.x[1]

        return results.x

    def __logisticSSE(self, params, t, y):
        y0, a, mgr, l = params
        return py.sum((logistic(t, y0, a, mgr, l) - y) ** 2)

    def __calcAsymptote(self):
        """Obtain the initial guess of the highest OD reading"""
        # Calculate asymptote using a sliding window of 3 data points
        stop = len(self.time) - 3
        maxA = -1
        for idx in range(1, stop):
            av = py.mean(self.rawcurve[idx:idx + 3])
            if av > maxA:
                maxA = av
        return maxA

    def __calcMGR(self):
        """Obtain the initial guess of the max growth"""
        # Calculate max growth rate using a sliding window of 4 data points
        stop = len(self.time) - 3
        maxGR = 0
        t = 0
        for idx in range(0, stop):
            # Growth rate calculation:
            # (log(i+3) - log(i)) / (time(i+3) - time(i))

            # Be sure to check for zeros
            ya = self.rawcurve[idx]
            yb = self.rawcurve[idx + 2]
            if ya <= 0:
                ya = 0.001
            if yb <= 0:
                yb = 0.001
            gr = ((py.log(yb) - py.log(ya)) /
                  (self.time[idx + 2] - self.time[idx]))
            if idx == 0 or gr > maxGR:
                maxGR = gr
                t = self.time[idx + 1]  # Midpoint time value

        return maxGR, t

def calcGrowth(logistic, asym):
    """
    Calculate growth level using an adjusted harmonic mean
    using a logistic model and its asymptote
    """
    return len(logistic) / py.sum((1 / (logistic + asym)))

def calcNewGrowth(logistic, asym, y0):
    """
    Calculate growth level using an adjusted harmonic mean
    using a logistic model, its asymptote, and its starting OD value
    """
    diff = asym - y0
    return len(logistic) / py.sum((1 / (logistic + diff)))

def calcNewGrowth2(logistic, asym, y0):
    """
    Calculate growth level using an adjusted harmonic mean
    using a logistic model, its asymptote, and its starting OD value
    """
    amplitude = asym - y0
    diff = logistic - y0
    return len(logistic) / py.sum((1 / (amplitude + diff)))

def calcGrowth2(logistic, asym):
    """
    Calculate growth level using an adjusted harmonic mean
    using a logistic model and its asymptote
    """
    return (len(logistic) / (asym * py.sum((1 / (logistic + asym))))) - 1

def calcExpGrowth(mgr, asym):
    """Calculate exponential growth value 'r'"""
    return 4 * mgr / asym

def calcAUC(data, y0, lag, mgr, asym, time):
    """
    Calculate the area under the curve of the logistic function
    using its integrated formula
    [ A( [A-y0] log[ exp( [4m(l-t)/A]+2 )+1 ]) / 4m ] + At
    """

    # First check that max growth rate is not zero
    # If so, calculate using the data instead of the equation
    if mgr == 0:
        auc = calcAUCData(data, time)
    else:
        timeS = time[0]
        timeE = time[-1]
        t1 = asym - y0
        #try:
        t2_s = py.log(py.exp((4 * mgr * (lag - timeS) / asym) + 2) + 1)
        t2_e = py.log(py.exp((4 * mgr * (lag - timeE) / asym) + 2) + 1)
        #except RuntimeWarning as rw:
            # Exponent is too large, setting to 10^3
        #    newexp = 1000
        #    t2_s = py.log(newexp + 1)
        #    t2_e = py.log(newexp + 1)
        t3 = 4 * mgr
        t4_s = asym * timeS
        t4_e = asym * timeE

        start = (asym * (t1 * t2_s) / t3) + t4_s
        end = (asym * (t1 * t2_e) / t3) + t4_e
        auc = end - start

    if py.absolute(auc) == float('Inf'):
        x = py.diff(time)
        auc = py.sum(x * data[1:])
    return auc

def calcAUCData(data, time):
    """Calculate the area under the curve based on data values"""
    ld = len(data)
    lt = len(time)
    if ld != lt:
        raise ValueError("len(data) {} != len(time) {}".format(ld, lt))
    try:
        return trapz(data, time)
    except Exception:
        raise

def calcShiftAUC(auc, y0, tF):
    """
    Calculate the area under the curve minus
    the area below the starting OD
    """
    return auc - (y0 * tF)

def growthClass(gLevel):
    """Determine growth class based on growth level"""
    if gLevel >= 0.75:
        return "+++"
    elif gLevel >= 0.50:
        return "++"
    elif gLevel >= 0.35:
        return "+"
    elif gLevel >= 0.25:
        return "-"
    else:
        return "--"

def logistic(t, y0, a, mgr, l):
    """Logistic modeling"""
    startOD = y0
    exponent = ((4 * mgr / a) * (l - t)) + 2
    try:
        denom = 1 + py.exp(exponent)
        lg = startOD + ((a - startOD) / denom)
    except RuntimeWarning as rw:
        util.printStatus("*" * 55)
        util.printStatus("RuntimeWarning: {}".format(rw))
        util.printStatus("   Exponent value for logistic "
                         "equation is {:.3f}".format(exponent[0]))
        util.printStatus("   This produces a large value in the denominator "
                         "of the logistic equation, probably due to a small "
                         "asymptote value and large max growth rate")
        util.printStatus("   Now setting denominator to value of 10^3")
        util.printStatus("   Predicted parameters:")
        util.printStatus("      y0: {:.3f}".format(y0))
        util.printStatus("      Lag: {:.3f}".format(l))
        util.printStatus("      MGR: {:.3f}".format(mgr))
        util.printStatus("      A: {:.3f}".format(a))
        util.printStatus("*" * 55)
        newdenom = []
        for e in exponent:
            if e > 500:
                newdenom.append(500)
            else:
                newdenom.append(e)
        denom = py.array(newdenom)
        lg = startOD + ((a - startOD) / denom)

    return lg
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re
import argparse
import binascii
import datetime
import random
import socket
import string
import sys
import uuid
import logging
import os
import errno

import filetimes, rpcBind, rpcRequest
from dcerpc import MSRPCHeader, MSRPCBindNak, MSRPCRequestHeader, MSRPCRespHeader
from kmsBase import kmsBase, UUID
from kmsRequestV4 import kmsRequestV4
from kmsRequestV5 import kmsRequestV5
from kmsRequestV6 import kmsRequestV6
from rpcBase import rpcBase
from kmsDB2Dict import kmsDB2Dict
from formatText import shell_message, justify

config = {}

def main():
        parser = argparse.ArgumentParser()
        parser.add_argument("ip", action="store", help='The IP address or hostname of the KMS server.', type=str)
        parser.add_argument("port", nargs="?", action="store", default=1688,
                            help='The port the KMS service is listening on. The default is \"1688\".', type=int)
        parser.add_argument("-m", "--mode", dest="mode",
                            choices=["WindowsVista","Windows7","Windows8","Windows8.1","Windows10",
                                     "Office2010","Office2013","Office2016","Office2019"], default="Windows8.1",
                            help='Use this flag to manually specify a Microsoft product for testing the server. The default is \"Windows81\".', type=str)
        parser.add_argument("-c", "--cmid", dest="cmid", default=None,
                            help='Use this flag to manually specify a CMID to use. If no CMID is specified, a random CMID will be generated.', type=str)
        parser.add_argument("-n", "--name", dest="machineName", default=None,
                            help='Use this flag to manually specify an ASCII machineName to use. If no machineName is specified,\
a random machineName will be generated.', type=str)
        parser.add_argument("-v", "--loglevel", dest="loglevel", action="store", default="ERROR", choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
                            help='Use this flag to set a Loglevel. The default is \"ERROR\".', type=str)
        parser.add_argument("-f", "--logfile", dest="logfile", action="store", default=os.path.dirname(os.path.abspath( __file__ )) + "/py2kms_client.log",
                            help='Use this flag to set an output Logfile. The default is \"pykms_client.log\".', type=str)

        config.update(vars(parser.parse_args()))

        logging.basicConfig(level=config['loglevel'], format='%(asctime)s %(levelname)-8s %(message)s',
                            datefmt='%a, %d %b %Y %H:%M:%S', filename=config['logfile'], filemode='w')

        checkConfig()
        config['call_id'] = 1
        updateConfig()
        s = socket.socket()
        logging.info("Connecting to %s on port %d..." % (config['ip'], config['port']))
        s.connect((config['ip'], config['port']))
        logging.info("Connection successful !")
        binder = rpcBind.handler(None, config)
        RPC_Bind = str(binder.generateRequest())
        logging.info("Sending RPC bind request...")
        shell_message(nshell = [-1, 1])
        s.send(RPC_Bind)
        try:
                shell_message(nshell = [-4, 7])
                bindResponse = s.recv(1024)
        except socket.error, e:
                if e.errno == errno.ECONNRESET:
                        logging.error("Connection reset by peer. Exiting...")
                        sys.exit()
                else:
                        raise
        if bindResponse == '' or not bindResponse:
                logging.error("No data received ! Exiting...")
                sys.exit()
        packetType = MSRPCHeader(bindResponse)['type']
        if packetType == rpcBase.packetType['bindAck']:
                logging.info("RPC bind acknowledged.")
                shell_message(nshell = 8)
                kmsRequest = createKmsRequest()
                requester = rpcRequest.handler(kmsRequest, config)
                s.send(str(requester.generateRequest()))
                shell_message(nshell = [-1, 12])
                response = s.recv(1024)
                logging.debug("Response: \n%s\n" % justify(binascii.b2a_hex(response)))
                shell_message(nshell = [-4, 20])
                parsed = MSRPCRespHeader(response)
                kmsData = readKmsResponse(parsed['pduData'], kmsRequest, config)
                kmsResp = kmsData['response']

                try:
                        hwid = kmsData['hwid']
                        logging.info("KMS Host HWID: %s" % binascii.b2a_hex(hwid).upper())
                except KeyError:
                        pass
                logging.info("KMS Host ePID: %s" % kmsResp['kmsEpid'].decode('utf-16le').encode('utf-8'))
                logging.info("KMS Host Current Client Count: %s" % kmsResp['currentClientCount'])
                logging.info("KMS VL Activation Interval: %s" % kmsResp['vLActivationInterval'])
                logging.info("KMS VL Renewal Interval: %s" % kmsResp['vLRenewalInterval'])
                shell_message(nshell = 21)

        elif packetType == rpcBase.packetType['bindNak']:
                logging.info(justify(MSRPCBindNak(bindResponse).dump(print_to_stdout = False)))
                sys.exit()
        else:
                logging.critical("Something went wrong.")
                sys.exit()

def checkConfig():
        if config['cmid'] is not None:
                try:
                        uuid.UUID(config['cmid'])
                except ValueError:
                        logging.error("Bad CMID. Exiting...")
                        sys.exit()
        if config['machineName'] is not None:
                if len(config['machineName']) < 2 or len(config['machineName']) > 63:
                        logging.error("Error: machineName must be between 2 and 63 characters in length.")
                        sys.exit()

def updateConfig():
        kmsdb = kmsDB2Dict()

        appitems = kmsdb[2]
        for appitem in appitems:
                kmsitems = appitem['KmsItems']
                for kmsitem in kmsitems:
                        # Threshold.
                        try:
                                count = int(kmsitem['NCountPolicy'])
                        except KeyError:
                                count = 25

                        name = re.sub('\(.*\)', '', kmsitem['DisplayName']).replace('2015', '').replace(' ', '')
                        if name == config['mode']:
                                skuitems = kmsitem['SkuItems']
                                # Select 'Enterprise' for Windows or 'Professional Plus' for Office.
                                # (improvement: choice could be also random: skuitem = random.choice(skuitems))
                                for skuitem in skuitems:
                                        if skuitem['DisplayName'].replace(' ','') == name + 'Enterprise' or \
                                           skuitem['DisplayName'].replace(' ','') == name[:6] + 'ProfessionalPlus' + name[6:]:

                                                config['KMSClientSkuID'] = skuitem['Id']
                                                config['RequiredClientCount'] = count
                                                config['KMSProtocolMajorVersion'] = int(float(kmsitem['DefaultKmsProtocol']))
                                                config['KMSProtocolMinorVersion'] = 0
                                                config['KMSClientLicenseStatus'] = 2
                                                config['KMSClientAppID'] = appitem['Id']
                                                config['KMSClientKMSCountedID'] = kmsitem['Id']
                                                break

def createKmsRequestBase():
        requestDict = kmsBase.kmsRequestStruct()
        requestDict['versionMinor'] = config['KMSProtocolMinorVersion']
        requestDict['versionMajor'] = config['KMSProtocolMajorVersion']
        requestDict['isClientVm'] = 0
        requestDict['licenseStatus'] = config['KMSClientLicenseStatus']
        requestDict['graceTime'] = 43200
        requestDict['applicationId'] = UUID(uuid.UUID(config['KMSClientAppID']).bytes_le)
        requestDict['skuId'] = UUID(uuid.UUID(config['KMSClientSkuID']).bytes_le)
        requestDict['kmsCountedId'] = UUID(uuid.UUID(config['KMSClientKMSCountedID']).bytes_le)
        requestDict['clientMachineId'] = UUID(uuid.UUID(config['cmid']).bytes_le if (config['cmid'] is not None) else uuid.uuid4().bytes_le)
        requestDict['previousClientMachineId'] = '\0' * 16 #requestDict['clientMachineId'] # I'm pretty sure this is supposed to be a null UUID.
        requestDict['requiredClientCount'] = config['RequiredClientCount']
        requestDict['requestTime'] = filetimes.dt_to_filetime(datetime.datetime.utcnow())
        requestDict['machineName'] = (config['machineName'] if (config['machineName'] is not None) else
                                      ''.join(random.choice(string.letters + string.digits) for i in range(random.randint(2,63)))).encode('utf-16le')
        requestDict['mnPad'] = '\0'.encode('utf-16le') * (63 - len(requestDict['machineName'].decode('utf-16le')))

        # Debug Stuff
        shell_message(nshell = 9)
        logging.debug("Request Base Dictionary: \n%s\n" % justify(requestDict.dump(print_to_stdout = False)))

        return requestDict

def createKmsRequest():
        # Update the call ID
        config['call_id'] += 1

        # KMS Protocol Major Version
        if config['KMSProtocolMajorVersion'] == 4:
                handler = kmsRequestV4(None, config)
        elif config['KMSProtocolMajorVersion'] == 5:
                handler = kmsRequestV5(None, config)
        elif config['KMSProtocolMajorVersion'] == 6:
                handler = kmsRequestV6(None, config)
        else:
                return None

        requestBase = createKmsRequestBase()
        return handler.generateRequest(requestBase)

def readKmsResponse(data, request, config):
        if config['KMSProtocolMajorVersion'] == 4:
                logging.info("Received V4 response")
                response = readKmsResponseV4(data, request)
        elif config['KMSProtocolMajorVersion'] == 5:
                logging.info("Received V5 response")
                response = readKmsResponseV5(data)
        elif config['KMSProtocolMajorVersion'] == 6:
                logging.info("Received V6 response")
                response = readKmsResponseV6(data)
        else:
                logging.info("Unhandled response version: %d.%d" % (config['KMSProtocolMajorVersion'], config['KMSProtocolMinorVersion']))
                logging.info("I'm not even sure how this happened...")
        return response

def readKmsResponseV4(data, request):
        response = kmsRequestV4.ResponseV4(data)
        hashed = kmsRequestV4(data, config).generateHash(bytearray(str(response['response'])))
        if hashed == response['hash']:
                logging.info("Response Hash has expected value !")
        return response

def readKmsResponseV5(data):
        response = kmsRequestV5.ResponseV5(data)
        decrypted = kmsRequestV5(data, config).decryptResponse(response)
        return decrypted

def readKmsResponseV6(data):
        response = kmsRequestV6.ResponseV5(data)
        decrypted = kmsRequestV6(data, config).decryptResponse(response)
        message = decrypted['message']
        return message

if __name__ == "__main__":
        main()
<commit_msg>Delete client.py<commit_after><|endoftext|>


<commit_before>from distutils.core import setup
from distutils.extension import Extension
from Cython.Distutils import build_ext
import numpy

setup(
    cmdclass = {'build_ext': build_ext},
    ext_modules = [Extension("Z_shooting", ["Z_shooting.pyx"], include_dirs=[numpy.get_include()])]
)<commit_msg>fixed cython setup<commit_after>from distutils.core import setup
from distutils.extension import Extension
from Cython.Distutils import build_ext
import numpy

setup(
    cmdclass = {'build_ext': build_ext},
    ext_modules = [Extension("Z_shooting", ["Z_shooting.pyx"],)],
    include_dirs=[numpy.get_include(),],
)<|endoftext|>