<commit_before>import logging
import re
import socket
import threading

from channel import Channel, ChannelList
from message import Message
from server  import Server

class Client:
	def __init__(self, server, nick, username, realname, **conf):
		self.sock = None
		self.nick = None
		self.server = None

		self.conf = {
			"port": 6667,
			"autorejoin": True,
			"autoconn": True,
			"channels": [],
			"prefixes": "&#"
		}

		self.conf.update(conf)
		self.conf.update({
			"server": server,
			"nick": nick,
			"username": username,
			"realname": realname,
		})

		self.channels = ChannelList()
		self.listeners = {}

		self.nickmod = 0

		if self.conf["autoconn"]:
			threading.Thread(name="main", target=self.connect).start()

	def connect(self):
		"""Attempt to connect to the server. Log in if successful."""

		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

		try:
			self.sock.connect((self.conf["server"], self.conf["port"]))
			self.server = Server(self.conf["server"], self.conf["port"])
			logging.info("Connected to %s", self.server.host)

			self.send("NICK", self.conf["nick"])
			self.send("USER", self.conf["username"], 8, "*", \
				self.conf["realname"])

			threading.Thread(name="listen", target=self.listen).start()
		except socket.error as e:
			logging.error("Cannot connect to %s: %s", self.conf["server"], e)
			self.sock.close()
			self.sock = None

	def send(self, *msg):
		"""Send a raw message to the server."""

		if not self.sock:
			logging.error("Not connected to server")
			return

		message = " ".join(map(str, msg[:-1])) + " :" + str(msg[-1])
		self.sock.sendall(bytes(message + "\r\n", "utf-8"))
		logging.debug("(%s) %s", threading.current_thread().name, message.rstrip())

	def say(self, to, msg):
		"""Send a message to a user/channel."""
		self.send("PRIVMSG", to, msg)

	def notice(self, to, msg):
		"""Send a notice to a user/channel."""
		self.send("NOTICE", to, msg)

	def ctcp_say(self, to, text):
		"""Send a CTCP PRIVMSG message."""
		self.say(to, "\x01{0}\x01".format(text))

	def ctcp_notice(self, to, text):
		"""Send a CTCP NOTICE message."""
		self.notice(to, "\x01{0}\x01".format(text))

	def action(self, to, msg):
		self.ctcp_say(to, "ACTION {0}".format(msg))

	def join(self, chan):
		"""Join a channel."""
		self.send("JOIN", chan)

	def part(self, chan, reason=None):
		self.send("PART", chan, reason or "")

	def listen(self):
		"""Listen for incoming messages from the IRC server."""
		if not self.sock:
			logging.error("Not connected to server")
			return

		buf = ""
		while True:
			try:
				buf += str(self.sock.recv(4096), "utf-8")
				*msgs, buf = buf.split("\r\n")
				for msg in msgs:
					m = Message.parse(msg)
					thread = threading.Thread(target=lambda: self.handle(m))
					logging.debug("(%s) handler thread %s [%s]",
						threading.current_thread().name, thread.name, m.raw)
					thread.start()
			except socket.error:
				logging.info("Disconnected from server")
				self.sock.close()
				self.sock = None
				break

	def add_listener(self, event, fn):
		"""Add a function to listen for the specified event."""
		if event not in self.listeners:
			self.listeners[event] = []
		self.listeners[event].append(fn)

	def remove_listener(self, event, fn):
		"""Remove a function as a listener from the specified event."""
		if event not in self.listeners:
			return
		self.listeners[event] = [l for l in self.listeners[event] if l != fn]

	def remove_listeners(self, event):
		"""Remove all functions listening for the specified event."""
		if event not in self.listeners:
			return
		self.listeners.pop(event)

	def emit(self, event, *params):
		"""Emit an event, and call all functions listening for it."""
		if event in self.listeners:
			for listener in self.listeners[event]:
				try:
					thread = threading.Thread(target=lambda: listener(*params))
					logging.debug("(%s) worker thread %s [%s, %s]",
						threading.current_thread().name, thread.name, event,
						listener.__name__)
					thread.start()
				except TypeError as e:
					logging.error("(%s) invalid number of parameters [%s]",
						threading.current_thread().name, listener.__name__)

	def _ctcp(self, fr, to, text, type):
		text = text[1:text.index("\x01")]
		parts = text.split()
		self.emit("ctcp", fr, to, text, type)
		self.emit("ctcp." + type, fr, to, text)

		if type == "privmsg":
			if text == "VERSION":
				self.emit("ctcp.version", fr, to)
			elif len(parts) > 1 and parts[0] == "ACTION":
				self.emit("action", fr, to, " ".join(parts[1:]))
			elif len(parts) > 1 and parts[0] == "PING":
				self.ctcp_notice(fr, "PONG " + " ".join(parts[1:]))

	def handle(self, msg):
		c = msg.command
		if c == "001":
			self.nick = self.conf["nick"] + self.nickmod * "_"
			self.emit("registered", msg.params[0])
		elif c == "004":
			self.server.usermodes = set(msg.params[3])
		elif c == "005":
			for p in msg.params:
				if "=" in p:
					param, value = p.split("=")
					if param == "CHANLIMIT":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.chlimit[typ] = int(num)
					elif param == "CHANMODES":
						modes = value.split(",")
						self.server.chmodes = dict(zip("abcd", map(set, modes)))
					elif param == "CHANTYPES":
						self.server.types = set(value)
					elif param == "CHANNELLEN":
						self.server.chlength = int(value)
					elif param == "IDCHAN":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.idlength[typ] = int(num)
					elif param == "KICKLEN":
						self.server.kicklength = int(value)
					elif param == "NICKLEN":
						self.server.nicklength = int(value)
					elif param == "PREFIX":
						if re.match(r"^\((.*)\)(.*)$", value):
							modes = value.split(")")[0][1:]
							prefixes = value.split(")")[1]

							self.server.prefix_mode = dict(zip(prefixes, modes))
							self.server.mode_prefix = dict(zip(modes, prefixes))

							self.server.chmodes["b"].update(modes)
					elif param == "TARGMAX":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.maxtargets[typ] = int(num) if num else 0
					elif param == "TOPICLEN":
						self.server.topiclength = int(value)
		elif c == "433":
			self.nickmod += 1
			self.send("NICK", self.conf["nick"] + self.nickmod * "_")
		elif c == "PING":
			self.send("PONG", msg.params[0])
			self.emit("ping", msg.params[0])
		elif c == "PONG":
			self.emit("pong", msg.params[0])
		elif c == "NOTICE":
			fr, to = msg.nick, msg.params[0]
			text = msg.params[1] or ""
			if text[0] == "\x01" and "\x01" in text[1:]:
				self._ctcp(fr, to, text, "notice")
			else:
				self.emit("notice", fr, to, text)
		elif c == "MODE":
			pass # TODO
		elif c == "NICK":
			nick = msg.params[0]
			if msg.nick == self.nick:
				self.nick = nick
			chans = list(filter(lambda c: msg.nick in c, self.channels.values()))
			for chan in chans:
				chan.users[nick] = chan.users[msg.nick]
				chan.users.pop(nick)
			self.emit("nick", msg.nick, nick, [c.name for c in chans])
		elif c == "375":
			self.server.motd = msg.params[1] + "\n"
		elif c == "372":
			self.server.motd += msg.params[1] + "\n"
		elif c == "376" or c == "422":
			self.server.motd += msg.params[1] + "\n"
			self.emit("motd", self.server.motd)
		elif c == "353":
			channel = self.channels[msg.params[2][1:]]
			if channel:
				users = msg.params[3].strip().split()
				for user in users:
					if user[0] in self.server.prefix_mode:
						channel.users[user[1:]] = user[0]
					else:
						channel.users[user] = ""
		elif c == "366":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				self.emit("names", msg.params[1], channel.users)
				self.send("MODE", msg.params[1])
		elif c == "332":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.topic = msg.params[2]
		# 301, 311, 312, 313, 317, 318, 319, 330, 321, 322, 323, 333
		elif c == "TOPIC":
			self.emit("topic", msg.params[0], msg.params[1], msg.nick, msg)
			channel = self.channels[msg.params[0][1:]]
			if channel:
				channel.topic = msg.params[1]
				channel.topicby = msg.nick
		elif c == "324":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.mode = msg.params[2]
		elif c == "329":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.created = int(msg.params[2])
		elif c == "JOIN":
			chan = msg.params[0]
			if self.nick == msg.nick:
				self.channels[chan[1:]] = Channel(chan)
			else:
				channel = self.channels[chan[1:]]
				if channel:
					channel.users[msg.nick] = ""
			self.emit("join", chan, msg.nick)
		elif c == "PART":
			chan = msg.params[0]
			self.emit("part", chan, msg.nick)
			if self.nick == msg.nick:
				self.channels.pop(chan)
			else:
				channel = self.channels[chan]
				if channel:
					del channel.users[msg.nick]
		elif c == "KICK":
			chan, who, reason, *rest = msg.params
			self.emit("kick", chan, who, msg.nick, reason)
			if self.nick == msg.nick:
				self.channels.pop(chan)
			else:
				channel = self.channels[chan]
				if channel:
					channel.users.pop(who)
		elif c == "KILL":
			nick = msg.params[0]
			channels = []
			for chan in self.channels:
				channels.append(chan.name)
				chan.users.pop(nick)
			self.emit("kill", nick, msg.params[1], channels)
		elif c == "PRIVMSG":
			fr, to = msg.nick, msg.params[0]
			text = " ".join(msg.params[1:])
			if text[0] == "\x01" and "\x01" in text[1:]:
				self._ctcp(fr, to, text, "privmsg")
			else:
				self.emit("message", fr, to, text)
			self.emit("pm", fr, text, msg)
		elif c == "INVITE":
			self.emit("invite", msg.params[1], msg.nick)
		elif c == "QUIT":
			if self.nick == msg.nick:
				return
			nick = msg.params[0]
			if msg.nick == self.nick:
				self.nick = nick
			chans = list(filter(lambda c: msg.nick in c, self.channels.values()))
			for chan in chans:
				chan.users.pop(nick)
			self.emit("quit", msg.nick, nick, [c.name for c in chans])
<commit_msg>forgot to change some things (aka oops)<commit_after>import logging
import re
import socket
import threading

from channel import Channel, ChannelList
from message import Message
from server  import Server

class Client:
	def __init__(self, server, nick, username, realname, **conf):
		self.sock = None
		self.nick = None
		self.server = None

		self.conf = {
			"port": 6667,
			"autorejoin": True,
			"autoconn": True,
			"channels": [],
			"prefixes": "&#"
		}

		self.conf.update(conf)
		self.conf.update({
			"server": server,
			"nick": nick,
			"username": username,
			"realname": realname,
		})

		self.channels = ChannelList()
		self.listeners = {}

		self.nickmod = 0

		if self.conf["autoconn"]:
			threading.Thread(name="main", target=self.connect).start()

	def connect(self):
		"""Attempt to connect to the server. Log in if successful."""

		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

		try:
			self.sock.connect((self.conf["server"], self.conf["port"]))
			self.server = Server(self.conf["server"], self.conf["port"])
			logging.info("Connected to %s", self.server.host)

			self.send("NICK", self.conf["nick"])
			self.send("USER", self.conf["username"], 8, "*", \
				self.conf["realname"])

			threading.Thread(name="listen", target=self.listen).start()
		except socket.error as e:
			logging.error("Cannot connect to %s: %s", self.conf["server"], e)
			self.sock.close()
			self.sock = None

	def send(self, *msg):
		"""Send a raw message to the server."""

		if not self.sock:
			logging.error("Not connected to server")
			return

		message = " ".join(map(str, msg[:-1])) + " :" + str(msg[-1])
		self.sock.sendall(bytes(message + "\r\n", "utf-8"))
		logging.debug("(%s) %s", threading.current_thread().name, message.rstrip())

	def say(self, to, msg):
		"""Send a message to a user/channel."""
		self.send("PRIVMSG", to, msg)

	def notice(self, to, msg):
		"""Send a notice to a user/channel."""
		self.send("NOTICE", to, msg)

	def ctcp_say(self, to, text):
		"""Send a CTCP PRIVMSG message."""
		self.say(to, "\x01{0}\x01".format(text))

	def ctcp_notice(self, to, text):
		"""Send a CTCP NOTICE message."""
		self.notice(to, "\x01{0}\x01".format(text))

	def action(self, to, msg):
		self.ctcp_say(to, "ACTION {0}".format(msg))

	def join(self, chan):
		"""Join a channel."""
		self.send("JOIN", chan)

	def part(self, chan, reason=None):
		self.send("PART", chan, reason or "")

	def listen(self):
		"""Listen for incoming messages from the IRC server."""
		if not self.sock:
			logging.error("Not connected to server")
			return

		buf = ""
		while True:
			try:
				buf += str(self.sock.recv(4096), "utf-8")
				*msgs, buf = buf.split("\r\n")
				for msg in msgs:
					m = Message.parse(msg)
					thread = threading.Thread(target=lambda: self.handle(m))
					logging.debug("(%s) handler thread %s [%s]",
						threading.current_thread().name, thread.name, m.raw)
					thread.start()
			except socket.error:
				logging.info("Disconnected from server")
				self.sock.close()
				self.sock = None
				break

	def add_listener(self, event, fn):
		"""Add a function to listen for the specified event."""
		if event not in self.listeners:
			self.listeners[event] = []
		self.listeners[event].append(fn)

	def remove_listener(self, event, fn):
		"""Remove a function as a listener from the specified event."""
		if event not in self.listeners:
			return
		self.listeners[event] = [l for l in self.listeners[event] if l != fn]

	def remove_listeners(self, event):
		"""Remove all functions listening for the specified event."""
		if event not in self.listeners:
			return
		self.listeners.pop(event)

	def emit(self, event, *params):
		"""Emit an event, and call all functions listening for it."""
		if event in self.listeners:
			for listener in self.listeners[event]:
				try:
					thread = threading.Thread(target=lambda: listener(*params))
					logging.debug("(%s) worker thread %s [%s, %s]",
						threading.current_thread().name, thread.name, event,
						listener.__name__)
					thread.start()
				except TypeError as e:
					logging.error("(%s) invalid number of parameters [%s]",
						threading.current_thread().name, listener.__name__)

	def _ctcp(self, fr, to, text, type):
		text = text[1:text.index("\x01")]
		parts = text.split()
		self.emit("ctcp", fr, to, text, type)
		self.emit("ctcp." + type, fr, to, text)

		if type == "privmsg":
			if text == "VERSION":
				self.emit("ctcp.version", fr, to)
			elif len(parts) > 1 and parts[0] == "ACTION":
				self.emit("action", fr, to, " ".join(parts[1:]))
			elif len(parts) > 1 and parts[0] == "PING":
				self.ctcp_notice(fr, "PONG " + " ".join(parts[1:]))

	def handle(self, msg):
		c = msg.command
		if c == "001":
			self.nick = self.conf["nick"] + self.nickmod * "_"
			self.emit("registered", msg.params[0])
		elif c == "004":
			self.server.usermodes = set(msg.params[3])
		elif c == "005":
			for p in msg.params:
				if "=" in p:
					param, value = p.split("=")
					if param == "CHANLIMIT":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.chlimit[typ] = int(num)
					elif param == "CHANMODES":
						modes = value.split(",")
						self.server.chmodes = dict(zip("abcd", map(set, modes)))
					elif param == "CHANTYPES":
						self.server.types = set(value)
					elif param == "CHANNELLEN":
						self.server.chlength = int(value)
					elif param == "IDCHAN":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.idlength[typ] = int(num)
					elif param == "KICKLEN":
						self.server.kicklength = int(value)
					elif param == "NICKLEN":
						self.server.nicklength = int(value)
					elif param == "PREFIX":
						if re.match(r"^\((.*)\)(.*)$", value):
							modes = value.split(")")[0][1:]
							prefixes = value.split(")")[1]

							self.server.prefix_mode = dict(zip(prefixes, modes))
							self.server.mode_prefix = dict(zip(modes, prefixes))

							self.server.chmodes["b"].update(modes)
					elif param == "TARGMAX":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.maxtargets[typ] = int(num) if num else 0
					elif param == "TOPICLEN":
						self.server.topiclength = int(value)
		elif c == "433":
			self.nickmod += 1
			self.send("NICK", self.conf["nick"] + self.nickmod * "_")
		elif c == "PING":
			self.send("PONG", msg.params[0])
			self.emit("ping", msg.params[0])
		elif c == "PONG":
			self.emit("pong", msg.params[0])
		elif c == "NOTICE":
			fr, to = msg.nick, msg.params[0]
			text = msg.params[1] or ""
			if text[0] == "\x01" and "\x01" in text[1:]:
				self._ctcp(fr, to, text, "notice")
			else:
				self.emit("notice", fr, to, text)
		elif c == "MODE":
			pass # TODO
		elif c == "NICK":
			nick = msg.params[0]
			if msg.nick == self.nick:
				self.nick = nick
			chans = list(filter(lambda c: msg.nick in c, self.channels.values()))
			for chan in chans:
				chan.users[nick] = chan.users[msg.nick]
				chan.users.pop(nick)
			self.emit("nick", msg.nick, nick, [c.name for c in chans])
		elif c == "375":
			self.server.motd = msg.params[1] + "\n"
		elif c == "372":
			self.server.motd += msg.params[1] + "\n"
		elif c == "376" or c == "422":
			self.server.motd += msg.params[1] + "\n"
			self.emit("motd", self.server.motd)
		elif c == "353":
			channel = self.channels[msg.params[2][1:]]
			if channel:
				users = msg.params[3].strip().split()
				for user in users:
					if user[0] in self.server.prefix_mode:
						channel.users[user[1:]] = user[0]
					else:
						channel.users[user] = ""
		elif c == "366":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				self.emit("names", msg.params[1], channel.users)
				self.send("MODE", msg.params[1])
		elif c == "332":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.topic = msg.params[2]
		# 301, 311, 312, 313, 317, 318, 319, 330, 321, 322, 323, 333
		elif c == "TOPIC":
			self.emit("topic", msg.params[0], msg.params[1], msg.nick, msg)
			channel = self.channels[msg.params[0][1:]]
			if channel:
				channel.topic = msg.params[1]
				channel.topicby = msg.nick
		elif c == "324":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.mode = msg.params[2]
		elif c == "329":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.created = int(msg.params[2])
		elif c == "JOIN":
			chan = msg.params[0]
			if self.nick == msg.nick:
				self.channels[chan[1:]] = Channel(chan)
			else:
				channel = self.channels[chan[1:]]
				if channel:
					channel.users[msg.nick] = ""
			self.emit("join", chan, msg.nick)
		elif c == "PART":
			chan = msg.params[0]
			self.emit("part", chan, msg.nick)
			if self.nick == msg.nick:
				self.channels.pop(chan)
			else:
				channel = self.channels[chan]
				if channel:
					del channel.users[msg.nick]
		elif c == "KICK":
			chan, who, reason, *rest = msg.params
			self.emit("kick", chan, who, msg.nick, reason)
			if self.nick == msg.nick:
				self.channels.pop(chan)
			else:
				channel = self.channels[chan]
				if channel:
					channel.users.pop(who)
		elif c == "KILL":
			nick = msg.params[0]
			channels = []
			for chan in self.channels:
				channels.append(chan.name)
				chan.users.pop(nick)
			self.emit("kill", nick, msg.params[1], channels)
		elif c == "PRIVMSG":
			fr, to = msg.nick, msg.params[0]
			text = " ".join(msg.params[1:])
			if text[0] == "\x01" and "\x01" in text[1:]:
				self._ctcp(fr, to, text, "privmsg")
			else:
				self.emit("message", fr, to, text)
			self.emit("pm", fr, text, msg)
		elif c == "INVITE":
			self.emit("invite", msg.params[1], msg.nick)
		elif c == "QUIT":
			if self.nick == msg.nick:
				return
			if msg.nick == self.nick:
				self.nick = nick
			chans = list(filter(lambda c: msg.nick in c, self.channels.values()))
			for chan in chans:
				chan.users.pop(msg.nick)
			self.emit("quit", msg.nick, [c.name for c in chans])
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, os

import mmap  # Thanks Steven @ http://stackoverflow.com/questions/4940032/search-for-string-in-txt-file-python

import subprocess

import readline

readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.parse_and_bind("set match-hidden-files off")

import argparse

def query(question, default, skipQuery=False):
    if skipQuery:
        return default
    sys.stdout.write(question + " [" + default + "] ? ")
    choice = input()
    if choice == '':
        return default
    return choice

class MyArgumentDefaultsHelpFormatter(argparse.HelpFormatter):

    def _split_lines(self, text, width):
        return text.splitlines()

    def _get_help_string(self, action):
        help = action.help
        if '%(default)' not in action.help:
            if action.default is not argparse.SUPPRESS:
                defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE]
                if action.option_strings or action.nargs in defaulting_nargs:
                    help += ' (default: "%(default)s")'
        return help

if __name__ == '__main__':

    print("-------------------------------------")
    print("Welcome to song-directory-to-songbook")
    print("-------------------------------------")

    parser = argparse.ArgumentParser(formatter_class = MyArgumentDefaultsHelpFormatter)

    parser.add_argument('--input',
                        help='path of the default song input directory',
                        default='examples/')
    parser.add_argument('--output',
                        help='name of the output pdf file',
                        default='Songbook.pdf')
    parser.add_argument('--template',
                        help='name of the LaTeX template file [specifies language, etc]',
                        default='template/english.tex')
    parser.add_argument('--manifest',
                        help='name of a file-avoiding manifest file [if desired]',
                        default='')
    parser.add_argument('--yes',
                        help='accept all, skip all queries',
                        nargs='?',
                        default='NULL')  # required, see below
    args = parser.parse_args()

    skipQueries = False
    if args.yes is not 'NULL':  # if exists and no contents, replaces 'NULL' by None
        print("Detected --yes parameter: will skip queries")
        skipQueries = True

    # Query the path of the song input directory
    inputDirectory = query("Please specify the path of the song input directory", args.input, skipQueries)
    print("Will use song input directory: " + inputDirectory)

    # Query the path of the song input directory
    outputFile = query("Please specify the name of the output pdf file", args.output, skipQueries)
    print("Will use the output pdf file: " + outputFile)
    outputFileName, outputFileExtension = os.path.splitext(outputFile)

    # Query the path of the template file
    templateFile = query("Please specify the path of the LaTeX template file [specifies language, format]", args.template, skipQueries)
    print("Will use template file: " + templateFile)

    # Query (optional) the path of a file-avoiding manifest file
    manifestFile = query("Please specify the name of a file-avoiding manifest file [if desired]", args.manifest, skipQueries)
    if manifestFile == "":
        print("Not using file-avoiding manifest file.")
    else:
        print("Will use file-avoiding manifest file: " + manifestFile)
        manifestFileFd = open(manifestFile, 'r', encoding="utf8")
        manifestMmap = mmap.mmap(manifestFileFd.fileno(), 0, access=mmap.ACCESS_READ)
        manifestFileFd.close()

    print("----------------------")

    templateFileFd = open(templateFile, 'r', encoding="utf8")
    s = templateFileFd.read()
    #sys.stdout.write(s)  #-- Screen output for debugging.

    rep = ""
    for dirname, dirnames, filenames in os.walk(inputDirectory):
        for filename in sorted(filenames):
            name, extension = os.path.splitext(filename)
            if manifestFile != "":
                if manifestMmap.find(name) != -1:
                    print("Skipping:", name)
                    continue
            rep += "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n"
            rep += "\\chapter{" + name + "}\n"  #-- Note that we use \\ instead of \.
            songName = name.split(" - ")[-1]
            #-- We cannot use [] yet (they will be replaced because choir), so use {{}}.
            rep += "\\index{{aux-song-index-file}}{" + songName + "}\n"
            rep += "\\begin{alltt}\n"
            print("os.path.join(dirname, filename)",os.path.join(dirname, filename))
            song = open(os.path.join(dirname, filename), encoding="utf8")
            rep += song.read()
            song.close()
            rep += "\\end{alltt}\n"
            rep += "\n"
    #sys.stdout.write(rep)  #-- Screen output for debugging.

    #-- replace chords delimiter ()
    rep = rep.replace("(","\\textbf{(")
    rep = rep.replace(")",")}")

    #-- replace choir delimiter []
    rep = rep.replace("[","\\textit{[")
    rep = rep.replace("]","]}")

    #-- now we can do this
    rep = rep.replace("{{aux-song-index-file}}","[aux-song-index-file]")

    #-- replace template contents
    s = s.replace("TITLE", outputFileName)
    s = s.replace("genSongbook",rep)

    outputFileTex = outputFileName + ".tex"
    outFd = open(outputFileTex, 'w', encoding="utf8")
    outFd.write(s)
    outFd.close()

    #http://stackoverflow.com/questions/6818102/detect-and-handle-a-latex-warning-error-generated-via-an-os-system-call-in-pytho
    #pdftex_process = subprocess.Popen(['pdflatex', '-interaction=nonstopmode', '%s'%topic], shell=False, stdout=subprocess.PIPE)
    pdftex_process = subprocess.call(['pdflatex', outputFileTex])
    pdftex_process = subprocess.call(['pdflatex', outputFileTex])
    os.remove("aux-song-index-file.idx")
    os.remove("aux-song-index-file.ilg")
    os.remove("aux-song-index-file.ind")
    os.remove(outputFileName + ".aux")
    os.remove(outputFileName + ".log")
    os.remove(outputFileName + ".out")
    os.remove(outputFileName + ".toc")

    os.remove(outputFileTex)  # may be interested in keeping
<commit_msg>adapt to paths set in --output<commit_after>#!/usr/bin/env python

import sys, os

import mmap  # Thanks Steven @ http://stackoverflow.com/questions/4940032/search-for-string-in-txt-file-python

import subprocess

import readline

readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.parse_and_bind("set match-hidden-files off")

import argparse

def query(question, default, skipQuery=False):
    if skipQuery:
        return default
    sys.stdout.write(question + " [" + default + "] ? ")
    choice = input()
    if choice == '':
        return default
    return choice

class MyArgumentDefaultsHelpFormatter(argparse.HelpFormatter):

    def _split_lines(self, text, width):
        return text.splitlines()

    def _get_help_string(self, action):
        help = action.help
        if '%(default)' not in action.help:
            if action.default is not argparse.SUPPRESS:
                defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE]
                if action.option_strings or action.nargs in defaulting_nargs:
                    help += ' (default: "%(default)s")'
        return help

if __name__ == '__main__':

    print("-------------------------------------")
    print("Welcome to song-directory-to-songbook")
    print("-------------------------------------")

    parser = argparse.ArgumentParser(formatter_class = MyArgumentDefaultsHelpFormatter)

    parser.add_argument('--input',
                        help='path of the default song input directory',
                        default='examples/')
    parser.add_argument('--output',
                        help='name of the output pdf file',
                        default='Songbook.pdf')
    parser.add_argument('--template',
                        help='name of the LaTeX template file [specifies language, etc]',
                        default='template/english.tex')
    parser.add_argument('--manifest',
                        help='name of a file-avoiding manifest file [if desired]',
                        default='')
    parser.add_argument('--yes',
                        help='accept all, skip all queries',
                        nargs='?',
                        default='NULL')  # required, see below
    args = parser.parse_args()

    skipQueries = False
    if args.yes is not 'NULL':  # if exists and no contents, replaces 'NULL' by None
        print("Detected --yes parameter: will skip queries")
        skipQueries = True

    # Query the path of the song input directory
    inputDirectory = query("Please specify the path of the song input directory", args.input, skipQueries)
    print("Will use song input directory: " + inputDirectory)

    # Query the path of the song output file
    outputFile = query("Please specify the name of the output pdf file", args.output, skipQueries)
    print("Will use the output pdf file: " + outputFile)
    outputFileDirAndName, outputFileExtension = os.path.splitext(outputFile)
    outputFileDir, outputFileName = os.path.split(outputFileDirAndName)

    # Query the path of the template file
    templateFile = query("Please specify the path of the LaTeX template file [specifies language, format]", args.template, skipQueries)
    print("Will use template file: " + templateFile)

    # Query (optional) the path of a file-avoiding manifest file
    manifestFile = query("Please specify the name of a file-avoiding manifest file [if desired]", args.manifest, skipQueries)
    if manifestFile == "":
        print("Not using file-avoiding manifest file.")
    else:
        print("Will use file-avoiding manifest file: " + manifestFile)
        manifestFileFd = open(manifestFile, 'r', encoding="utf8")
        manifestMmap = mmap.mmap(manifestFileFd.fileno(), 0, access=mmap.ACCESS_READ)
        manifestFileFd.close()

    print("----------------------")

    templateFileFd = open(templateFile, 'r', encoding="utf8")
    s = templateFileFd.read()
    #sys.stdout.write(s)  #-- Screen output for debugging.

    rep = ""
    for dirname, dirnames, filenames in os.walk(inputDirectory):
        for filename in sorted(filenames):
            name, extension = os.path.splitext(filename)
            if manifestFile != "":
                if manifestMmap.find(name) != -1:
                    print("Skipping:", name)
                    continue
            rep += "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n"
            rep += "\\chapter{" + name + "}\n"  #-- Note that we use \\ instead of \.
            songName = name.split(" - ")[-1]
            #-- We cannot use [] yet (they will be replaced because choir), so use {{}}.
            rep += "\\index{{aux-song-index-file}}{" + songName + "}\n"
            rep += "\\begin{alltt}\n"
            print("os.path.join(dirname, filename)",os.path.join(dirname, filename))
            song = open(os.path.join(dirname, filename), encoding="utf8")
            rep += song.read()
            song.close()
            rep += "\\end{alltt}\n"
            rep += "\n"
    #sys.stdout.write(rep)  #-- Screen output for debugging.

    #-- replace chords delimiter ()
    rep = rep.replace("(","\\textbf{(")
    rep = rep.replace(")",")}")

    #-- replace choir delimiter []
    rep = rep.replace("[","\\textit{[")
    rep = rep.replace("]","]}")

    #-- now we can do this
    rep = rep.replace("{{aux-song-index-file}}","[aux-song-index-file]")

    #-- replace template contents
    s = s.replace("TITLE", outputFileName)
    s = s.replace("genSongbook",rep)

    outputFileTex = outputFileDirAndName + ".tex"
    outFd = open(outputFileTex, 'w', encoding="utf8")
    outFd.write(s)
    outFd.close()

    #http://stackoverflow.com/questions/6818102/detect-and-handle-a-latex-warning-error-generated-via-an-os-system-call-in-pytho
    #pdftex_process = subprocess.Popen(['pdflatex', '-interaction=nonstopmode', '%s'%topic], shell=False, stdout=subprocess.PIPE)
    pdftex_process = subprocess.call(['pdflatex', '-output-directory='+outputFileDir, outputFileTex]) # '-aux-directory='+outputFileDir
    pdftex_process = subprocess.call(['pdflatex', '-output-directory='+outputFileDir, outputFileTex]) # '-aux-directory='+outputFileDir

    os.remove(os.path.join(outputFileDir, "aux-song-index-file.idx"))

    if os.path.exists(os.path.join(outputFileDir, "aux-song-index-file.ilg")):
        os.remove(os.path.join(outputFileDir, "aux-song-index-file.ilg"))
    else:
        print("No .ilg file, which is frequent on Windows.")

    if os.path.exists(os.path.join(outputFileDir, "aux-song-index-file.ind")):
        os.remove(os.path.join(outputFileDir, "aux-song-index-file.ind"))
    else:
        print("No .ind file, which is frequent on Windows.")

    os.remove(outputFileDirAndName + ".aux")
    os.remove(outputFileDirAndName + ".log")
    os.remove(outputFileDirAndName + ".out")
    os.remove(outputFileDirAndName + ".toc")

    os.remove(outputFileTex)  # may be interested in keeping

    print('DONE! Generated', outputFileDirAndName+'.pdf')<|endoftext|>


<commit_before># PDU v1/v2c two-way proxy
from pysnmp.proto import rfc3411, error
from pysnmp.proto.api import v1, v2c
from pysnmp.smi import exval

# 2.1.1

__v1ToV2ValueMap = {
    v1.Integer.tagSet: v2c.Integer32(),
    v1.OctetString.tagSet: v2c.OctetString(),
    v1.Null.tagSet: v2c.Null(),
    v1.ObjectIdentifier.tagSet: v2c.ObjectIdentifier(),
    v1.IpAddress.tagSet: v2c.IpAddress(),
    v1.Counter.tagSet: v2c.Counter32(),
    v1.Gauge.tagSet: v2c.Gauge32(),
    v1.TimeTicks.tagSet: v2c.TimeTicks(),
    v1.Opaque.tagSet: v2c.Opaque()
    }

__v2ToV1ValueMap = { # XXX do not re-create same-type items?
    v2c.Integer32.tagSet: v1.Integer(),
    v2c.OctetString.tagSet: v1.OctetString(),
    v2c.Null.tagSet: v1.Null(),
    v2c.ObjectIdentifier.tagSet: v1.ObjectIdentifier(),
    v2c.IpAddress.tagSet: v1.IpAddress(),
    v2c.Counter32.tagSet: v1.Counter(),
    v2c.Gauge32.tagSet: v1.Gauge(),
    v2c.TimeTicks.tagSet: v1.TimeTicks(),
    v2c.Opaque.tagSet: v1.Opaque()
    }

# PDU map

__v1ToV2PduMap = {
    v1.GetRequestPDU.tagSet: v2c.GetRequestPDU(),
    v1.GetNextRequestPDU.tagSet: v2c.GetNextRequestPDU(),
    v1.SetRequestPDU.tagSet: v2c.SetRequestPDU(),
    v1.GetResponsePDU.tagSet: v2c.ResponsePDU(),
    v1.TrapPDU.tagSet: v2c.SNMPv2TrapPDU()
    }

__v2ToV1PduMap = {
    v2c.GetRequestPDU.tagSet: v1.GetRequestPDU(),
    v2c.GetNextRequestPDU.tagSet: v1.GetNextRequestPDU(),
    v2c.SetRequestPDU.tagSet: v1.SetRequestPDU(),
    v2c.ResponsePDU.tagSet: v1.GetResponsePDU(),
    v2c.SNMPv2TrapPDU.tagSet: v1.TrapPDU(),
    v2c.GetBulkRequestPDU.tagSet: v1.GetNextRequestPDU() # 4.1.1
    }

# Trap map

__v1ToV2TrapMap = {
    0: (1,3,6,1,6,3,1,1,5,1),
    1: (1,3,6,1,6,3,1,1,5,2),
    2: (1,3,6,1,6,3,1,1,5,3),
    3: (1,3,6,1,6,3,1,1,5,4),
    4: (1,3,6,1,6,3,1,1,5,5),
    5: (1,3,6,1,6,3,1,1,5,6)
    }

__v2ToV1TrapMap = {
    (1,3,6,1,6,3,1,1,5,1): 0,
    (1,3,6,1,6,3,1,1,5,2): 1,
    (1,3,6,1,6,3,1,1,5,3): 2,
    (1,3,6,1,6,3,1,1,5,4): 3,
    (1,3,6,1,6,3,1,1,5,5): 4,
    (1,3,6,1,6,3,1,1,5,6): 5
    }

# 4.3

__v2ToV1ErrorMap = {
    0:  0,
    1:  1,
    5:  5,
    10: 3,
    9:  3,
    7:  3,
    8:  3,
    12: 3,
    6:  2,
    17: 2,
    11: 2,
    18: 2,
    13: 5,
    14: 5,
    15: 5,
    16: 2
    }

def v1ToV2(v1Pdu, origV2Pdu=None):
    pduType = v1Pdu.tagSet
    v2Pdu = __v1ToV2PduMap[pduType].clone()
    v2c.apiPDU.setDefaults(v2Pdu)

    if not rfc3411.notificationClassPDUs.has_key(pduType):
        requestId = v1.apiPDU.getRequestID(v1Pdu)
        v2c.apiPDU.setRequestID(v2Pdu, long(v1.apiPDU.getRequestID(v1Pdu)))

    v2VarBinds = []

    # 3.1
    if rfc3411.notificationClassPDUs.has_key(pduType):
        # 3.1.1
        sysUpTime = v1.apiTrapPDU.getTimeStamp(v1Pdu)

        # 3.1.2
        genericTrap = v1.apiTrapPDU.getGenericTrap(v1Pdu)
        if genericTrap == 6:
            snmpTrapOID = v1.apiTrapPDU.getEnterprise(v1Pdu) + (0,) + \
                          v1.apiTrapPDU.getSpecificTrap(v1Pdu)

        # 3.1.3
        else:
            snmpTrapOID = __v1ToV2TrapMap[genericTrap]

        v2VarBinds.append((sysUpTime, None))
        v2VarBinds.append((snmpTrapOID, None))

        # 3.1.4 --> done below

    # Translate Var-Binds
    for oid, v1Val in v1.apiPDU.getVarBinds(v1Pdu):
        v2VarBinds.append(
            (oid, __v1ToV2ValueMap[v1Val.tagSet].clone(v1Val))
            )

    if rfc3411.responseClassPDUs.has_key(pduType):
        # 4.1.2.2 --> one-to-one mapping
        v2c.apiPDU.setErrorStatus(
            v2Pdu, int(v1.apiPDU.getErrorStatus(v1Pdu))
            )
        v2c.apiPDU.setErrorIndex(
            v2Pdu, int(v1.apiPDU.getErrorIndex(v1Pdu))
            )

        # 4.1.2.1 --> no-op

    v2c.apiPDU.setVarBinds(v2Pdu, v2VarBinds)

    return v2Pdu

def v2ToV1(v2Pdu, origV1Pdu=None):
    pduType = v2Pdu.tagSet

    v1Pdu = __v2ToV1PduMap[pduType].clone()
    v1.apiPDU.setDefaults(v1Pdu)

    v2VarBinds = v2c.apiPDU.getVarBinds(v2Pdu)
    v1VarBinds = []

    # 3.2
    if rfc3411.notificationClassPDUs.has_key(pduType):
        # 3.2.1
        snmpTrapOID = v2VarBinds[0][0]
        if __v2ToV1TrapMap.has_key(snmpTrapOID):
            for oid, val in v2VarBinds:
                # snmpTrapEnterprise
                if oid == (1, 3, 6, 1, 6, 3, 1, 1, 4, 3, 0):
                    v1.apiTrapPDU.setEnterprise(v1Pdu, val)
                    break
            else:
                # snmpTraps
                v1.apiTrapPDU.setEnterprise(v1Pdu, (1, 3, 6, 1, 6, 3, 1, 1, 5))
        else:
            if snmpTrapOID[-2] == 0:
                v1.apiTrapPDU.setEnterprise(v1Pdu, snmpTrapOID[:-2])
            else:
                v1.apiTrapPDU.setEnterprise(v1Pdu, snmpTrapOID[:-1])

        # 3.2.2
        for oid, val in v2VarBinds:
            # snmpTrapAddress
            if oid == (1, 3, 6, 1, 6, 3, 18, 1, 3, 0):
                v1.apiTrapPDU.setAgentAddress(v1Pdu, val)
                break
        else:
            v1.apiTrapPDU.setAgentAddress(v1Pdu, '0.0.0.0')

        # 3.2.3
        if __v2ToV1TrapMap.has_key(snmpTrapOID):
            v1.apiTrapPDU.setGenericTrap(v1Pdu, __v2ToV1TrapMap[snmpTrapOID])
        else:
            v1.apiTrapPDU.setGenericTrap(v1Pdu, 0)

        # 3.2.4
        if __v2ToV1TrapMap.has_key(snmpTrapOID):
            v1.apiTrapPDU.setSpecificTrap(v1Pdu, 0)
        else:
            v1.apiTrapPDU.setSpecificTrap(v1Pdu, snmpTrapOID[-1])

        # 3.2.5
        v1.apiTrapPDU.setTimeStamp(v1Pdu, v2c.apiTrapPDU.getTimeStamp(v2Pdu))

        # 3.2.6 --> done below

    if rfc3411.responseClassPDUs.has_key(pduType):
        idx = len(v2VarBinds)-1
        while idx >= 0:
            # 4.1.2.1
            oid, val = v2VarBinds[idx]
            if v2c.Counter64.tagSet == val.tagSet:
                if origV1Pdu.tagSet == v1.GetRequestPDU.tagSet:
                    v1.apiPDU.setErrorStatus(v1Pdu, 2)
                    v1.apiPDU.setErrorIndex(v1Pdu, idx+1)
                    break
                elif origV1Pdu.tagSet == v1.GetNextRequestPDU.tagSet:
                    raise error.StatusInformation(idx=idx, pdu=v2Pdu)
                else:
                    raise error.ProtocolError('Counter64 on the way')

            # 4.1.2.2.1&2
            if exval.noSuchInstance.tagSet == val.tagSet or \
               exval.noSuchObject.tagSet == val.tagSet or \
               exval.endOfMib.tagSet == val.tagSet:
                v1.apiPDU.setErrorStatus(v1Pdu, 2)
                v1.apiPDU.setErrorIndex(v1Pdu, idx+1)

            idx = idx - 1

        # 4.1.2.3.1
        v2ErrorStatus = v2c.apiPDU.getErrorStatus(v2Pdu)
        if v2ErrorStatus:
            v1.apiPDU.setErrorStatus(
                v1Pdu, __v2ToV1ErrorMap[v2ErrorStatus]
                )
            v1.apiPDU.setErrorIndex(v1Pdu, v2c.apiPDU.getErrorIndex(v2Pdu))

    # Translate Var-Binds
    for oid, v2Val in v2VarBinds:
        v1VarBinds.append(
            (oid, __v2ToV1ValueMap[v2Val.tagSet].clone(v2Val))
            )

    v1.apiPDU.setVarBinds(v1Pdu, v1VarBinds)

    return v1Pdu
<commit_msg>* use Trap API for v1 trap PDU management * trapOID is second in v2 Trap PDU var-binding<commit_after># PDU v1/v2c two-way proxy
from pysnmp.proto import rfc3411, error
from pysnmp.proto.api import v1, v2c
from pysnmp.smi import exval

# 2.1.1

__v1ToV2ValueMap = {
    v1.Integer.tagSet: v2c.Integer32(),
    v1.OctetString.tagSet: v2c.OctetString(),
    v1.Null.tagSet: v2c.Null(),
    v1.ObjectIdentifier.tagSet: v2c.ObjectIdentifier(),
    v1.IpAddress.tagSet: v2c.IpAddress(),
    v1.Counter.tagSet: v2c.Counter32(),
    v1.Gauge.tagSet: v2c.Gauge32(),
    v1.TimeTicks.tagSet: v2c.TimeTicks(),
    v1.Opaque.tagSet: v2c.Opaque()
    }

__v2ToV1ValueMap = { # XXX do not re-create same-type items?
    v2c.Integer32.tagSet: v1.Integer(),
    v2c.OctetString.tagSet: v1.OctetString(),
    v2c.Null.tagSet: v1.Null(),
    v2c.ObjectIdentifier.tagSet: v1.ObjectIdentifier(),
    v2c.IpAddress.tagSet: v1.IpAddress(),
    v2c.Counter32.tagSet: v1.Counter(),
    v2c.Gauge32.tagSet: v1.Gauge(),
    v2c.TimeTicks.tagSet: v1.TimeTicks(),
    v2c.Opaque.tagSet: v1.Opaque()
    }

# PDU map

__v1ToV2PduMap = {
    v1.GetRequestPDU.tagSet: v2c.GetRequestPDU(),
    v1.GetNextRequestPDU.tagSet: v2c.GetNextRequestPDU(),
    v1.SetRequestPDU.tagSet: v2c.SetRequestPDU(),
    v1.GetResponsePDU.tagSet: v2c.ResponsePDU(),
    v1.TrapPDU.tagSet: v2c.SNMPv2TrapPDU()
    }

__v2ToV1PduMap = {
    v2c.GetRequestPDU.tagSet: v1.GetRequestPDU(),
    v2c.GetNextRequestPDU.tagSet: v1.GetNextRequestPDU(),
    v2c.SetRequestPDU.tagSet: v1.SetRequestPDU(),
    v2c.ResponsePDU.tagSet: v1.GetResponsePDU(),
    v2c.SNMPv2TrapPDU.tagSet: v1.TrapPDU(),
    v2c.GetBulkRequestPDU.tagSet: v1.GetNextRequestPDU() # 4.1.1
    }

# Trap map

__v1ToV2TrapMap = {
    0: (1,3,6,1,6,3,1,1,5,1),
    1: (1,3,6,1,6,3,1,1,5,2),
    2: (1,3,6,1,6,3,1,1,5,3),
    3: (1,3,6,1,6,3,1,1,5,4),
    4: (1,3,6,1,6,3,1,1,5,5),
    5: (1,3,6,1,6,3,1,1,5,6)
    }

__v2ToV1TrapMap = {
    (1,3,6,1,6,3,1,1,5,1): 0,
    (1,3,6,1,6,3,1,1,5,2): 1,
    (1,3,6,1,6,3,1,1,5,3): 2,
    (1,3,6,1,6,3,1,1,5,4): 3,
    (1,3,6,1,6,3,1,1,5,5): 4,
    (1,3,6,1,6,3,1,1,5,6): 5
    }

# 4.3

__v2ToV1ErrorMap = {
    0:  0,
    1:  1,
    5:  5,
    10: 3,
    9:  3,
    7:  3,
    8:  3,
    12: 3,
    6:  2,
    17: 2,
    11: 2,
    18: 2,
    13: 5,
    14: 5,
    15: 5,
    16: 2
    }

def v1ToV2(v1Pdu, origV2Pdu=None):
    pduType = v1Pdu.tagSet
    v2Pdu = __v1ToV2PduMap[pduType].clone()
    v2c.apiPDU.setDefaults(v2Pdu)

    if not rfc3411.notificationClassPDUs.has_key(pduType):
        requestId = v1.apiPDU.getRequestID(v1Pdu)
        v2c.apiPDU.setRequestID(v2Pdu, long(v1.apiPDU.getRequestID(v1Pdu)))

    v2VarBinds = []

    # 3.1
    if rfc3411.notificationClassPDUs.has_key(pduType):
        # 3.1.1
        sysUpTime = v1.apiTrapPDU.getTimeStamp(v1Pdu)

        # 3.1.2
        genericTrap = v1.apiTrapPDU.getGenericTrap(v1Pdu)
        if genericTrap == 6:
            snmpTrapOID = v1.apiTrapPDU.getEnterprise(v1Pdu) + (0,) + \
                          v1.apiTrapPDU.getSpecificTrap(v1Pdu)

        # 3.1.3
        else:
            snmpTrapOID = __v1ToV2TrapMap[genericTrap]

        v2VarBinds.append((sysUpTime, None))
        v2VarBinds.append((snmpTrapOID, None))

        # 3.1.4 --> done below

    # Translate Var-Binds
    for oid, v1Val in v1.apiPDU.getVarBinds(v1Pdu):
        v2VarBinds.append(
            (oid, __v1ToV2ValueMap[v1Val.tagSet].clone(v1Val))
            )

    if rfc3411.responseClassPDUs.has_key(pduType):
        # 4.1.2.2 --> one-to-one mapping
        v2c.apiPDU.setErrorStatus(
            v2Pdu, int(v1.apiPDU.getErrorStatus(v1Pdu))
            )
        v2c.apiPDU.setErrorIndex(
            v2Pdu, int(v1.apiPDU.getErrorIndex(v1Pdu))
            )

        # 4.1.2.1 --> no-op

    v2c.apiPDU.setVarBinds(v2Pdu, v2VarBinds)

    return v2Pdu

def v2ToV1(v2Pdu, origV1Pdu=None):
    pduType = v2Pdu.tagSet

    v1Pdu = __v2ToV1PduMap[pduType].clone()

    v2VarBinds = v2c.apiPDU.getVarBinds(v2Pdu)
    v1VarBinds = []

    if rfc3411.notificationClassPDUs.has_key(pduType):
        v1.apiTrapPDU.setDefaults(v1Pdu)
    else:
        v1.apiPDU.setDefaults(v1Pdu)

    # 3.2
    if rfc3411.notificationClassPDUs.has_key(pduType):
        # 3.2.1
        snmpTrapOID = v2VarBinds[1][0]
        if __v2ToV1TrapMap.has_key(snmpTrapOID):
            for oid, val in v2VarBinds:
                # snmpTrapEnterprise
                if oid == (1, 3, 6, 1, 6, 3, 1, 1, 4, 3, 0):
                    v1.apiTrapPDU.setEnterprise(v1Pdu, val)
                    break
            else:
                # snmpTraps
                v1.apiTrapPDU.setEnterprise(v1Pdu, (1, 3, 6, 1, 6, 3, 1, 1, 5))
        else:
            if snmpTrapOID[-2] == 0:
                v1.apiTrapPDU.setEnterprise(v1Pdu, snmpTrapOID[:-2])
            else:
                v1.apiTrapPDU.setEnterprise(v1Pdu, snmpTrapOID[:-1])

        # 3.2.2
        for oid, val in v2VarBinds:
            # snmpTrapAddress
            if oid == (1, 3, 6, 1, 6, 3, 18, 1, 3, 0):
                v1.apiTrapPDU.setAgentAddr(v1Pdu, val)
                break
        else:
            v1.apiTrapPDU.setAgentAddr(v1Pdu, '0.0.0.0')

        # 3.2.3
        if __v2ToV1TrapMap.has_key(snmpTrapOID):
            v1.apiTrapPDU.setGenericTrap(v1Pdu, __v2ToV1TrapMap[snmpTrapOID])
        else:
            v1.apiTrapPDU.setGenericTrap(v1Pdu, 0)

        # 3.2.4
        if __v2ToV1TrapMap.has_key(snmpTrapOID):
            v1.apiTrapPDU.setSpecificTrap(v1Pdu, 0)
        else:
            v1.apiTrapPDU.setSpecificTrap(v1Pdu, snmpTrapOID[-1])

        # 3.2.5
        v1.apiTrapPDU.setTimeStamp(v1Pdu, v2VarBinds[0][1])

        # 3.2.6 --> done below

    if rfc3411.responseClassPDUs.has_key(pduType):
        idx = len(v2VarBinds)-1
        while idx >= 0:
            # 4.1.2.1
            oid, val = v2VarBinds[idx]
            if v2c.Counter64.tagSet == val.tagSet:
                if origV1Pdu.tagSet == v1.GetRequestPDU.tagSet:
                    v1.apiPDU.setErrorStatus(v1Pdu, 2)
                    v1.apiPDU.setErrorIndex(v1Pdu, idx+1)
                    break
                elif origV1Pdu.tagSet == v1.GetNextRequestPDU.tagSet:
                    raise error.StatusInformation(idx=idx, pdu=v2Pdu)
                else:
                    raise error.ProtocolError('Counter64 on the way')

            # 4.1.2.2.1&2
            if exval.noSuchInstance.tagSet == val.tagSet or \
               exval.noSuchObject.tagSet == val.tagSet or \
               exval.endOfMib.tagSet == val.tagSet:
                v1.apiPDU.setErrorStatus(v1Pdu, 2)
                v1.apiPDU.setErrorIndex(v1Pdu, idx+1)

            idx = idx - 1

        # 4.1.2.3.1
        v2ErrorStatus = v2c.apiPDU.getErrorStatus(v2Pdu)
        if v2ErrorStatus:
            v1.apiPDU.setErrorStatus(
                v1Pdu, __v2ToV1ErrorMap[v2ErrorStatus]
                )
            v1.apiPDU.setErrorIndex(v1Pdu, v2c.apiPDU.getErrorIndex(v2Pdu))

    # Translate Var-Binds
    for oid, v2Val in v2VarBinds:
        v1VarBinds.append(
            (oid, __v2ToV1ValueMap[v2Val.tagSet].clone(v2Val))
            )

    if rfc3411.notificationClassPDUs.has_key(pduType):
        v1.apiTrapPDU.setVarBinds(v1Pdu, v1VarBinds)
    else:
        v1.apiPDU.setVarBinds(v1Pdu, v1VarBinds)

    return v1Pdu
<|endoftext|>


<commit_before>import os
import json
from itertools import chain

from jinja2 import Environment, FileSystemLoader, filters

filters.FILTERS['repr'] = repr

TYPE_MAP = {'oneOf': 'Union',
            "array": "List",
            "boolean": "Bool",
            "number": "CFloat",
            "string": "Unicode",
            "object": "Any",
           }

CLASS_ALIASES = {
    'PositionChannel': {'base': 'PositionChannelDef', 'type': 'channel'},
    'ChannelWithLegend': {'base': 'ChannelDefWithLegend', 'type': 'channel'},
    'Field': {'base': 'FieldDef', 'type': 'channel'},
    'OrderChannel': {'base': 'OrderChannelDef', 'type': 'channel'}
    }

def getpath(*args):
    return os.path.abspath(os.path.join(os.path.dirname(__file__), *args))

class SchemaProperty(object):
    """Class Wrapper for a property in a VegaLite Schema

    This class exposes methods used in the templating.
    """
    def __init__(self, schema, name, top):
        self.name = name
        self.schema = schema
        self.top = top

        self.properties = {k: SchemaProperty(v, k, self.top)
                           for k, v in self.schema.get('properties', {}).items()}

    @property
    def subtypes(self):
        trait = self.trait_name
        if trait == 'Union':
            return [self.__class__(s, '', self.top) for s in self.schema['oneOf']]
        elif trait == 'List':
            return [self.__class__(self.schema['items'], '', self.top)]
        else:
            return []

    @property
    def type(self):
        if '$ref' in self.schema:
            return '$ref'
        elif 'oneOf' in self.schema:
            return 'oneOf'
        else:
            return self.schema.get('type', '')

    @property
    def trait_name(self):
        if self.refname:
            trait = self.top.definitions[self.refname].trait_name
            if trait == 'Any':
                return 'Instance'
            else:
                return self.refname
        else:
            return TYPE_MAP.get(self.type, "Any")

    @property
    def trait_descr(self):
        trait = self.trait_name
        if trait == 'Instance':
            return self.refname
        elif self.subtypes:
            return '{0}({1})'.format(trait, ', '.join(t.trait_descr
                                                      for t in self.subtypes))
        else:
            return trait

    @property
    def trait_fulldef(self):
        trait = self.trait_name
        kwds = "allow_none=True, default_value=None"
        if 'minimum' in self.schema:
            kwds += ', min={0}'.format(self.schema['minimum'])
        if 'maximum' in self.schema:
            kwds += ', max={0}'.format(self.schema['maximum'])
        if self.description:
            kwds += ', help="""{0}"""'.format(self.short_description)

        if trait == 'Union':
            return ('T.Union([{0}])'
                    ''.format(', '.join(t.trait_fulldef for t in self.subtypes)))
        elif trait == 'List':
            return 'T.List({0}, {1})'.format(*(t.trait_fulldef for t in self.subtypes), kwds)
        elif trait == 'Instance':
            return 'T.Instance({0}, {1})'.format(self.refname, kwds)
        elif trait == self.refname:
            return '{0}({1})'.format(self.refname, kwds)
        else:
            return 'T.{0}({1})'.format(trait, kwds)

    @property
    def description(self):
        return self.schema.get('description', '')

    @property
    def short_description(self):
        if self.description:
            return self.description.split('(e.g.')[0].split('.')[0] + '.'
        else:
            return ''

    @property
    def long_description(self):
        return self.description

    @property
    def refname(self):
        if '$ref' in self.schema:
            return self.schema['$ref'].split('/')[-1]
        else:
            return None

    @property
    def imports(self):
        def gen_imports():
            if self.refname:
                yield self.refname
            for t in self.subtypes:
                yield from t.imports
            for v in self.properties.values():
                yield from v.imports
        return sorted(gen_imports())

    @property
    def attributes(self):
        return [v for k, v in sorted(self.properties.items())]

    @property
    def enum(self):
        return self.schema.get('enum', [])

class VegaLiteSchema(SchemaProperty):
    """
    This is a wrapper for the vegalite JSON schema that provides tools to
    export Python wrappers.
    """
    def __init__(self, schema_file=None):
        if schema_file is None:
            schema_file = getpath('..', 'altair', 'schema',
                                  'vega-lite-schema.json')
        with open(schema_file) as f:
            schema = json.load(f)

        self.definitions = {k: SchemaProperty(v, k, self)
                            for k, v in schema['definitions'].items()}

        template_path = getpath('templates')
        self.templates = Environment(loader=FileSystemLoader(template_path))

        super(VegaLiteSchema, self).__init__(schema, 'VegaLiteSchema', self)

    def write_interface(self, path=None):
        # Make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_interface')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing code to {0}".format(path))

        # Write Init File
        template = self.templates.get_template('__init__.py.tpl')
        header="Auto-generated Python wrappers for Vega-Lite Schema"
        print(" - Writing __init__.py")
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(objects=sorted(self.definitions),
                                    header=header))

        # Write Class Definition files
        templates = {'string': self.templates.get_template('enum.py.tpl'),
                     'object': self.templates.get_template('object.py.tpl')}
        for key, prop in sorted(self.definitions.items()):
            if prop.type not in templates:
                raise ValueError("No template for type={0}".format(prop.type))
            outfile = os.path.join(path, '{0}.py'.format(key.lower()))
            print(" - Writing {0}".format(os.path.basename(outfile)))
            with open(outfile, 'w') as f:
                f.write(templates[prop.type].render(cls=prop))

    def write_interface_tests(self, path=None):
        # Make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_interface', 'tests')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing tests to {0}".format(path))

        # Write test Init File
        template = self.templates.get_template('__init__.py.tpl')
        header = 'Auto-generated tests for Vega-Lite Schema wrappers'
        print(" - Writing __init__.py")
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(objects=[], header=header))

        # Write test file
        template = self.templates.get_template('test_instantiations.py.tpl')
        classes = [prop for key, prop in sorted(self.definitions.items())]
        print(" - Writing test_instantiations.py")
        with open(os.path.join(path, 'test_instantiations.py'), 'w') as f:
            f.write(template.render(classes=classes))

    def write_wrappers(self, path=None):
        # make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_wrappers')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing wrappers to {0}".format(path))

        # get all channel class aliases
        classes = sorted([(k, v['base'])
                          for k, v in CLASS_ALIASES.items()
                          if v['type'] == 'channel'])

        # write the init file
        template = self.templates.get_template('__init__.py.tpl')
        header = 'Wrappers for low-level schema objects'
        objects = [cls[0] for cls in classes]
        print(" - Writing __init__.py")
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(header=header, objects=objects))

        # write the class definition files
        template = self.templates.get_template('channel_defs.py.tpl')
        for cls, basename in sorted(classes):
            base = self.definitions[basename]
            filename = os.path.join(path, '{0}.py'.format(cls.lower()))
            print("- Writing {0}".format(filename))
            with open(filename, 'w') as f:
                f.write(template.render(cls=cls, base=base))

if __name__ == '__main__':
    schema = VegaLiteSchema()
    schema.write_interface()
    schema.write_interface_tests()
    schema.write_wrappers()
<commit_msg>PEP8 to re-trigger CI<commit_after>import os
import json
from itertools import chain

from jinja2 import Environment, FileSystemLoader, filters

filters.FILTERS['repr'] = repr

TYPE_MAP = {'oneOf': 'Union',
            "array": "List",
            "boolean": "Bool",
            "number": "CFloat",
            "string": "Unicode",
            "object": "Any",
            }

CLASS_ALIASES = {
    'PositionChannel': {'base': 'PositionChannelDef', 'type': 'channel'},
    'ChannelWithLegend': {'base': 'ChannelDefWithLegend', 'type': 'channel'},
    'Field': {'base': 'FieldDef', 'type': 'channel'},
    'OrderChannel': {'base': 'OrderChannelDef', 'type': 'channel'}
    }

def getpath(*args):
    return os.path.abspath(os.path.join(os.path.dirname(__file__), *args))

class SchemaProperty(object):
    """Class Wrapper for a property in a VegaLite Schema

    This class exposes methods used in the templating.
    """
    def __init__(self, schema, name, top):
        self.name = name
        self.schema = schema
        self.top = top

        self.properties = {k: SchemaProperty(v, k, self.top)
                           for k, v in self.schema.get('properties',
                                                       {}).items()}

    @property
    def subtypes(self):
        trait = self.trait_name
        if trait == 'Union':
            return [self.__class__(s, '', self.top)
                    for s in self.schema['oneOf']]
        elif trait == 'List':
            return [self.__class__(self.schema['items'], '', self.top)]
        else:
            return []

    @property
    def type(self):
        if '$ref' in self.schema:
            return '$ref'
        elif 'oneOf' in self.schema:
            return 'oneOf'
        else:
            return self.schema.get('type', '')

    @property
    def trait_name(self):
        if self.refname:
            trait = self.top.definitions[self.refname].trait_name
            if trait == 'Any':
                return 'Instance'
            else:
                return self.refname
        else:
            return TYPE_MAP.get(self.type, "Any")

    @property
    def trait_descr(self):
        trait = self.trait_name
        if trait == 'Instance':
            return self.refname
        elif self.subtypes:
            return '{0}({1})'.format(trait, ', '.join(t.trait_descr
                                                      for t in self.subtypes))
        else:
            return trait

    @property
    def trait_fulldef(self):
        trait = self.trait_name
        kwds = "allow_none=True, default_value=None"
        if 'minimum' in self.schema:
            kwds += ', min={0}'.format(self.schema['minimum'])
        if 'maximum' in self.schema:
            kwds += ', max={0}'.format(self.schema['maximum'])
        if self.description:
            kwds += ', help="""{0}"""'.format(self.short_description)

        if trait == 'Union':
            return ('T.Union([{0}])'
                    ''.format(', '.join(t.trait_fulldef
                                        for t in self.subtypes)))
        elif trait == 'List':
            return 'T.List({0}, {1})'.format(*(t.trait_fulldef
                                               for t in self.subtypes), kwds)
        elif trait == 'Instance':
            return 'T.Instance({0}, {1})'.format(self.refname, kwds)
        elif trait == self.refname:
            return '{0}({1})'.format(self.refname, kwds)
        else:
            return 'T.{0}({1})'.format(trait, kwds)

    @property
    def description(self):
        return self.schema.get('description', '')

    @property
    def short_description(self):
        if self.description:
            return self.description.split('(e.g.')[0].split('.')[0] + '.'
        else:
            return ''

    @property
    def long_description(self):
        return self.description

    @property
    def refname(self):
        if '$ref' in self.schema:
            return self.schema['$ref'].split('/')[-1]
        else:
            return None

    @property
    def imports(self):
        def gen_imports():
            if self.refname:
                yield self.refname
            for t in self.subtypes:
                yield from t.imports
            for v in self.properties.values():
                yield from v.imports
        return sorted(gen_imports())

    @property
    def attributes(self):
        return [v for k, v in sorted(self.properties.items())]

    @property
    def enum(self):
        return self.schema.get('enum', [])

class VegaLiteSchema(SchemaProperty):
    """
    This is a wrapper for the vegalite JSON schema that provides tools to
    export Python wrappers.
    """
    def __init__(self, schema_file=None):
        if schema_file is None:
            schema_file = getpath('..', 'altair', 'schema',
                                  'vega-lite-schema.json')
        with open(schema_file) as f:
            schema = json.load(f)

        self.definitions = {k: SchemaProperty(v, k, self)
                            for k, v in schema['definitions'].items()}

        template_path = getpath('templates')
        self.templates = Environment(loader=FileSystemLoader(template_path))

        super(VegaLiteSchema, self).__init__(schema, 'VegaLiteSchema', self)

    def write_interface(self, path=None):
        # Make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_interface')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing code to {0}".format(path))

        # Write Init File
        template = self.templates.get_template('__init__.py.tpl')
        header = "Auto-generated Python wrappers for Vega-Lite Schema"
        print(" - Writing __init__.py")
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(objects=sorted(self.definitions),
                                    header=header))

        # Write Class Definition files
        templates = {'string': self.templates.get_template('enum.py.tpl'),
                     'object': self.templates.get_template('object.py.tpl')}
        for key, prop in sorted(self.definitions.items()):
            if prop.type not in templates:
                raise ValueError("No template for type={0}".format(prop.type))
            outfile = os.path.join(path, '{0}.py'.format(key.lower()))
            print(" - Writing {0}".format(os.path.basename(outfile)))
            with open(outfile, 'w') as f:
                f.write(templates[prop.type].render(cls=prop))

    def write_interface_tests(self, path=None):
        # Make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_interface', 'tests')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing tests to {0}".format(path))

        # Write test Init File
        template = self.templates.get_template('__init__.py.tpl')
        header = 'Auto-generated tests for Vega-Lite Schema wrappers'
        print(" - Writing __init__.py")
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(objects=[], header=header))

        # Write test file
        template = self.templates.get_template('test_instantiations.py.tpl')
        classes = [prop for key, prop in sorted(self.definitions.items())]
        print(" - Writing test_instantiations.py")
        with open(os.path.join(path, 'test_instantiations.py'), 'w') as f:
            f.write(template.render(classes=classes))

    def write_wrappers(self, path=None):
        # make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_wrappers')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing wrappers to {0}".format(path))

        # get all channel class aliases
        classes = sorted([(k, v['base'])
                          for k, v in CLASS_ALIASES.items()
                          if v['type'] == 'channel'])

        # write the init file
        template = self.templates.get_template('__init__.py.tpl')
        header = 'Wrappers for low-level schema objects'
        objects = [cls[0] for cls in classes]
        print(" - Writing __init__.py")
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(header=header, objects=objects))

        # write the class definition files
        template = self.templates.get_template('channel_defs.py.tpl')
        for cls, basename in sorted(classes):
            base = self.definitions[basename]
            filename = os.path.join(path, '{0}.py'.format(cls.lower()))
            print("- Writing {0}".format(filename))
            with open(filename, 'w') as f:
                f.write(template.render(cls=cls, base=base))

if __name__ == '__main__':
    schema = VegaLiteSchema()
    schema.write_interface()
    schema.write_interface_tests()
    schema.write_wrappers()
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import flax
import jax
import jax.numpy as jnp
import numpy as np

class Optimizer(flax.optim.OptimizerDef):
  """Momentum optimizer that stores state using half-precision."""

  @flax.struct.dataclass
  class HyperParams:
    learning_rate: np.ndarray
    beta: np.ndarray
    grad_norm_clip: np.ndarray

  @flax.struct.dataclass
  class State:
    momentum: np.ndarray

  def __init__(self,
               learning_rate=None,
               beta=0.9,
               dtype='bfloat16',
               grad_norm_clip=None):
    hyper_params = Optimizer.HyperParams(learning_rate, beta, grad_norm_clip)
    super().__init__(hyper_params)
    self.dtype = dict(bfloat16=jnp.bfloat16, float32=jnp.float32)

  def init_param_state(self, param):
    return Optimizer.State(jnp.zeros_like(param, dtype=jnp.float16))

  def apply_gradient(self, hyper_params, params, state, grads):
    step = state.step
    params_flat, treedef = jax.tree_flatten(params)
    states_flat = treedef.flatten_up_to(state.param_states)
    grads_flat = treedef.flatten_up_to(grads)

    # Optionally resize the global gradient to a maximum norm.
    if hyper_params.grad_norm_clip:
      grads_l2 = jnp.sqrt(sum([jnp.vdot(p, p) for p in grads_flat]))
      grads_factor = jnp.minimum(1.0, hyper_params.grad_norm_clip / grads_l2)
      grads_flat = jax.tree_map(lambda param: grads_factor * param, grads_flat)

    out = [
        self.apply_param_gradient(step, hyper_params, param, state, grad)
        for param, state, grad in zip(params_flat, states_flat, grads_flat)
    ]

    new_params_flat, new_states_flat = list(zip(*out)) if out else ((), ())
    new_params = jax.tree_unflatten(treedef, new_params_flat)
    new_param_states = jax.tree_unflatten(treedef, new_states_flat)
    new_state = flax.optim.OptimizerState(step + 1, new_param_states)
    return new_params, new_state

  def apply_param_gradient(self, step, hyper_params, param, state, grad):
    del step
    assert hyper_params.learning_rate is not None, 'no learning rate provided.'
    momentum = state.momentum
    new_momentum = hyper_params.beta * momentum + grad
    new_param = param - hyper_params.learning_rate * new_momentum
    new_state = Optimizer.State(new_momentum.astype(jnp.float16))
    return new_param, new_state
<commit_msg>Bugfix dtype.<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import flax
import jax
import jax.numpy as jnp
import numpy as np

class Optimizer(flax.optim.OptimizerDef):
  """Momentum optimizer that stores state using half-precision."""

  @flax.struct.dataclass
  class HyperParams:
    learning_rate: np.ndarray
    beta: np.ndarray
    grad_norm_clip: np.ndarray

  @flax.struct.dataclass
  class State:
    momentum: np.ndarray

  def __init__(self,
               learning_rate=None,
               beta=0.9,
               dtype='bfloat16',
               grad_norm_clip=None):
    hyper_params = Optimizer.HyperParams(learning_rate, beta, grad_norm_clip)
    super().__init__(hyper_params)
    self.dtype = dict(bfloat16=jnp.bfloat16, float32=jnp.float32)[dtype]

  def init_param_state(self, param):
    return Optimizer.State(jnp.zeros_like(param, dtype=self.dtype))

  def apply_gradient(self, hyper_params, params, state, grads):
    step = state.step
    params_flat, treedef = jax.tree_flatten(params)
    states_flat = treedef.flatten_up_to(state.param_states)
    grads_flat = treedef.flatten_up_to(grads)

    # Optionally resize the global gradient to a maximum norm.
    if hyper_params.grad_norm_clip:
      grads_l2 = jnp.sqrt(sum([jnp.vdot(p, p) for p in grads_flat]))
      grads_factor = jnp.minimum(1.0, hyper_params.grad_norm_clip / grads_l2)
      grads_flat = jax.tree_map(lambda param: grads_factor * param, grads_flat)

    out = [
        self.apply_param_gradient(step, hyper_params, param, state, grad)
        for param, state, grad in zip(params_flat, states_flat, grads_flat)
    ]

    new_params_flat, new_states_flat = list(zip(*out)) if out else ((), ())
    new_params = jax.tree_unflatten(treedef, new_params_flat)
    new_param_states = jax.tree_unflatten(treedef, new_states_flat)
    new_state = flax.optim.OptimizerState(step + 1, new_param_states)
    return new_params, new_state

  def apply_param_gradient(self, step, hyper_params, param, state, grad):
    del step
    assert hyper_params.learning_rate is not None, 'no learning rate provided.'
    momentum = state.momentum
    new_momentum = hyper_params.beta * momentum + grad
    new_param = param - hyper_params.learning_rate * new_momentum
    new_state = Optimizer.State(new_momentum.astype(self.dtype))
    return new_param, new_state
<|endoftext|>


<commit_before># coding=utf8
"""
admin.py - Sopel Admin Module
Copyright 2010-2011, Michael Yanovich, Alek Rollyson, and Edward Powell
Copyright © 2012, Elad Alfassa <elad@fedoraproject.org>
Licensed under the Eiffel Forum License 2.

http://sopel.chat/

"""
from __future__ import unicode_literals

import re
from sopel import formatting
from sopel.module import commands, priority, OP, HALFOP, require_privilege
from sopel.tools import Identifier

def default_mask(trigger):
    welcome = formatting.color('Welcome to:', formatting.colors.PURPLE)
    chan = formatting.color(trigger.sender, formatting.colors.TEAL)
    topic_ = formatting.bold('Topic:')
    topic_ = formatting.color('| ' + topic_, formatting.colors.PURPLE)
    arg = formatting.color('{}', formatting.colors.GREEN)
    return '{} {} {} {}'.format(welcome, chan, topic_, arg)

@require_privilege(OP)
@commands('op')
def op(bot, trigger):
    """
    Command to op users in a room. If no nick is given,
    sopel will op the nick who sent the command
    """
    if bot.privileges[trigger.sender][bot.nick] < OP:
        return bot.reply("I'm not a channel operator!")
    nick = trigger.group(2)
    channel = trigger.sender
    if not nick:
        nick = trigger.nick
    bot.write(['MODE', channel, "+o", nick])

@require_privilege(OP)
@commands('deop')
def deop(bot, trigger):
    """
    Command to deop users in a room. If no nick is given,
    sopel will deop the nick who sent the command
    """
    if bot.privileges[trigger.sender][bot.nick] < OP:
        return bot.reply("I'm not a channel operator!")
    nick = trigger.group(2)
    channel = trigger.sender
    if not nick:
        nick = trigger.nick
    bot.write(['MODE', channel, "-o", nick])

@require_privilege(OP)
@commands('voice')
def voice(bot, trigger):
    """
    Command to voice users in a room. If no nick is given,
    sopel will voice the nick who sent the command
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    nick = trigger.group(2)
    channel = trigger.sender
    if not nick:
        nick = trigger.nick
    bot.write(['MODE', channel, "+v", nick])

@require_privilege(OP)
@commands('devoice')
def devoice(bot, trigger):
    """
    Command to devoice users in a room. If no nick is given,
    sopel will devoice the nick who sent the command
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    nick = trigger.group(2)
    channel = trigger.sender
    if not nick:
        nick = trigger.nick
    bot.write(['MODE', channel, "-v", nick])

@require_privilege(OP)
@commands('kick')
@priority('high')
def kick(bot, trigger):
    """
    Kick a user from the channel.
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    nick = opt
    channel = trigger.sender
    reasonidx = 2
    if not opt.is_nick():
        if argc < 3:
            return
        nick = text[2]
        channel = opt
        reasonidx = 3
    reason = ' '.join(text[reasonidx:])
    if nick != bot.config.core.nick:
        bot.write(['KICK', channel, nick, reason])

def configureHostMask(mask):
    if mask == '*!*@*':
        return mask
    if re.match('^[^.@!/]+$', mask) is not None:
        return '%s!*@*' % mask
    if re.match('^[^@!]+$', mask) is not None:
        return '*!*@%s' % mask

    m = re.match('^([^!@]+)@$', mask)
    if m is not None:
        return '*!%s@*' % m.group(1)

    m = re.match('^([^!@]+)@([^@!]+)$', mask)
    if m is not None:
        return '*!%s@%s' % (m.group(1), m.group(2))

    m = re.match('^([^!@]+)!(^[!@]+)@?$', mask)
    if m is not None:
        return '%s!%s@*' % (m.group(1), m.group(2))
    return ''

@require_privilege(OP)
@commands('ban')
@priority('high')
def ban(bot, trigger):
    """
    This give admins the ability to ban a user.
    The bot must be a Channel Operator for this command to work.
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    banmask = opt
    channel = trigger.sender
    if not opt.is_nick():
        if argc < 3:
            return
        channel = opt
        banmask = text[2]
    banmask = configureHostMask(banmask)
    if banmask == '':
        return
    bot.write(['MODE', channel, '+b', banmask])

@require_privilege(OP)
@commands('unban')
def unban(bot, trigger):
    """
    This give admins the ability to unban a user.
    The bot must be a Channel Operator for this command to work.
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    banmask = opt
    channel = trigger.sender
    if not opt.is_nick():
        if argc < 3:
            return
        channel = opt
        banmask = text[2]
    banmask = configureHostMask(banmask)
    if banmask == '':
        return
    bot.write(['MODE', channel, '-b', banmask])

@require_privilege(OP)
@commands('quiet')
def quiet(bot, trigger):
    """
    This gives admins the ability to quiet a user.
    The bot must be a Channel Operator for this command to work.
    """
    if bot.privileges[trigger.sender][bot.nick] < OP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    quietmask = opt
    channel = trigger.sender
    if not opt.is_nick():
        if argc < 3:
            return
        quietmask = text[2]
        channel = opt
    quietmask = configureHostMask(quietmask)
    if quietmask == '':
        return
    bot.write(['MODE', channel, '+q', quietmask])

@require_privilege(OP)
@commands('unquiet')
def unquiet(bot, trigger):
    """
    This gives admins the ability to unquiet a user.
    The bot must be a Channel Operator for this command to work.
    """
    if bot.privileges[trigger.sender][bot.nick] < OP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    quietmask = opt
    channel = trigger.sender
    if not opt.is_nick():
        if argc < 3:
            return
        quietmask = text[2]
        channel = opt
    quietmask = configureHostMask(quietmask)
    if quietmask == '':
        return
    bot.write(['MODE', channel, '-q', quietmask])

@require_privilege(OP)
@commands('kickban', 'kb')
@priority('high')
def kickban(bot, trigger):
    """
    This gives admins the ability to kickban a user.
    The bot must be a Channel Operator for this command to work.
    .kickban [#chan] user1 user!*@* get out of here
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 4:
        return
    opt = Identifier(text[1])
    nick = opt
    mask = text[2]
    reasonidx = 3
    if not opt.is_nick():
        if argc < 5:
            return
        channel = opt
        nick = text[2]
        mask = text[3]
        reasonidx = 4
    reason = ' '.join(text[reasonidx:])
    mask = configureHostMask(mask)
    if mask == '':
        return
    bot.write(['MODE', channel, '+b', mask])
    bot.write(['KICK', channel, nick, ' :', reason])

@require_privilege(OP)
@commands('topic')
def topic(bot, trigger):
    """
    This gives ops the ability to change the topic.
    The bot must be a Channel Operator for this command to work.
    """
    purple, green, bold = '\x0306', '\x0310', '\x02'
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    if not trigger.group(2):
        return
    channel = trigger.sender.lower()

    narg = 1
    mask = None
    mask = bot.db.get_channel_value(channel, 'topic_mask')
    mask = mask or default_mask(trigger)
    mask = mask.replace('%s', '{}')
    narg = len(re.findall('{}', mask))

    top = trigger.group(2)
    args = []
    if top:
        args = top.split('~', narg)

    if len(args) != narg:
        message = "Not enough arguments. You gave {}, it requires {}.".format(
            len(args), narg)
        return bot.say(message)
    topic = mask.format(*args)

    bot.write(('TOPIC', channel + ' :' + topic))

@require_privilege(OP)
@commands('tmask')
def set_mask(bot, trigger):
    """
    Set the mask to use for .topic in the current channel. {} is used to allow
    substituting in chunks of text.
    """
    bot.db.set_channel_value(trigger.sender, 'topic_mask', trigger.group(2))
    bot.say("Gotcha, " + trigger.nick)

@require_privilege(OP)
@commands('showmask')
def show_mask(bot, trigger):
    """Show the topic mask for the current channel."""
    mask = bot.db.get_channel_value(trigger.sender, 'topic_mask')
    mask = mask or default_mask(trigger)
    bot.say(mask)
<commit_msg>[adminchannel] Fix .kick & .kickban<commit_after># coding=utf8
"""
admin.py - Sopel Admin Module
Copyright 2010-2011, Michael Yanovich, Alek Rollyson, and Edward Powell
Copyright © 2012, Elad Alfassa <elad@fedoraproject.org>
Licensed under the Eiffel Forum License 2.

http://sopel.chat/

"""
from __future__ import unicode_literals

import re
from sopel import formatting
from sopel.module import commands, priority, OP, HALFOP, require_privilege
from sopel.tools import Identifier

def default_mask(trigger):
    welcome = formatting.color('Welcome to:', formatting.colors.PURPLE)
    chan = formatting.color(trigger.sender, formatting.colors.TEAL)
    topic_ = formatting.bold('Topic:')
    topic_ = formatting.color('| ' + topic_, formatting.colors.PURPLE)
    arg = formatting.color('{}', formatting.colors.GREEN)
    return '{} {} {} {}'.format(welcome, chan, topic_, arg)

@require_privilege(OP)
@commands('op')
def op(bot, trigger):
    """
    Command to op users in a room. If no nick is given,
    sopel will op the nick who sent the command
    """
    if bot.privileges[trigger.sender][bot.nick] < OP:
        return bot.reply("I'm not a channel operator!")
    nick = trigger.group(2)
    channel = trigger.sender
    if not nick:
        nick = trigger.nick
    bot.write(['MODE', channel, "+o", nick])

@require_privilege(OP)
@commands('deop')
def deop(bot, trigger):
    """
    Command to deop users in a room. If no nick is given,
    sopel will deop the nick who sent the command
    """
    if bot.privileges[trigger.sender][bot.nick] < OP:
        return bot.reply("I'm not a channel operator!")
    nick = trigger.group(2)
    channel = trigger.sender
    if not nick:
        nick = trigger.nick
    bot.write(['MODE', channel, "-o", nick])

@require_privilege(OP)
@commands('voice')
def voice(bot, trigger):
    """
    Command to voice users in a room. If no nick is given,
    sopel will voice the nick who sent the command
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    nick = trigger.group(2)
    channel = trigger.sender
    if not nick:
        nick = trigger.nick
    bot.write(['MODE', channel, "+v", nick])

@require_privilege(OP)
@commands('devoice')
def devoice(bot, trigger):
    """
    Command to devoice users in a room. If no nick is given,
    sopel will devoice the nick who sent the command
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    nick = trigger.group(2)
    channel = trigger.sender
    if not nick:
        nick = trigger.nick
    bot.write(['MODE', channel, "-v", nick])

@require_privilege(OP)
@commands('kick')
@priority('high')
def kick(bot, trigger):
    """
    Kick a user from the channel.
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    nick = opt
    channel = trigger.sender
    reasonidx = 2
    if not opt.is_nick():
        if argc < 3:
            return
        nick = text[2]
        channel = opt
        reasonidx = 3
    reason = ' '.join(text[reasonidx:])
    if nick != bot.config.core.nick:
        bot.write(['KICK', channel, nick], reason)

def configureHostMask(mask):
    if mask == '*!*@*':
        return mask
    if re.match('^[^.@!/]+$', mask) is not None:
        return '%s!*@*' % mask
    if re.match('^[^@!]+$', mask) is not None:
        return '*!*@%s' % mask

    m = re.match('^([^!@]+)@$', mask)
    if m is not None:
        return '*!%s@*' % m.group(1)

    m = re.match('^([^!@]+)@([^@!]+)$', mask)
    if m is not None:
        return '*!%s@%s' % (m.group(1), m.group(2))

    m = re.match('^([^!@]+)!(^[!@]+)@?$', mask)
    if m is not None:
        return '%s!%s@*' % (m.group(1), m.group(2))
    return ''

@require_privilege(OP)
@commands('ban')
@priority('high')
def ban(bot, trigger):
    """
    This give admins the ability to ban a user.
    The bot must be a Channel Operator for this command to work.
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    banmask = opt
    channel = trigger.sender
    if not opt.is_nick():
        if argc < 3:
            return
        channel = opt
        banmask = text[2]
    banmask = configureHostMask(banmask)
    if banmask == '':
        return
    bot.write(['MODE', channel, '+b', banmask])

@require_privilege(OP)
@commands('unban')
def unban(bot, trigger):
    """
    This give admins the ability to unban a user.
    The bot must be a Channel Operator for this command to work.
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    banmask = opt
    channel = trigger.sender
    if not opt.is_nick():
        if argc < 3:
            return
        channel = opt
        banmask = text[2]
    banmask = configureHostMask(banmask)
    if banmask == '':
        return
    bot.write(['MODE', channel, '-b', banmask])

@require_privilege(OP)
@commands('quiet')
def quiet(bot, trigger):
    """
    This gives admins the ability to quiet a user.
    The bot must be a Channel Operator for this command to work.
    """
    if bot.privileges[trigger.sender][bot.nick] < OP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    quietmask = opt
    channel = trigger.sender
    if not opt.is_nick():
        if argc < 3:
            return
        quietmask = text[2]
        channel = opt
    quietmask = configureHostMask(quietmask)
    if quietmask == '':
        return
    bot.write(['MODE', channel, '+q', quietmask])

@require_privilege(OP)
@commands('unquiet')
def unquiet(bot, trigger):
    """
    This gives admins the ability to unquiet a user.
    The bot must be a Channel Operator for this command to work.
    """
    if bot.privileges[trigger.sender][bot.nick] < OP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 2:
        return
    opt = Identifier(text[1])
    quietmask = opt
    channel = trigger.sender
    if not opt.is_nick():
        if argc < 3:
            return
        quietmask = text[2]
        channel = opt
    quietmask = configureHostMask(quietmask)
    if quietmask == '':
        return
    bot.write(['MODE', channel, '-q', quietmask])

@require_privilege(OP)
@commands('kickban', 'kb')
@priority('high')
def kickban(bot, trigger):
    """
    This gives admins the ability to kickban a user.
    The bot must be a Channel Operator for this command to work.
    .kickban [#chan] user1 user!*@* get out of here
    """
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    text = trigger.group().split()
    argc = len(text)
    if argc < 4:
        return
    opt = Identifier(text[1])
    nick = opt
    mask = text[2]
    channel = trigger.sender
    reasonidx = 3
    if not opt.is_nick():
        if argc < 5:
            return
        channel = opt
        nick = text[2]
        mask = text[3]
        reasonidx = 4
    reason = ' '.join(text[reasonidx:])
    mask = configureHostMask(mask)
    if mask == '':
        return
    bot.write(['MODE', channel, '+b', mask])
    bot.write(['KICK', channel, nick], reason)

@require_privilege(OP)
@commands('topic')
def topic(bot, trigger):
    """
    This gives ops the ability to change the topic.
    The bot must be a Channel Operator for this command to work.
    """
    purple, green, bold = '\x0306', '\x0310', '\x02'
    if bot.privileges[trigger.sender][bot.nick] < HALFOP:
        return bot.reply("I'm not a channel operator!")
    if not trigger.group(2):
        return
    channel = trigger.sender.lower()

    narg = 1
    mask = None
    mask = bot.db.get_channel_value(channel, 'topic_mask')
    mask = mask or default_mask(trigger)
    mask = mask.replace('%s', '{}')
    narg = len(re.findall('{}', mask))

    top = trigger.group(2)
    args = []
    if top:
        args = top.split('~', narg)

    if len(args) != narg:
        message = "Not enough arguments. You gave {}, it requires {}.".format(
            len(args), narg)
        return bot.say(message)
    topic = mask.format(*args)

    bot.write(('TOPIC', channel + ' :' + topic))

@require_privilege(OP)
@commands('tmask')
def set_mask(bot, trigger):
    """
    Set the mask to use for .topic in the current channel. {} is used to allow
    substituting in chunks of text.
    """
    bot.db.set_channel_value(trigger.sender, 'topic_mask', trigger.group(2))
    bot.say("Gotcha, " + trigger.nick)

@require_privilege(OP)
@commands('showmask')
def show_mask(bot, trigger):
    """Show the topic mask for the current channel."""
    mask = bot.db.get_channel_value(trigger.sender, 'topic_mask')
    mask = mask or default_mask(trigger)
    bot.say(mask)
<|endoftext|>