<commit_before>import re
from pprint import pprint
import networkx as nx
import matplotlib.pyplot as plt

class Course:
    """Template for course using data from text file"""
    def __init__(self, name, desc, course_id, prereqs, off_w, level):
        self.name = name
        self.course_id = course_id
        self.prereqs = prereqs
        self.off_w = off_w
        self.desc = desc
        self.level = level

#empty list for all Course objects
courses = []

#opening file
with open("raw-data.txt", "r") as f:
    for line in f:
        #finding lines with title of course
        if line[0:3] == "CSE":
            #index of "("-1 is index of space after name
            name = line[8:line.find("(")-1]
            course_id = line[0:7]

            #access line after line containing name; next line has description
            next_line = next(f)

            #finding index of character after "Prerequisite"
            prereq_ind = next_line.find("Prerequisite: ")
            len_prereq = len("Prerequisite: ")

            #finding and assigning description
            desc = next_line[:prereq_ind]

            #assigning level
            level = line[4:5]

            #starting search for prereqs
            prereqs = []
            #checking to see if any prereq's exist
            #no they don't
            if prereq_ind == -1:
                prereqs = None
                #off_w is offered with
                off_w = None
            #yes they do
            else:
                final_part = next_line[prereq_ind+len_prereq:]
                final_part_list = final_part.split("Offered")
                #all of these now must have prereq's
                prereq_raw = final_part_list[0]
                if len(final_part_list) == 2:
                    #only some will have joint classes
                    off_w_raw = final_part_list[1]
                    off_re = re.compile("[A-Z].[A-Z]{1,3}\s\d\d\d")
                    off_w = off_re.findall(off_w_raw)
                else:
                    #if they don't, connect this with correct variable
                    off_w = None

                #splitting into each seperate prereq
                prereq_split = prereq_raw.split(";")
                #iterating through and regexing out excess
                for i in prereq_split:
                    if "or" in i:
                        p = re.compile("[A-Z].[A-Z]{1,3}\s\d\d\d")
                        #append the ENTIRE tuple of options to the prereq list
                        p_tup = tuple(p.findall(i))
                        prereqs.append(p_tup)
                        #the prereq list is tuple-ified at the end of the process
                    else:
                        p = re.compile("[A-Z].[A-Z]{1,3}\s\d\d\d")
                        #append EACH ITEM from the list of options
                        for j in p.findall(i):
                            prereqs.append(j)

                #making prereqs immutable via tuple
                prereqs = tuple(prereqs)

            courses.append(Course(name, desc, course_id, prereqs, off_w, level))

G = nx.Graph()
#for course in courses:
#    print course.course_id + ": " + str(course.prereqs)

#options edges
opt_edge = []

#required edges
req_edge = []

#connecting courses with prereqs
for course in courses:
    #checking for prereqs
    if course.prereqs:
        for prereq in course.prereqs:
            #determining whether there are prereq options (in a tuple)
            if type(prereq) is tuple:
                #looping through the ACTUAL prereq courses inside tuple
                for act_prereq in prereq:
                    #adding edge to graph
                    G.add_edge(act_prereq, course.course_id)
                    #adding edge to list for later formatting
                    opt_edge.append((act_prereq, course.course_id))
            else:
                #adding edge to graph
                G.add_edge(prereq, course.course_id)
                #adding edge to list for later formatting
                req_edge.append((prereq, course.course_id))

pos = nx.spring_layout(G)

#drawing nodes
nx.draw_networkx_nodes(G,pos, node_size=700)

#drawing edges
nx.draw_networkx_edges(G, pos, edgelist=G.edges(data=True), width=1)

#labels
nx.draw_networkx_labels(G, pos, font_size=6, font_family='sans-serif')

plt.axis("off")
plt.show()

<commit_msg>Added NetworkX path attributes<commit_after>import re
from pprint import pprint
import networkx as nx
import matplotlib.pyplot as plt

class Course:
    """Template for course using data from text file"""
    def __init__(self, name, desc, course_id, prereqs, off_w, level):
        self.name = name
        self.course_id = course_id
        self.prereqs = prereqs
        self.off_w = off_w
        self.desc = desc
        self.level = level

#empty list for all Course objects
courses = []

#opening file
with open("raw-data.txt", "r") as f:
    for line in f:
        #finding lines with title of course
        if line[0:3] == "CSE":
            #index of "("-1 is index of space after name
            name = line[8:line.find("(")-1]
            course_id = line[0:7]

            #access line after line containing name; next line has description
            next_line = next(f)

            #finding index of character after "Prerequisite"
            prereq_ind = next_line.find("Prerequisite: ")
            len_prereq = len("Prerequisite: ")

            #finding and assigning description
            desc = next_line[:prereq_ind]

            #assigning level
            level = line[4:5]

            #starting search for prereqs
            prereqs = []
            #checking to see if any prereq's exist
            #no they don't
            if prereq_ind == -1:
                prereqs = None
                #off_w is offered with
                off_w = None
            #yes they do
            else:
                final_part = next_line[prereq_ind+len_prereq:]
                final_part_list = final_part.split("Offered")
                #all of these now must have prereq's
                prereq_raw = final_part_list[0]
                if len(final_part_list) == 2:
                    #only some will have joint classes
                    off_w_raw = final_part_list[1]
                    off_re = re.compile("[A-Z].[A-Z]{1,3}\s\d\d\d")
                    off_w = off_re.findall(off_w_raw)
                else:
                    #if they don't, connect this with correct variable
                    off_w = None

                #splitting into each seperate prereq
                prereq_split = prereq_raw.split(";")
                #iterating through and regexing out excess
                for i in prereq_split:
                    if "or" in i:
                        p = re.compile("[A-Z].[A-Z]{1,3}\s\d\d\d")
                        #append the ENTIRE tuple of options to the prereq list
                        p_tup = tuple(p.findall(i))
                        prereqs.append(p_tup)
                        #the prereq list is tuple-ified at the end of the process
                    else:
                        p = re.compile("[A-Z].[A-Z]{1,3}\s\d\d\d")
                        #append EACH ITEM from the list of options
                        for j in p.findall(i):
                            prereqs.append(j)

                #making prereqs immutable via tuple
                prereqs = tuple(prereqs)

            courses.append(Course(name, desc, course_id, prereqs, off_w, level))

G = nx.Graph()
#for course in courses:
#    print course.course_id + ": " + str(course.prereqs)

#options edges
opt_edge = []

#required edges
req_edge = []

#connecting courses with prereqs
for course in courses:
    #checking for prereqs
    if course.prereqs:
        for prereq in course.prereqs:
            #determining whether there are prereq options (in a tuple)
            if type(prereq) is tuple:
                #looping through the ACTUAL prereq courses inside tuple
                for act_prereq in prereq:
                    #adding edge to graph
                    G.add_edge(act_prereq, course.course_id)
                    #adding edge to list for later formatting
                    opt_edge.append((act_prereq, course.course_id))
            else:
                #adding edge to graph
                G.add_edge(prereq, course.course_id)
                #adding edge to list for later formatting
                req_edge.append((prereq, course.course_id))

pos = nx.spring_layout(G)

#drawing nodes
nx.draw_networkx_nodes(G,pos, node_size=700)

#drawing required edges
nx.draw_networkx_edges(G, pos, edgelist=req_edge, width=1)
#drawing edges connecting prereqs with options
nx.draw_networkx_edges(G, pos, edgelist=opt_edge, width=0.6,
                        edge_color="b", style="dashed")
#labels
nx.draw_networkx_labels(G, pos, font_size=6, font_family='sans-serif')

plt.axis("off")
plt.show()

<|endoftext|>


<commit_before>#!/usr/bin/python3
import sys, time, os, re, string, configparser, logging, psycopg2
from collections import defaultdict, namedtuple
from daemon import Daemon

class IceKing(Daemon):

	format = re.compile(
		r"(?P<host>[\d\.]+)\s"
		r"(?P<identity>\S*)\s"
		r"(?P<user>\S*)\s"
		r"\[(?P<time>.*?)\]\s"
		r'"(?P<request>.*?)"\s'
		r"(?P<status>\d+)\s"
		r"(?P<bytes>\S*)\s"
		r'"(?P<referer>.*?)"\s'
		r'"(?P<user_agent>.*?)"\s*'
	)

	LogItem = namedtuple('LogItem',
		['host', 'identity', 'user', 'time', 'request',
		'status', 'bytes', 'referer', 'user_agent'])

	timeout = 0.5
	logger = None

	def parse(self, line):
		match = self.format.match(line)
		if match:
			return self.LogItem( **match.groupdict() )

	def connect(self, host, port, db, user, pw):
		conn = psycopg2.connect(host=host, port=port, database=db, user=user, password=pw)
		return conn

	def get_mount_point(self, mount_point):
		url = mount_point.split(' ')[1][1:]
		self.cur.execute('SELECT id FROM mount_points WHERE (mount_point = %s) AND ((date_added <= now() AND date_retired >= now()) OR (date_added <= now() AND date_retired is NULL))', [url])
		if self.cur.rowcount == 0:
			return None
		else:
			id = self.cur.fetchone()[0]
			return id

	def get_or_add_ua(self, ua):
		self.cur.execute('SELECT id FROM user_agents WHERE ua_string = %s', [ua])
		if self.cur.rowcount == 0:
			self.cur.execute('INSERT INTO user_agents (ua_string) VALUES (%s) RETURNING id', [ua])
			id = self.cur.fetchone()[0]
			if id:
				self.db.commit()
				return id
			else:
				logger.error('Cannot add user-agent: %s', ua)
		else:
			id = self.cur.fetchone()[0]
			return id

	def get_or_add_ref(self, url):
		self.cur.execute('SELECT id FROM referers WHERE url = %s', [url])
		if self.cur.rowcount == 0:
			self.cur.execute('INSERT INTO referers (url) VALUES (%s) RETURNING id', [url])
			id = self.cur.fetchone()[0]
			if id:
				self.db.commit()
				return id
			else:
				logger.error('cannot add referer: %s', url)
		else:
			id = self.cur.fetchone()[0]
			return id

	def log(self, ip, mount, ref, ua, bytes):
		self.cur.execute('INSERT INTO log (ip, mount_point_id, referer_id, user_agent_id, bytes) VALUES (%s, %s, %s, %s, %s) RETURNING id', [ip, mount, ref, ua, bytes])
		id = self.cur.fetchone()[0]
		if id:
			self.db.commit()
			return id
		else:
			logger.error('Cannot log row: %s %s %s %d', mount, ref, ua, bytes)

	def process_line(self, line):
		line = line.rstrip('\n')
		self.logger.debug(line)
		data = self.parse(line)
		mount = self.get_mount_point(data.request)
		if mount is not None:
			ua = self.get_or_add_ua(data.user_agent)
			ref = self.get_or_add_ref(data.referer)
			id = self.log(data.host, mount, ref, ua, data.bytes)

	def open_file(self, filename):
		file = open(filename, 'r')
		return file

	def read_loop(self, file):
		if self.process_all:
			self.logger.info('Logging entire contents of %s to database.', file.name)
			for line in file:
				self.process_line(line)

		if self.watch:
			self.logger.info('Watching %s for new lines to log to database...', file.name)
			st_results = os.stat(file.name)
			st_size = st_results[6]

			self.logger.debug('%s size is %d bytes, moving to end and watching.', file.name, st_size)
			file.seek(st_size)

			while 1:
				where = file.tell()
				line = file.readline()
				if not line:
					time.sleep(self.timeout)
					file.seek(where)
				else:
					self.process_line(line)

	def run(self):
		config = configparser.SafeConfigParser()
		config.read('config.ini')

		logfile = config.get('main','log')

		if not logfile:
			import os.getcwd
			logfile = '%s/iceking.log' % os.getcwd()

		loglevel = config.get('main','log_level')

		if loglevel == 'critical':
			loglevel = logging.CRITICAL
		elif loglevel == 'error':
			loglevel = logging.ERROR
		elif loglevel == 'warning':
			loglevel = logging.WARNING
		elif loglevel == 'info':
			loglevel = logging.INFO
		elif loglevel == 'debug':
			loglevel = logging.DEBUG
		else:
			loglevel = logging.WARNING

		logging.basicConfig(format='%(asctime)s [%(levelname)s]: %(message)s', filename=logfile, level=logging.INFO)
		self.logger = logging.getLogger('root')

		self.logger.info('IceKing starting...')
		self.logger.setLevel(loglevel)

		if sys.argv[1] == 'no-daemon':
			log_con = logging.StreamHandler()
			log_con.setLevel(loglevel)
			log_con.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s]: %(message)s'))
			self.logger.addHandler(log_con)

		self.db = self.connect(
			config.get('database', 'host'),
			config.get('database','port'),
			config.get('database','db_name'),
			config.get('database','user'),
			config.get('database', 'password')
		)

		self.cur = self.db.cursor()
		file = self.open_file(config.get('main','filename'))
		self.timeout = config.getfloat('main','timeout')
		self.process_all = config.getboolean('main', 'process_all')
		self.watch = config.getboolean('main','watch')
		self.read_loop(file)

	def stop(self):
		self.logger.setLevel(logging.INFO)
		self.logger.info('Shutting down.')
		super(Daemon, self).stop()

	def restart(self):
		self.logger.setLevel(logging.INFO)
		self.logger.info('Restarting...')
		super(Daemon, self).restart()

if __name__ == "__main__":
	daemon = IceKing('/var/run/iceking.pid')

	if len(sys.argv) == 2:
			if 'start' == sys.argv[1]:
				daemon.start()
			elif 'stop' == sys.argv[1]:
				daemon.stop()
			elif 'restart' == sys.argv[1]:
				daemon.restart()
			elif 'no-daemon':
				daemon.run()
			else:
				print("Unknown command")
				sys.exit(2)
			sys.exit(0)
	else:
		print("usage: %s start|stop|restart|no-daemon" % sys.argv[0])
		sys.exit(2)
<commit_msg>use timestamp from log instead of when we log it<commit_after>#!/usr/bin/python3
import sys, time, os, re, string, configparser, logging, psycopg2
from collections import defaultdict, namedtuple
from daemon import Daemon

class IceKing(Daemon):

	format = re.compile(
		r"(?P<host>[\d\.]+)\s"
		r"(?P<identity>\S*)\s"
		r"(?P<user>\S*)\s"
		r"\[(?P<time>.*?)\]\s"
		r'"(?P<request>.*?)"\s'
		r"(?P<status>\d+)\s"
		r"(?P<bytes>\S*)\s"
		r'"(?P<referer>.*?)"\s'
		r'"(?P<user_agent>.*?)"\s*'
	)

	LogItem = namedtuple('LogItem',
		['host', 'identity', 'user', 'time', 'request',
		'status', 'bytes', 'referer', 'user_agent'])

	timeout = 0.5
	logger = None

	def parse(self, line):
		match = self.format.match(line)
		if match:
			return self.LogItem( **match.groupdict() )

	def connect(self, host, port, db, user, pw):
		conn = psycopg2.connect(host=host, port=port, database=db, user=user, password=pw)
		return conn

	def get_mount_point(self, mount_point):
		url = mount_point.split(' ')[1][1:]
		self.cur.execute('SELECT id FROM mount_points WHERE (mount_point = %s) AND ((date_added <= now() AND date_retired >= now()) OR (date_added <= now() AND date_retired is NULL))', [url])
		if self.cur.rowcount == 0:
			return None
		else:
			id = self.cur.fetchone()[0]
			return id

	def get_or_add_ua(self, ua):
		self.cur.execute('SELECT id FROM user_agents WHERE ua_string = %s', [ua])
		if self.cur.rowcount == 0:
			self.cur.execute('INSERT INTO user_agents (ua_string) VALUES (%s) RETURNING id', [ua])
			id = self.cur.fetchone()[0]
			if id:
				self.db.commit()
				return id
			else:
				logger.error('Cannot add user-agent: %s', ua)
		else:
			id = self.cur.fetchone()[0]
			return id

	def get_or_add_ref(self, url):
		self.cur.execute('SELECT id FROM referers WHERE url = %s', [url])
		if self.cur.rowcount == 0:
			self.cur.execute('INSERT INTO referers (url) VALUES (%s) RETURNING id', [url])
			id = self.cur.fetchone()[0]
			if id:
				self.db.commit()
				return id
			else:
				logger.error('cannot add referer: %s', url)
		else:
			id = self.cur.fetchone()[0]
			return id

	def log(self, ip, datetime, mount, ref, ua, bytes):
		self.cur.execute('INSERT INTO log (ip, timestamp, mount_point_id, referer_id, user_agent_id, bytes) VALUES (%s, %s::timestamptz, %s, %s, %s, %s) RETURNING id', [ip, datetime, mount, ref, ua, bytes])
		id = self.cur.fetchone()[0]
		if id:
			self.db.commit()
			return id
		else:
			logger.error('Cannot log row: %s %s %s %d', mount, ref, ua, bytes)

	def process_line(self, line):
		line = line.rstrip('\n')
		self.logger.debug(line)
		data = self.parse(line)
		mount = self.get_mount_point(data.request)
		if mount is not None:
			ua = self.get_or_add_ua(data.user_agent)
			ref = self.get_or_add_ref(data.referer)
			id = self.log(data.host, data.time, mount, ref, ua, data.bytes)

	def open_file(self, filename):
		file = open(filename, 'r')
		return file

	def read_loop(self, file):
		if self.process_all:
			self.logger.info('Logging entire contents of %s to database.', file.name)
			for line in file:
				self.process_line(line)

		if self.watch:
			self.logger.info('Watching %s for new lines to log to database...', file.name)
			st_results = os.stat(file.name)
			st_size = st_results[6]

			self.logger.debug('%s size is %d bytes, moving to end and watching.', file.name, st_size)
			file.seek(st_size)

			while 1:
				where = file.tell()
				line = file.readline()
				if not line:
					time.sleep(self.timeout)
					file.seek(where)
				else:
					self.process_line(line)

	def run(self):
		config = configparser.SafeConfigParser()
		config.read('config.ini')

		logfile = config.get('main','log')

		if not logfile:
			import os.getcwd
			logfile = '%s/iceking.log' % os.getcwd()

		loglevel = config.get('main','log_level')

		if loglevel == 'critical':
			loglevel = logging.CRITICAL
		elif loglevel == 'error':
			loglevel = logging.ERROR
		elif loglevel == 'warning':
			loglevel = logging.WARNING
		elif loglevel == 'info':
			loglevel = logging.INFO
		elif loglevel == 'debug':
			loglevel = logging.DEBUG
		else:
			loglevel = logging.WARNING

		logging.basicConfig(format='%(asctime)s [%(levelname)s]: %(message)s', filename=logfile, level=logging.INFO)
		self.logger = logging.getLogger('root')

		self.logger.info('IceKing starting...')
		self.logger.setLevel(loglevel)

		if sys.argv[1] == 'no-daemon':
			log_con = logging.StreamHandler()
			log_con.setLevel(loglevel)
			log_con.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s]: %(message)s'))
			self.logger.addHandler(log_con)

		self.db = self.connect(
			config.get('database', 'host'),
			config.get('database','port'),
			config.get('database','db_name'),
			config.get('database','user'),
			config.get('database', 'password')
		)

		self.cur = self.db.cursor()
		file = self.open_file(config.get('main','filename'))
		self.timeout = config.getfloat('main','timeout')
		self.process_all = config.getboolean('main', 'process_all')
		self.watch = config.getboolean('main','watch')
		self.read_loop(file)

	def stop(self):
		self.logger.setLevel(logging.INFO)
		self.logger.info('Shutting down.')
		super(Daemon, self).stop()

	def restart(self):
		self.logger.setLevel(logging.INFO)
		self.logger.info('Restarting...')
		super(Daemon, self).restart()

if __name__ == "__main__":
	daemon = IceKing('/var/run/iceking.pid')

	if len(sys.argv) == 2:
			if 'start' == sys.argv[1]:
				daemon.start()
			elif 'stop' == sys.argv[1]:
				daemon.stop()
			elif 'restart' == sys.argv[1]:
				daemon.restart()
			elif 'no-daemon':
				daemon.run()
			else:
				print("Unknown command")
				sys.exit(2)
			sys.exit(0)
	else:
		print("usage: %s start|stop|restart|no-daemon" % sys.argv[0])
		sys.exit(2)
<|endoftext|>


<commit_before>import os, shutil, string, random
import tempfile, urllib.request, json, urllib.parse, base64
from flask import Flask, render_template, session, request, redirect, url_for
from reverseproxied import ReverseProxied

application = app = Flask(__name__)
app.wsgi_app = ReverseProxied(app.wsgi_app)

# change this, of course!
app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT'
app.debug = True

ENV_GITHUBCLIENTID = 'githubclientid'
ENV_GITHUBSECRET = 'githubsecret'
ENV_DEVTOKEN = 'githubtoken'

SESSION_TEMPDIR = 'tempdir'
SESSION_GITHUBSTATE = 'githubstate'
SESSION_ACCESSTOKEN = 'accesstoken'
SESSION_URLINPUT = 'urlinput'
SESSION_EDITCONTENT = 'editcontent'
SESSION_MAINFILE = 'mainfile'
SESSION_MAINSHA = 'mainsha'

RUN_EDIT = 'run_edit'
AUTH_REQUEST = 'auth_request'
AUTH_FORGET = 'auth_forget'
GITHUB_COMMIT = 'github_commit'

URL_GITHUBAUTHORIZE = 'https://github.com/login/oauth/authorize'
URL_GITHUBRETRIEVETOKEN = 'https://github.com/login/oauth/access_token'

github_client_id = os.environ[ENV_GITHUBCLIENTID]
github_client_secret = os.environ[ENV_GITHUBSECRET]

def newtempdir():
    if SESSION_TEMPDIR in session:
        try:
            shutil.rmtree(tempdir())
        except:
            pass
    session[SESSION_TEMPDIR] = tempfile.mkdtemp()

def tempdir():
    return session[SESSION_TEMPDIR]

# newgithubstate
# Set a random string as local session state for github, if not
# already set
def newgithubstate():
    state = session.get(SESSION_GITHUBSTATE, '')
    if SESSION_GITHUBSTATE not in session:
        N = 20
        state = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(N))
        session[SESSION_GITHUBSTATE] = state
    return state

# checkgithubstate
# return True if state matches, otherwise False
def checkgithubstate(state):
    return state == session.get(SESSION_GITHUBSTATE, '')

# getredirecturl
# Return URL to redirect to from github
# Needs some scrubbing to put in acceptible format
def getredirecturl():
    url = url_for('root', _external=True)
    if url.endswith(":80"):
        url = url[:-3]
    return url

# githubauthurl
# generate a redirect URL to github
def githubauthurl():
    #print(url_for('root', _external=True))
    url = URL_GITHUBAUTHORIZE + "?"
    url += "client_id="+github_client_id
    url += "&redirect_uri="+getredirecturl()
    url += "&scope=repo&state="+newgithubstate()
    print(url)
    return url

# githubretrievetoken
# retrieve an access token for the user via URL_GITHUBRETRIEVETOKEN
def githubretrievetoken(code):
    gitrequest = urllib.request.Request(URL_GITHUBRETRIEVETOKEN)
    gitrequest.add_header('Accept', 'application/json')
    parameters = {'client_id':github_client_id,
        'client_secret':github_client_secret,
        'code':code,
        'redirect_uri':getredirecturl()}
    data = urllib.parse.urlencode(parameters)
    data = data.encode('utf-8')
    response = urllib.request.urlopen(gitrequest, data)
    jsresponse = json.loads(response.read().decode("utf-8"))
    if 'access_token' in jsresponse:
        session[SESSION_ACCESSTOKEN] = jsresponse.get('access_token')

# githubforgetauth
# forget the current github authorization
def githubforgetauth():
    session[SESSION_ACCESSTOKEN] = ""

# Root path
# If GET with repo data, use the exec.html template
# If GET with nothing, plain old index.html
# Otherwise, if POST, index.html with pre-populated github path
@app.route('/', methods=['POST', 'GET'])
def root():
    github_token = session.get(SESSION_ACCESSTOKEN)
    github_loggedin = True if github_token else False
    if request.method == 'GET':
        if 'user' in request.args:
            user = request.args.get('user')
            repo = request.args.get('repo')
            name = request.args.get('name')
            path = request.args.get('path','')
            return render_template('exec.html',
                user=user,
                repo=repo,
                name=name,
                path=path)
        elif 'code' in request.args and 'state' in request.args:
            # Github authorization response - check if valid
            if checkgithubstate(request.args.get('state')):
                githubretrievetoken(request.args.get('code'))
            return redirect(url_for('root'))
        else:
            return render_template('index.html',
                edit=session.get(SESSION_URLINPUT,''),
                editcontent = session.get(SESSION_EDITCONTENT,'print("Hello, world.")'),
                github = github_loggedin)
    elif request.method == 'POST':
        if RUN_EDIT in request.form:
            # user is requesting to open a new page with editor
            return render_template('index.html',
                edit=request.form[RUN_EDIT],
                editcontent = '',
                github = github_loggedin)
        elif AUTH_REQUEST in request.form:
            # user is requesting authorization from github
            return redirect(githubauthurl())
        elif AUTH_FORGET in request.form:
            # user is requesting to forget our authorization
            githubforgetauth()
            return redirect(url_for('root'))

@app.route('/favicon.ico')
def favicon():
    return app.send_static_file('favicon.ico')

@app.route('/_debug')
def debug():
    return render_template('test.html')

# Default handler for any single file name: these are pulled
# from the Python source temporary cache folder
@app.route('/<filename>')
def file(filename):
    with open(os.path.join(tempdir(),filename)) as f:
        return f.read()

# Build a valid github URL out of user, repo, path and name
# Perhaps there is a github API for doing this that will be more
# stable?
def githubpath(user, repo, path, name):
    retval = "https://github.com/{0}/{1}/blob/master/".format(user,repo)
    if path:
        retval += path
        if name not in path:
            retval +=  "/" + name
    else:
        retval += name
    return retval

## /api/v1/update
##
## Inform server of edits/updates to the editor content and url_input
## Server will cache the content in session data and recover it
## when the page is reloaded.
@app.route('/api/v1/update', methods=['PUT'])
def v1_update():
    content = request.json
    session[SESSION_EDITCONTENT] = content.get('editcontent','')
    session[SESSION_URLINPUT] = content.get('url_input','')
    return json.dumps({'success':True}, 200, {'ContentType':'application/json'})

## /api/v1/commit
##
## Server will issue a commit to github
@app.route('/api/v1/commit', methods=['PUT'])
def v1_commit():
    content = request.json
    content = request.json
    user = content.get('user')
    repo = content.get('repo')
    path = content.get('path','')
    name = content.get('name','')
    editcontent = content.get('editcontent','')
    msg = content.get('commitmsg','')
    if path and not path.endswith(name):
        path += "/" + name
    else:
        path += name
    url = "https://api.github.com/repos/{0}/{1}/contents/{2}".format(user, repo, path)
    token = session.get(SESSION_ACCESSTOKEN)
    # formulate the call to github
    gitrequest = urllib.request.Request(url, method='PUT')
    gitrequest.add_header('Content-Type', 'application/json; charset=utf-8')
    gitrequest.add_header('Accept', 'application/json')
    gitrequest.add_header('Authorization', 'token {0}'.format(token))
    parameters = {'message':msg,
        'content':base64.b64encode(editcontent.encode('utf-8')).decode('utf-8'),
        'sha':session[SESSION_MAINSHA]}
    data = json.dumps(parameters).encode('utf-8')
    gitrequest.add_header('Content-Length', len(data))
    try:
        response = urllib.request.urlopen(gitrequest, data)
        jsresponse = json.loads(response.read().decode("utf-8"))
        return json.dumps({'success':True}, 200, {'ContentType':'application/json'})
    except urllib.error.HTTPError as err:
        print("Github error: " + err.msg + " " + str(err.code) + ", token was ", token)
        return json.dumps({'success':False, 'message':err.msg}), 200, {'ContentType':'application/json'}

## /api/v1/load
##
## Retrieve specified user/repo/path from Github
## and cache the file and any other files in its directory
## Return the primary file name and content
## If given a path or root folder, will attempt to identify a single
## python file as the main file to execute.
## Input/json: user, repo, path, name (of file)
## Output/json: full github URL (path), file (name), content
##
## Function is too big..
@app.route('/api/v1/load', methods=['PUT'])
def v1_load():
    content = request.json
    user = content.get('user')
    repo = content.get('repo')
    path = content.get('path','')
    name = content.get('name','')
    mainfile = ""
    mainsha = ""
    newtempdir()
    url = "https://api.github.com/repos/{0}/{1}/contents/{2}".format(user, repo, path)
    token = session.get(SESSION_ACCESSTOKEN, os.environ.get(ENV_DEVTOKEN))
    # token = os.environ[ENV_DEVTOKEN]
    urllib.request.urlcleanup()
    gitrequest = urllib.request.Request(url)
    gitrequest.add_header('Authorization', 'token {0}'.format(token))
    try:
        maincontent = ""
        response = urllib.request.urlopen(gitrequest)
        jsresponse = json.loads(response.read().decode("utf-8"))
        if 'size' in jsresponse:        # if response is a single file
            jsresponse = [jsresponse]   # make a list of it
        for f in jsresponse:
            if f['type'] == 'file':
                ismain = False
                foundname = f['name']
                # First python file found? Make it THE ONE!
                if (mainfile == "" and len(foundname) > 3 and foundname[-3:] == '.py' or
                    # Foud a python file called main.py or __main__.py? Make IT the one!
                    foundname in ["main.py", "__main__.py"] or
                # Found a python file that the user actually wanted? Make IT THE ONE!
                    foundname == name):
                    mainfile = foundname
                    mainsha = f['sha']
                    ismain = True
                fileurl = f['download_url']
                # Read each file in the directory, regardless...
                gitrequest = urllib.request.Request(fileurl)
                gitrequest.add_header('Authorization', 'token {0}'.format(token))
                gitrequest.add_header('Pragma', 'no-cache')
                rfile = urllib.request.urlopen(gitrequest)
                with open(os.path.join(tempdir(), foundname), 'w') as f:
                    temp = rfile.read().decode("utf-8")
                    f.write(temp)
                    # If this is THE file, hold on to the content
                    if ismain:
                        maincontent = temp
        # All files read, save primary name and sha
        session[SESSION_MAINFILE] = mainfile
        session[SESSION_MAINSHA] = mainsha
        # All files read, return
        return json.dumps({'success':True,
                    'name':mainfile,
                    'path':githubpath(user,repo,path,mainfile),
                    'content':maincontent}), 200, {'ContentType':'application/json'}
    except urllib.error.HTTPError as err:
        print("Github error: " + err.msg + ", token was ", token)
        return json.dumps({'success':False, 'message':err.msg}), 200, {'ContentType':'application/json'}

## Execute specific script from github
## Input user, repo, filename, path fragment (not including file name)
@app.route('/_exec/<user>/<repo>/<name>/<path:path>/')
@app.route('/_exec/<user>/<repo>/<name>/')
def exec_github(user, repo, name, path=""):
    return render_template('exec.html', user=user, repo=repo, name=name, path=path)

if __name__ == "__main__":
    app.run(host=os.environ['IP'],port=int(os.environ['PORT']))

<commit_msg>support for robust flask key<commit_after>import os, shutil, string, random
import tempfile, urllib.request, json, urllib.parse, base64
from flask import Flask, render_template, session, request, redirect, url_for
from reverseproxied import ReverseProxied

application = app = Flask(__name__)
app.wsgi_app = ReverseProxied(app.wsgi_app)

ENV_GITHUBCLIENTID = 'githubclientid'
ENV_GITHUBSECRET = 'githubsecret'
ENV_DEVTOKEN = 'githubtoken'
ENV_FLASKSECRET = 'flasksecret'

SESSION_TEMPDIR = 'tempdir'
SESSION_GITHUBSTATE = 'githubstate'
SESSION_ACCESSTOKEN = 'accesstoken'
SESSION_URLINPUT = 'urlinput'
SESSION_EDITCONTENT = 'editcontent'
SESSION_MAINFILE = 'mainfile'
SESSION_MAINSHA = 'mainsha'

RUN_EDIT = 'run_edit'
AUTH_REQUEST = 'auth_request'
AUTH_FORGET = 'auth_forget'
GITHUB_COMMIT = 'github_commit'

URL_GITHUBAUTHORIZE = 'https://github.com/login/oauth/authorize'
URL_GITHUBRETRIEVETOKEN = 'https://github.com/login/oauth/access_token'

github_client_id = os.environ[ENV_GITHUBCLIENTID]
github_client_secret = os.environ[ENV_GITHUBSECRET]
app.secret_key = os.environ.get(ENV_FLASKSECRET,'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT')
app.debug = True

def newtempdir():
    if SESSION_TEMPDIR in session:
        try:
            shutil.rmtree(tempdir())
        except:
            pass
    session[SESSION_TEMPDIR] = tempfile.mkdtemp()

def tempdir():
    return session[SESSION_TEMPDIR]

# newgithubstate
# Set a random string as local session state for github, if not
# already set
def newgithubstate():
    state = session.get(SESSION_GITHUBSTATE, '')
    if SESSION_GITHUBSTATE not in session:
        N = 20
        state = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(N))
        session[SESSION_GITHUBSTATE] = state
    return state

# checkgithubstate
# return True if state matches, otherwise False
def checkgithubstate(state):
    return state == session.get(SESSION_GITHUBSTATE, '')

# getredirecturl
# Return URL to redirect to from github
# Needs some scrubbing to put in acceptible format
def getredirecturl():
    url = url_for('root', _external=True)
    if url.endswith(":80"):
        url = url[:-3]
    return url

# githubauthurl
# generate a redirect URL to github
def githubauthurl():
    #print(url_for('root', _external=True))
    url = URL_GITHUBAUTHORIZE + "?"
    url += "client_id="+github_client_id
    url += "&redirect_uri="+getredirecturl()
    url += "&scope=repo&state="+newgithubstate()
    print(url)
    return url

# githubretrievetoken
# retrieve an access token for the user via URL_GITHUBRETRIEVETOKEN
def githubretrievetoken(code):
    gitrequest = urllib.request.Request(URL_GITHUBRETRIEVETOKEN)
    gitrequest.add_header('Accept', 'application/json')
    parameters = {'client_id':github_client_id,
        'client_secret':github_client_secret,
        'code':code,
        'redirect_uri':getredirecturl()}
    data = urllib.parse.urlencode(parameters)
    data = data.encode('utf-8')
    response = urllib.request.urlopen(gitrequest, data)
    jsresponse = json.loads(response.read().decode("utf-8"))
    if 'access_token' in jsresponse:
        session[SESSION_ACCESSTOKEN] = jsresponse.get('access_token')

# githubforgetauth
# forget the current github authorization
def githubforgetauth():
    session[SESSION_ACCESSTOKEN] = ""

# Root path
# If GET with repo data, use the exec.html template
# If GET with nothing, plain old index.html
# Otherwise, if POST, index.html with pre-populated github path
@app.route('/', methods=['POST', 'GET'])
def root():
    github_token = session.get(SESSION_ACCESSTOKEN)
    github_loggedin = True if github_token else False
    if request.method == 'GET':
        if 'user' in request.args:
            user = request.args.get('user')
            repo = request.args.get('repo')
            name = request.args.get('name')
            path = request.args.get('path','')
            return render_template('exec.html',
                user=user,
                repo=repo,
                name=name,
                path=path)
        elif 'code' in request.args and 'state' in request.args:
            # Github authorization response - check if valid
            if checkgithubstate(request.args.get('state')):
                githubretrievetoken(request.args.get('code'))
            return redirect(url_for('root'))
        else:
            return render_template('index.html',
                edit=session.get(SESSION_URLINPUT,''),
                editcontent = session.get(SESSION_EDITCONTENT,'print("Hello, world.")'),
                github = github_loggedin)
    elif request.method == 'POST':
        if RUN_EDIT in request.form:
            # user is requesting to open a new page with editor
            return render_template('index.html',
                edit=request.form[RUN_EDIT],
                editcontent = '',
                github = github_loggedin)
        elif AUTH_REQUEST in request.form:
            # user is requesting authorization from github
            return redirect(githubauthurl())
        elif AUTH_FORGET in request.form:
            # user is requesting to forget our authorization
            githubforgetauth()
            return redirect(url_for('root'))

@app.route('/favicon.ico')
def favicon():
    return app.send_static_file('favicon.ico')

@app.route('/_debug')
def debug():
    return render_template('test.html')

# Default handler for any single file name: these are pulled
# from the Python source temporary cache folder
@app.route('/<filename>')
def file(filename):
    with open(os.path.join(tempdir(),filename)) as f:
        return f.read()

# Build a valid github URL out of user, repo, path and name
# Perhaps there is a github API for doing this that will be more
# stable?
def githubpath(user, repo, path, name):
    retval = "https://github.com/{0}/{1}/blob/master/".format(user,repo)
    if path:
        retval += path
        if name not in path:
            retval +=  "/" + name
    else:
        retval += name
    return retval

## /api/v1/update
##
## Inform server of edits/updates to the editor content and url_input
## Server will cache the content in session data and recover it
## when the page is reloaded.
@app.route('/api/v1/update', methods=['PUT'])
def v1_update():
    content = request.json
    session[SESSION_EDITCONTENT] = content.get('editcontent','')
    session[SESSION_URLINPUT] = content.get('url_input','')
    return json.dumps({'success':True}, 200, {'ContentType':'application/json'})

## /api/v1/commit
##
## Server will issue a commit to github
@app.route('/api/v1/commit', methods=['PUT'])
def v1_commit():
    content = request.json
    content = request.json
    user = content.get('user')
    repo = content.get('repo')
    path = content.get('path','')
    name = content.get('name','')
    editcontent = content.get('editcontent','')
    msg = content.get('commitmsg','')
    if path and not path.endswith(name):
        path += "/" + name
    else:
        path += name
    url = "https://api.github.com/repos/{0}/{1}/contents/{2}".format(user, repo, path)
    token = session.get(SESSION_ACCESSTOKEN)
    # formulate the call to github
    gitrequest = urllib.request.Request(url, method='PUT')
    gitrequest.add_header('Content-Type', 'application/json; charset=utf-8')
    gitrequest.add_header('Accept', 'application/json')
    gitrequest.add_header('Authorization', 'token {0}'.format(token))
    parameters = {'message':msg,
        'content':base64.b64encode(editcontent.encode('utf-8')).decode('utf-8'),
        'sha':session[SESSION_MAINSHA]}
    data = json.dumps(parameters).encode('utf-8')
    gitrequest.add_header('Content-Length', len(data))
    try:
        response = urllib.request.urlopen(gitrequest, data)
        jsresponse = json.loads(response.read().decode("utf-8"))
        return json.dumps({'success':True}, 200, {'ContentType':'application/json'})
    except urllib.error.HTTPError as err:
        print("Github error: " + err.msg + " " + str(err.code) + ", token was ", token)
        return json.dumps({'success':False, 'message':err.msg}), 200, {'ContentType':'application/json'}

## /api/v1/load
##
## Retrieve specified user/repo/path from Github
## and cache the file and any other files in its directory
## Return the primary file name and content
## If given a path or root folder, will attempt to identify a single
## python file as the main file to execute.
## Input/json: user, repo, path, name (of file)
## Output/json: full github URL (path), file (name), content
##
## Function is too big..
@app.route('/api/v1/load', methods=['PUT'])
def v1_load():
    content = request.json
    user = content.get('user')
    repo = content.get('repo')
    path = content.get('path','')
    name = content.get('name','')
    mainfile = ""
    mainsha = ""
    newtempdir()
    url = "https://api.github.com/repos/{0}/{1}/contents/{2}".format(user, repo, path)
    token = session.get(SESSION_ACCESSTOKEN, os.environ.get(ENV_DEVTOKEN))
    # token = os.environ[ENV_DEVTOKEN]
    urllib.request.urlcleanup()
    gitrequest = urllib.request.Request(url)
    gitrequest.add_header('Authorization', 'token {0}'.format(token))
    try:
        maincontent = ""
        response = urllib.request.urlopen(gitrequest)
        jsresponse = json.loads(response.read().decode("utf-8"))
        if 'size' in jsresponse:        # if response is a single file
            jsresponse = [jsresponse]   # make a list of it
        for f in jsresponse:
            if f['type'] == 'file':
                ismain = False
                foundname = f['name']
                # First python file found? Make it THE ONE!
                if (mainfile == "" and len(foundname) > 3 and foundname[-3:] == '.py' or
                    # Foud a python file called main.py or __main__.py? Make IT the one!
                    foundname in ["main.py", "__main__.py"] or
                # Found a python file that the user actually wanted? Make IT THE ONE!
                    foundname == name):
                    mainfile = foundname
                    mainsha = f['sha']
                    ismain = True
                fileurl = f['download_url']
                # Read each file in the directory, regardless...
                gitrequest = urllib.request.Request(fileurl)
                gitrequest.add_header('Authorization', 'token {0}'.format(token))
                gitrequest.add_header('Pragma', 'no-cache')
                rfile = urllib.request.urlopen(gitrequest)
                with open(os.path.join(tempdir(), foundname), 'w') as f:
                    temp = rfile.read().decode("utf-8")
                    f.write(temp)
                    # If this is THE file, hold on to the content
                    if ismain:
                        maincontent = temp
        # All files read, save primary name and sha
        session[SESSION_MAINFILE] = mainfile
        session[SESSION_MAINSHA] = mainsha
        # All files read, return
        return json.dumps({'success':True,
                    'name':mainfile,
                    'path':githubpath(user,repo,path,mainfile),
                    'content':maincontent}), 200, {'ContentType':'application/json'}
    except urllib.error.HTTPError as err:
        print("Github error: " + err.msg + ", token was ", token)
        return json.dumps({'success':False, 'message':err.msg}), 200, {'ContentType':'application/json'}

## Execute specific script from github
## Input user, repo, filename, path fragment (not including file name)
@app.route('/_exec/<user>/<repo>/<name>/<path:path>/')
@app.route('/_exec/<user>/<repo>/<name>/')
def exec_github(user, repo, name, path=""):
    return render_template('exec.html', user=user, repo=repo, name=name, path=path)

if __name__ == "__main__":
    app.run(host=os.environ['IP'],port=int(os.environ['PORT']))

<|endoftext|>


<commit_before>__plugin_name__      = "RedstonerUtils"
__plugin_version__   = "3.0"
__plugin_mainclass__ = "foobar"

import sys
from traceback import format_exc as print_traceback

# damn pythonloader changed the PATH
sys.path += ['', '/usr/lib/python2.7', '/usr/lib/python2.7/plat-linux2', '/usr/lib/python2.7/lib-tk', '/usr/lib/python2.7/lib-old', '/usr/lib/python2.7/lib-dynload', '/usr/local/lib/python2.7/dist-packages', '/usr/lib/python2.7/dist-packages', '/usr/lib/pymodules/python2.7', '/usr/lib/pyshared/python2.7']

try:
    # Library that adds a bunch of re-usable methods which are used in nearly all other modules
    from helpers import *
except:
    print("[RedstonerUtils] ERROR: Failed to import helpers:")
    print(print_traceback())

@hook.enable
def on_enable():
    info("RedstonerUtils enabled!")

@hook.disable
def on_disable():
    shared["modules"]["reports"].stop_reporting()
    info("RedstonerUtils disabled!")

info("Loading RedstonerUtils...")

# Import all modules, in this order
shared["load_modules"] = [
    # Collection of tiny utilities
    "misc",
    # Adds chat for staff using /ac <text or ,<text>
    "adminchat",
    # Adds /badge, allows to give players achievements
    "badges",
    # Adds /calc, toggles automatic solving of Math expressions in chat
    "calc",
    # Plugin to locate laggy chunks. /lc <n> lists chunks with more than n entities
    "lagchunks",
    # Adds /report and /rp, Stores reports with time and location
    "reports",
    # Adds group-chat with /chatgroup and /cgt to toggle normal chat into group mode
    "chatgroups",
    # Adds /token, reads and writes from the database to generate pronouncable (and thus memorable) registration-tokens for the website
    "webtoken",
    # Adds /lol, broadcasts random funyy messages. A bit like the splash text in the menu
    "saylol",
    # Shows the owner of a skull when right-clicked
    "skullclick",
    # Adds /listen, highlights chat and plays a sound when your name was mentioned
    "mentio",
    # Adds /cycler, swaps the hotbar with inventory when player changes slot from right->left or left->right
    "cycle",
    # Adds /getmotd & /setmotd to update the motd on the fly (no reboot)
    "motd",
    # AnswerBot. Hides stupid questions from chat and tells the sender about /faq or the like
    "abot",
    # Adds '/forcefield', creates forcefield for players who want it.
    "forcefield",
    # Adds /damnspam, creates timeout for buttons/levers to mitigate button spam.
    "damnspam",
    # Adds /check, useful to lookup details about a player
    "check",
    # Adds /imout, displays fake leave/join messages
    "imout"
    # NOTICE: If you add something here, please add a small descriptive comment above!
]
shared["modules"] = {}
for module in shared["load_modules"]:
    try:
        shared["modules"][module] = __import__(module)
        info("Module %s loaded." % module)
    except:
        error("Failed to import module %s:" % module)
        error(print_traceback())<commit_msg>Adds AdminNotes<commit_after>__plugin_name__      = "RedstonerUtils"
__plugin_version__   = "3.0"
__plugin_mainclass__ = "foobar"

import sys
from traceback import format_exc as print_traceback

# damn pythonloader changed the PATH
sys.path += ['', '/usr/lib/python2.7', '/usr/lib/python2.7/plat-linux2', '/usr/lib/python2.7/lib-tk', '/usr/lib/python2.7/lib-old', '/usr/lib/python2.7/lib-dynload', '/usr/local/lib/python2.7/dist-packages', '/usr/lib/python2.7/dist-packages', '/usr/lib/pymodules/python2.7', '/usr/lib/pyshared/python2.7']

try:
    # Library that adds a bunch of re-usable methods which are used in nearly all other modules
    from helpers import *
except:
    print("[RedstonerUtils] ERROR: Failed to import helpers:")
    print(print_traceback())

@hook.enable
def on_enable():
    info("RedstonerUtils enabled!")

@hook.disable
def on_disable():
    shared["modules"]["reports"].stop_reporting()
    info("RedstonerUtils disabled!")

info("Loading RedstonerUtils...")

# Import all modules, in this order
shared["load_modules"] = [
    # Collection of tiny utilities
    "misc",
    # Adds chat for staff using /ac <text or ,<text>
    "adminchat",
    # Adds /badge, allows to give players achievements
    "badges",
    # Adds /calc, toggles automatic solving of Math expressions in chat
    "calc",
    # Plugin to locate laggy chunks. /lc <n> lists chunks with more than n entities
    "lagchunks",
    # Adds /report and /rp, Stores reports with time and location
    "reports",
    # Adds group-chat with /chatgroup and /cgt to toggle normal chat into group mode
    "chatgroups",
    # Adds /token, reads and writes from the database to generate pronouncable (and thus memorable) registration-tokens for the website
    "webtoken",
    # Adds /lol, broadcasts random funyy messages. A bit like the splash text in the menu
    "saylol",
    # Shows the owner of a skull when right-clicked
    "skullclick",
    # Adds /listen, highlights chat and plays a sound when your name was mentioned
    "mentio",
    # Adds /cycler, swaps the hotbar with inventory when player changes slot from right->left or left->right
    "cycle",
    # Adds /getmotd & /setmotd to update the motd on the fly (no reboot)
    "motd",
    # AnswerBot. Hides stupid questions from chat and tells the sender about /faq or the like
    "abot",
    # Adds '/forcefield', creates forcefield for players who want it.
    "forcefield",
    # Adds /damnspam, creates timeout for buttons/levers to mitigate button spam.
    "damnspam",
    # Adds /check, useful to lookup details about a player
    "check",
    # Adds /imout, displays fake leave/join messages
    "adminnotes",
    # Adds /an, a command you can use to share thoughts/plans/news
    "imout"
    # NOTICE: If you add something here, please add a small descriptive comment above!
]
shared["modules"] = {}
for module in shared["load_modules"]:
    try:
        shared["modules"][module] = __import__(module)
        info("Module %s loaded." % module)
    except:
        error("Failed to import module %s:" % module)
        error(print_traceback())<|endoftext|>


<commit_before>import gym
import os
import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from baselines.common.vec_env.subproc_vec_env import SubprocVecEnv
from envs import make_env
from model import ActorCritic
from torch.autograd import Variable
import argparse

parser = argparse.ArgumentParser(description='A3C')
parser.add_argument('--lr', type=float, default=7e-4,
                    help='learning rate (default: 7e-4)')
parser.add_argument('--eps', type=float, default=1e-5,
                    help='RMSprop optimizer epsilon (default: 1e-5)')
parser.add_argument('--alpha', type=float, default=0.99,
                    help='RMSprop optimizer apha (default: 0.99)')
parser.add_argument('--gamma', type=float, default=0.99,
                    help='discount factor for rewards (default: 0.99)')
parser.add_argument('--entropy-coef', type=float, default=0.01,
                    help='entropy term coefficient (default: 0.01)')
parser.add_argument('--value-loss-coef', type=float, default=0.5,
                    help='value loss coefficient (default: 0.5)')
parser.add_argument('--max-grad-norm', type=float, default=0.5,
                    help='value loss coefficient (default: 0.5)')
parser.add_argument('--seed', type=int, default=1,
                    help='random seed (default: 1)')
parser.add_argument('--num-processes', type=int, default=16,
                    help='how many training CPU processes to use (default: 16)')
parser.add_argument('--num-steps', type=int, default=5,
                    help='number of forward steps in A2C (default: 5)')
parser.add_argument('--num-stack', type=int, default=4,
                    help='number of frames to stack (default: 4)')
parser.add_argument('--log-interval', type=int, default=10,
                    help='log interval, one log per n updates (default: 10)')
parser.add_argument('--num-frames', type=int, default=10e6,
                    help='number of frames to train (default: 10e6)')
parser.add_argument('--env-name', default='PongNoFrameskip-v4',
                    help='environment to train on (default: PongNoFrameskip-v4)')
parser.add_argument('--log-dir', default='/tmp/gym/',
                    help='directory to save agent logs (default: /tmp/gym)')
parser.add_argument('--no-cuda', action='store_true', default=False,
                    help='disables CUDA training')

args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()

num_updates = int(args.num_frames) // args.num_steps // args.num_processes

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

try:
    os.makedirs(args.log_dir)
except OSError:
    pass

def main():
    os.environ['OMP_NUM_THREADS'] = '1'

    envs = SubprocVecEnv([make_env(args.env_name, args.seed, i, args.log_dir)
                            for i in range(args.num_processes)])

    actor_critic = ActorCritic(envs.observation_space.shape[0] * args.num_stack, envs.action_space)

    if args.cuda:
        actor_critic.cuda()

    optimizer = optim.RMSprop(actor_critic.parameters(), args.lr, eps=args.eps, alpha=args.alpha)
    #optimizer = KFACOptimizer(actor_critic, damping=1e-2, kl_clip=0.01, stat_decay=0.99)

    obs_shape = envs.observation_space.shape
    obs_shape = (obs_shape[0] * args.num_stack, obs_shape[1], obs_shape[2])

    states = torch.zeros(args.num_steps + 1, args.num_processes, *obs_shape)
    current_state = torch.zeros(args.num_processes, *obs_shape)
    counts = 0

    def update_current_state(state):
        state = torch.from_numpy(np.stack(state)).float()
        current_state[:, :-1] = current_state[:, 1:]
        current_state[:, -1] = state

    state = envs.reset()
    update_current_state(state)

    rewards = torch.zeros(args.num_steps, args.num_processes, 1)
    returns = torch.zeros(args.num_steps + 1, args.num_processes, 1)

    actions = torch.LongTensor(args.num_steps, args.num_processes)
    masks = torch.zeros(args.num_steps, args.num_processes, 1)

    # These variables are used to compute average rewards for all processes.
    # Note that rewards are clipped so you need to use a monitor (see envs.py)
    # to get true rewards.
    episode_rewards = torch.zeros([args.num_processes, 1])
    final_rewards = torch.zeros([args.num_processes, 1])

    if args.cuda:
        states = states.cuda()
        current_state = current_state.cuda()
        rewards = rewards.cuda()
        returns = returns.cuda()
        actions = actions.cuda()
        masks = masks.cuda()

    for j in range(num_updates):
        for step in range(args.num_steps):
            # Sample actions
            _, logits = actor_critic(Variable(states[step], volatile=True))
            probs = F.softmax(logits)
            log_probs = F.log_softmax(logits).data
            actions[step] = probs.multinomial().data

            cpu_actions = actions[step].cpu()
            cpu_actions = cpu_actions.numpy()

            # Obser reward and next state
            state, reward, done, info = envs.step(cpu_actions)

            reward = torch.from_numpy(np.expand_dims(np.stack(reward), 1)).float()
            episode_rewards += reward

            np_masks = np.array([0.0 if done_ else 1.0 for done_ in done])

            # If done then clean the history of observations.
            pt_masks = torch.from_numpy(np_masks.reshape(np_masks.shape[0], 1, 1, 1)).float()
            if args.cuda:
                pt_masks = pt_masks.cuda()
            current_state *= pt_masks

            update_current_state(state)
            states[step + 1].copy_(current_state)
            rewards[step].copy_(reward)
            masks[step].copy_(torch.from_numpy(np_masks))

            final_rewards *= masks[step].cpu()
            final_rewards += (1 - masks[step].cpu()) * episode_rewards

            episode_rewards *= masks[step].cpu()

        # Reshape to do in a single forward pass for all steps
        values, logits = actor_critic(Variable(states.view(-1, *states.size()[-3:])))
        log_probs = F.log_softmax(logits)
        probs = F.softmax(logits)

        # Unreshape
        logits_size = (args.num_steps + 1, args.num_processes, logits.size(-1))

        log_probs = F.log_softmax(logits).view(logits_size)[:-1]
        probs = F.softmax(logits).view(logits_size)[:-1]

        values = values.view(args.num_steps + 1, args.num_processes, 1)
        logits = logits.view(logits_size)[:-1]

        action_log_probs = log_probs.gather(2, Variable(actions.unsqueeze(2)))

        dist_entropy = -(log_probs * probs).sum(-1).mean()

        returns[-1] = values[-1].data

        for step in reversed(range(args.num_steps)):
            returns[step] = returns[step + 1] * \
                args.gamma * masks[step] + rewards[step]

        value_loss = (values[:-1] - Variable(returns[:-1])).pow(2).mean()

        advantages = returns[:-1] - values[:-1].data
        action_loss = -(Variable(advantages) * action_log_probs).mean()

        optimizer.zero_grad()
        (value_loss * args.value_loss_coef + action_loss - dist_entropy * args.entropy_coef).backward()

        nn.utils.clip_grad_norm(actor_critic.parameters(), args.max_grad_norm)
        optimizer.step()

        states[0].copy_(states[-1])

        if j % args.log_interval == 0:
            print("Updates {}, num frames {}, mean clipped reward {:.5f}, max clipped reward {:.1f}, entropy {:.5f}, value loss {:.5f}, policy loss {:.5f}".format(
                j, j * args.num_processes * args.num_steps, final_rewards.mean(), final_rewards.max(), -dist_entropy.data[0], value_loss.data[0], action_loss.data[0]))

if __name__ == "__main__":
    main()
<commit_msg>Add warning message and more stats<commit_after>import gym
import os
import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from baselines.common.vec_env.subproc_vec_env import SubprocVecEnv
from envs import make_env
from model import ActorCritic
from torch.autograd import Variable
import argparse

parser = argparse.ArgumentParser(description='A3C')
parser.add_argument('--lr', type=float, default=7e-4,
                    help='learning rate (default: 7e-4)')
parser.add_argument('--eps', type=float, default=1e-5,
                    help='RMSprop optimizer epsilon (default: 1e-5)')
parser.add_argument('--alpha', type=float, default=0.99,
                    help='RMSprop optimizer apha (default: 0.99)')
parser.add_argument('--gamma', type=float, default=0.99,
                    help='discount factor for rewards (default: 0.99)')
parser.add_argument('--entropy-coef', type=float, default=0.01,
                    help='entropy term coefficient (default: 0.01)')
parser.add_argument('--value-loss-coef', type=float, default=0.5,
                    help='value loss coefficient (default: 0.5)')
parser.add_argument('--max-grad-norm', type=float, default=0.5,
                    help='value loss coefficient (default: 0.5)')
parser.add_argument('--seed', type=int, default=1,
                    help='random seed (default: 1)')
parser.add_argument('--num-processes', type=int, default=16,
                    help='how many training CPU processes to use (default: 16)')
parser.add_argument('--num-steps', type=int, default=5,
                    help='number of forward steps in A2C (default: 5)')
parser.add_argument('--num-stack', type=int, default=4,
                    help='number of frames to stack (default: 4)')
parser.add_argument('--log-interval', type=int, default=10,
                    help='log interval, one log per n updates (default: 10)')
parser.add_argument('--num-frames', type=int, default=10e6,
                    help='number of frames to train (default: 10e6)')
parser.add_argument('--env-name', default='PongNoFrameskip-v4',
                    help='environment to train on (default: PongNoFrameskip-v4)')
parser.add_argument('--log-dir', default='/tmp/gym/',
                    help='directory to save agent logs (default: /tmp/gym)')
parser.add_argument('--no-cuda', action='store_true', default=False,
                    help='disables CUDA training')

args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()

num_updates = int(args.num_frames) // args.num_steps // args.num_processes

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

try:
    os.makedirs(args.log_dir)
except OSError:
    pass

def main():
    print("#######")
    print("WARNING: All rewards are clipped so you need to use a monitor (see envs.py) to get true rewards")
    print("#######")

    os.environ['OMP_NUM_THREADS'] = '1'

    envs = SubprocVecEnv([make_env(args.env_name, args.seed, i, args.log_dir)
                            for i in range(args.num_processes)])

    actor_critic = ActorCritic(envs.observation_space.shape[0] * args.num_stack, envs.action_space)

    if args.cuda:
        actor_critic.cuda()

    optimizer = optim.RMSprop(actor_critic.parameters(), args.lr, eps=args.eps, alpha=args.alpha)
    #optimizer = KFACOptimizer(actor_critic, damping=1e-2, kl_clip=0.01, stat_decay=0.99)

    obs_shape = envs.observation_space.shape
    obs_shape = (obs_shape[0] * args.num_stack, obs_shape[1], obs_shape[2])

    states = torch.zeros(args.num_steps + 1, args.num_processes, *obs_shape)
    current_state = torch.zeros(args.num_processes, *obs_shape)
    counts = 0

    def update_current_state(state):
        state = torch.from_numpy(np.stack(state)).float()
        current_state[:, :-1] = current_state[:, 1:]
        current_state[:, -1] = state

    state = envs.reset()
    update_current_state(state)

    rewards = torch.zeros(args.num_steps, args.num_processes, 1)
    returns = torch.zeros(args.num_steps + 1, args.num_processes, 1)

    actions = torch.LongTensor(args.num_steps, args.num_processes)
    masks = torch.zeros(args.num_steps, args.num_processes, 1)

    # These variables are used to compute average rewards for all processes.
    episode_rewards = torch.zeros([args.num_processes, 1])
    final_rewards = torch.zeros([args.num_processes, 1])

    if args.cuda:
        states = states.cuda()
        current_state = current_state.cuda()
        rewards = rewards.cuda()
        returns = returns.cuda()
        actions = actions.cuda()
        masks = masks.cuda()

    for j in range(num_updates):
        for step in range(args.num_steps):
            # Sample actions
            _, logits = actor_critic(Variable(states[step], volatile=True))
            probs = F.softmax(logits)
            log_probs = F.log_softmax(logits).data
            actions[step] = probs.multinomial().data

            cpu_actions = actions[step].cpu()
            cpu_actions = cpu_actions.numpy()

            # Obser reward and next state
            state, reward, done, info = envs.step(cpu_actions)

            reward = torch.from_numpy(np.expand_dims(np.stack(reward), 1)).float()
            episode_rewards += reward

            np_masks = np.array([0.0 if done_ else 1.0 for done_ in done])

            # If done then clean the history of observations.
            pt_masks = torch.from_numpy(np_masks.reshape(np_masks.shape[0], 1, 1, 1)).float()
            if args.cuda:
                pt_masks = pt_masks.cuda()
            current_state *= pt_masks

            update_current_state(state)
            states[step + 1].copy_(current_state)
            rewards[step].copy_(reward)
            masks[step].copy_(torch.from_numpy(np_masks).unsqueeze(1))

            final_rewards *= masks[step].cpu()
            final_rewards += (1 - masks[step].cpu()) * episode_rewards

            episode_rewards *= masks[step].cpu()

        # Reshape to do in a single forward pass for all steps
        values, logits = actor_critic(Variable(states.view(-1, *states.size()[-3:])))
        log_probs = F.log_softmax(logits)
        probs = F.softmax(logits)

        # Unreshape
        logits_size = (args.num_steps + 1, args.num_processes, logits.size(-1))

        log_probs = F.log_softmax(logits).view(logits_size)[:-1]
        probs = F.softmax(logits).view(logits_size)[:-1]

        values = values.view(args.num_steps + 1, args.num_processes, 1)
        logits = logits.view(logits_size)[:-1]

        action_log_probs = log_probs.gather(2, Variable(actions.unsqueeze(2)))

        dist_entropy = -(log_probs * probs).sum(-1).mean()

        returns[-1] = values[-1].data

        for step in reversed(range(args.num_steps)):
            returns[step] = returns[step + 1] * \
                args.gamma * masks[step] + rewards[step]

        value_loss = (values[:-1] - Variable(returns[:-1])).pow(2).mean()

        advantages = returns[:-1] - values[:-1].data
        action_loss = -(Variable(advantages) * action_log_probs).mean()

        optimizer.zero_grad()
        (value_loss * args.value_loss_coef + action_loss - dist_entropy * args.entropy_coef).backward()

        nn.utils.clip_grad_norm(actor_critic.parameters(), args.max_grad_norm)
        optimizer.step()

        states[0].copy_(states[-1])

        if j % args.log_interval == 0:
            print("Updates {}, num frames {}, mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}, entropy {:.5f}, value loss {:.5f}, policy loss {:.5f}".format(
                j, j * args.num_processes * args.num_steps, final_rewards.mean(), final_rewards.median(), final_rewards.min(), final_rewards.max(), -dist_entropy.data[0], value_loss.data[0], action_loss.data[0]))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>__plugin_name__      = "RedstonerUtils"
__plugin_version__   = "3.0"
__plugin_mainclass__ = "foobar"

import sys
from traceback import format_exc as print_traceback

# damn pythonloader changed the PATH
sys.path += ['', '/usr/lib/python2.7', '/usr/lib/python2.7/plat-linux2', '/usr/lib/python2.7/lib-tk', '/usr/lib/python2.7/lib-old', '/usr/lib/python2.7/lib-dynload', '/usr/local/lib/python2.7/dist-packages', '/usr/lib/python2.7/dist-packages', '/usr/lib/pymodules/python2.7', '/usr/lib/pyshared/python2.7']

try:
    # Library that adds a bunch of re-usable methods which are used in nearly all other modules
    from helpers import *
except:
    print("[RedstonerUtils] ERROR: Failed to import helpers:")
    print(print_traceback())

@hook.enable
def on_enable():
    info("RedstonerUtils enabled!")

@hook.disable
def on_disable():
    shared["modules"]["reports"].stop_reporting()
    info("RedstonerUtils disabled!")

info("Loading RedstonerUtils...")

# Import all modules, in this order
shared["load_modules"] = [
    # Collection of tiny utilities
    "misc",
    # Adds chat for staff using /ac <text or ,<text>
    "adminchat",
    # Adds /badge, allows to give players achievements
    "badges",
    # Adds /calc, toggles automatic solving of Math expressions in chat
    "calc",
    # Plugin to locate laggy chunks. /lc <n> lists chunks with more than n entities
    "lagchunks",
    # Adds /report and /rp, Stores reports with time and location
    "reports",
    # Adds group-chat with /chatgroup and /cgt to toggle normal chat into group mode
    "chatgroups",
    # Adds /token, reads and writes from the database to generate pronouncable (and thus memorable) registration-tokens for the website
    "webtoken",
    # Adds /lol, broadcasts random funyy messages. A bit like the splash text in the menu
    "saylol",
    # Shows the owner of a skull when right-clicked
    "skullclick",
    # Adds /listen, highlights chat and plays a sound when your name was mentioned
    "mentio",
    # Adds /cycler, swaps the hotbar with inventory when player changes slot from right->left or left->right
    "cycle",
    # Adds /getmotd & /setmotd to update the motd on the fly (no reboot)
    "motd",
    # AnswerBot. Hides stupid questions from chat and tells the sender about /faq or the like
    "abot",
    # Adds '/forcefield', creates forcefield for players who want it.
    "forcefield",
    # Adds /damnspam, creates timeout for buttons/levers to mitigate button spam.
    "damnspam",
    # Adds /check, useful to lookup details about a player
    "check",
    # Adds /imout, displays fake leave/join messages
    "adminnotes",
    # Adds /an, a command you can use to share thoughts/plans/news
    "imout"
    # NOTICE: If you add something here, please add a small descriptive comment above!
]
shared["modules"] = {}
for module in shared["load_modules"]:
    try:
        shared["modules"][module] = __import__(module)
        info("Module %s loaded." % module)
    except:
        error("Failed to import module %s:" % module)
        error(print_traceback())<commit_msg>added snowbrawl to modules<commit_after>__plugin_name__      = "RedstonerUtils"
__plugin_version__   = "3.0"
__plugin_mainclass__ = "foobar"

import sys
from traceback import format_exc as print_traceback

# damn pythonloader changed the PATH
sys.path += ['', '/usr/lib/python2.7', '/usr/lib/python2.7/plat-linux2', '/usr/lib/python2.7/lib-tk', '/usr/lib/python2.7/lib-old', '/usr/lib/python2.7/lib-dynload', '/usr/local/lib/python2.7/dist-packages', '/usr/lib/python2.7/dist-packages', '/usr/lib/pymodules/python2.7', '/usr/lib/pyshared/python2.7']

try:
    # Library that adds a bunch of re-usable methods which are used in nearly all other modules
    from helpers import *
except:
    print("[RedstonerUtils] ERROR: Failed to import helpers:")
    print(print_traceback())

@hook.enable
def on_enable():
    info("RedstonerUtils enabled!")

@hook.disable
def on_disable():
    shared["modules"]["reports"].stop_reporting()
    info("RedstonerUtils disabled!")

info("Loading RedstonerUtils...")

# Import all modules, in this order
shared["load_modules"] = [
    # Collection of tiny utilities
    "misc",
    # Adds chat for staff using /ac <text or ,<text>
    "adminchat",
    # Adds /badge, allows to give players achievements
    "badges",
    # Adds /calc, toggles automatic solving of Math expressions in chat
    "calc",
    # Plugin to locate laggy chunks. /lc <n> lists chunks with more than n entities
    "lagchunks",
    # Adds /report and /rp, Stores reports with time and location
    "reports",
    # Adds group-chat with /chatgroup and /cgt to toggle normal chat into group mode
    "chatgroups",
    # Adds /token, reads and writes from the database to generate pronouncable (and thus memorable) registration-tokens for the website
    "webtoken",
    # Adds /lol, broadcasts random funyy messages. A bit like the splash text in the menu
    "saylol",
    # Shows the owner of a skull when right-clicked
    "skullclick",
    # Adds /listen, highlights chat and plays a sound when your name was mentioned
    "mentio",
    # Adds /cycler, swaps the hotbar with inventory when player changes slot from right->left or left->right
    "cycle",
    # Adds /getmotd & /setmotd to update the motd on the fly (no reboot)
    "motd",
    # AnswerBot. Hides stupid questions from chat and tells the sender about /faq or the like
    "abot",
    # Adds '/forcefield', creates forcefield for players who want it.
    "forcefield",
    # Adds /damnspam, creates timeout for buttons/levers to mitigate button spam.
    "damnspam",
    # Adds /check, useful to lookup details about a player
    "check",
    # Adds /imout, displays fake leave/join messages
    "adminnotes",
    # Adds /an, a command you can use to share thoughts/plans/news
    "imout",
    #adds snowbrawl minigame
    "snowbrawl.py"
    # NOTICE: If you add something here, please add a small descriptive comment above!
]
shared["modules"] = {}
for module in shared["load_modules"]:
    try:
        shared["modules"][module] = __import__(module)
        info("Module %s loaded." % module)
    except:
        error("Failed to import module %s:" % module)
        error(print_traceback())<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import with_statement
import os, sys, shutil, zipfile, platform
from contextlib import closing
from zipfile import ZipFile, ZIP_DEFLATED

## Update odmtools.meta.data whenever creating a release
from odmtools.meta import data

BASE_DIR = os.path.dirname(os.path.realpath(__file__))
SETUP_DIR = os.path.join(BASE_DIR, 'setup')
DIST_DIR = os.path.join(SETUP_DIR, 'Dist/')
WIN_DIR = os.path.join(SETUP_DIR, "Windows")
MAC_DIR = os.path.join(SETUP_DIR, "Mac")

MAC_WORK_DIR = os.path.join(MAC_DIR, "Temp")
WORK_DIR = os.path.join(WIN_DIR, "Temp")

ICON_DIR = os.path.join('odmtools', 'common', "icons")
WIN_ICON_FILE = os.path.join(ICON_DIR, "ODMTools.ico")
MAC_ICON_FILE = os.path.join(ICON_DIR, "ODMTools.icns")

APP_DIR = os.path.join(MAC_DIR, "ODMTools.app")
# Location of Windows files
APP_FILE = os.path.join(BASE_DIR, "ODMTools.py")
MAKE_FILE = os.path.realpath(__file__)
VERSION_FILE = os.path.join(SETUP_DIR, "version.txt")

# Location of Innosetup Installer
INNO_SCRIPT = os.path.join(WIN_DIR, "odmtools_setup.iss")
INNO_EXECUTABLE = '"C:\\Program Files (x86)\\Inno Setup 5\\ISCC.exe"'
ICE_SCRIPT = os.path.join(MAC_DIR, "ODMTools.packproj")
ICE_EXECUTABLE ='freeze'

def check_if_dirs_exist():
    try:
        if sys.platform == 'win32':
            print "Trying to open WIN_DIR: ",
            if not os.path.exists(WIN_DIR):
                os.mkdir(WIN_DIR)
                print "Created: ", WIN_DIR
            print "Success"

            print "Trying to confirm that INNO_SCRIPT exists: ",
            assert os.path.exists(INNO_SCRIPT)
            print "Success: ", INNO_SCRIPT
        elif sys.platform =="darwin":
            print "Trying to open MAC_DIR: "
            assert os.path.exists(MAC_DIR)
            print "Success"

        print "Trying to open WORK_DIR: ",
        if not os.path.exists(WORK_DIR):
            print "Failed... Trying to create the folder...",
            os.mkdir(WORK_DIR)
            print "Created: ", WORK_DIR
        print "Success"

        print "Trying to open DIST_DIR: ",
        if not os.path.exists(DIST_DIR):
            print "Failed... Trying to create the folder...",
            os.mkdir(DIST_DIR)
            print "Created: ", DIST_DIR
        print "Success"

        print "Trying to open ICON_DIR: ",
        assert os.path.exists(ICON_DIR)
        print "Success"

    except Exception as e:
        print e

def zipdir(basedir, archivename):
    assert os.path.isdir(basedir)
    with closing(ZipFile(archivename, "w", ZIP_DEFLATED)) as z:
        for root, dirs, files in os.walk(basedir):
            #NOTE: ignore empty directories
            for fn in files:
                absfn = os.path.join(root, fn)
                zfn = absfn[len(basedir)+len(os.sep):]
                z.write(absfn, zfn)
def printInfo():
    print "============================================================="
    print "=             ODMTools Installer                             "
    print "= Be sure to update odmtools/meta/data with every release    "
    print "= Building release: {version}".format(version=data.version),
    print "\n= Platform: {platform}, {architecture}".format(platform=sys.platform, architecture=platform.architecture()), "\n="
    print "============================================================="
    print "Environment Variables: "
    print ("APP_FILE: ", APP_FILE)
    print ("MAKE_FILE: ", MAKE_FILE)
    print ("BASE_DIR: ", BASE_DIR)
    print ("SETUP_DIR: ", SETUP_DIR)
    print ("DIST_DIR: ", DIST_DIR)

    ## Windows Specific Files/Directories
    print ("WIN_DIR: ", WIN_DIR)
    print ("WIN_ICON_FILE: ", WIN_ICON_FILE)
    print ("WORK_DIR: ", WORK_DIR)
    print ("INNO_SCRIPT: ", INNO_SCRIPT)
    print ("INNO_EXECUTABLE: ", INNO_EXECUTABLE)

    ## OSX Specific Files/Directories
    print ("MAC_DIR: ", MAC_DIR)
    print ("MAC_ICON_FILE: ", MAC_ICON_FILE)
    print ("MAC_WORK_DIR: ", MAC_WORK_DIR)

    print "============================================================="
    print "Checking if the required directories exist"

    check_if_dirs_exist()

def obtain_exe_filename(console=False):
    if console:
        return "{app}_{version}_{os}_{arch}_{type}".format(app=data.app_name,
        version=data.version, os=sys.platform, arch='x86_64', type= "console")
    else:
        return "{app}_{version}_{os}_{arch}".format(app=data.app_name,
        version=data.version, os=sys.platform, arch='x86_64')

def delete_old_out_dir():
    loc_exists = os.path.exists(DIST_DIR)
    isFile = os.path.isfile(DIST_DIR)
    isDir = os.path.isdir(DIST_DIR)
    if loc_exists and isFile:
        print "Removing file DIST_DIR"
        os.remove(DIST_DIR)
    elif loc_exists and isDir:
        print "Removing directory DIST_DIR"
        shutil.rmtree(DIST_DIR)
    else:
        print "Nothing to remove"

def run_pyinstaller(console=False):
    try:
        if console:
            ## Console Version
            os.system('pyinstaller '
                '--clean '
                '--distpath=%s ' % WIN_DIR +
                '--workpath=%s ' % WORK_DIR +
                '--specpath=%s ' % WIN_DIR +
                '--upx-dir=%s ' % BASE_DIR +
                '--icon=%s ' % WIN_ICON_FILE +
                '--version-file=%s ' % VERSION_FILE +
                '--noconfirm ' + APP_FILE)
        else:
            ## Non Console Version
            os.system('pyinstaller '
                '--clean '
                '--distpath=%s ' % WIN_DIR +
                '--workpath=%s ' % WORK_DIR +
                '--specpath=%s ' % WIN_DIR +
                '--upx-dir=%s ' % BASE_DIR +
                '--icon=%s ' % WIN_ICON_FILE +
                '--version-file=%s ' % VERSION_FILE +
                '--noconsole '
                '--noconfirm ' + APP_FILE)

        return True
    except Exception as e:
        print (e)
        return False

def mac_pyinstaller():
    try:
        os.system('pyinstaller '
            '--clean '
            '--distpath=%s ' % DIST_DIR +
            '--workpath=%s ' % MAC_WORK_DIR +
            '--specpath=%s ' % MAC_DIR +
            '--upx-dir=%s ' % BASE_DIR +
            '--icon=%s ' % MAC_ICON_FILE +
            '--version-file=%s ' % VERSION_FILE +
            '--windowed '
            #'--onefile '
            #'--hidden-import="libwx_osx_cocoau-3.0.0.0.0.dylib" '
            '--noconfirm ' + APP_FILE)

        os.system("cp /anaconda/envs/odmtools/lib/libwx_osx_cocoau-3.0.0.0.0.dylib %s" % os.path.join(APP_DIR, "Contents/MacOS/"))
        #copy "libwx_osx_cocoau-3.0.0.0.0.dylib"
        return True
    except Exception as e:
        print (e)
        return False

def move_to_dist(filename):
    assert filename

    if not os.path.isdir(DIST_DIR):
        os.mkdir(DIST_DIR)

    print "Moving {filename} to {dist}".format(filename=os.path.abspath(filename), dist=DIST_DIR),
    try:
        shutil.move(os.path.abspath(filename), DIST_DIR)
        print "Success"
    except shutil.Error as e:
        print (e)

def run_inno():
    os.system(INNO_EXECUTABLE + " " + INNO_SCRIPT)

def run_no_installer():
    # Need to finish, Not functional
    raise ("Not functional yet")
    filename = obtain_exe_filename()

    zipdir(os.path.join('odmtools'), filename)
    move_to_dist(filename)

def run_iceberg():
    os.system(ICE_EXECUTABLE + " "+ ICE_SCRIPT)

def main():
    delete_old_out_dir()
    printInfo()

    if sys.platform == 'win32':

        print "Creating Windows Executable..."
        if run_pyinstaller():
            run_inno()

        print "Creating Windows Executable Console..."
        if run_pyinstaller(console=True):
            run_inno()

    elif sys.platform =='darwin':
        if(mac_pyinstaller()):
            run_iceberg()

    elif sys.platform == 'linux2':
        ## Testing, not officially supported
        run_no_installer()

if __name__ == '__main__':
    main()

# OUT_DIR = os.path.join(BASE_DIR, 'dist')
# GTK_ZIP = os.path.join('tools', 'windows', 'gtk', 'gtk_to_copy.zip')
# INNO_SCRIPT = os.path.join('tools', 'config-inno.iss')
# INNO_EXECUTABLE = '"c:\\Program Files\\Inno Setup 5\\ISCC.exe"'

# class Unzip(object):
#    def __init__(self, from_file, to_dir, verbose = False):
#        """Removed for brevity, you can find a recipe similar to this in the Cook Book"""

# def run_pyinstaller():
#    # A hack really, but remember setup.py will run on import
#    sys.argv.append('py2exe')
#    import setup

# def unzip_gtk():
#    Unzip(GTK_ZIP, OUT_DIR)

# def run_inno():
#    os.system(INNO_EXECUTABLE + " " + INNO_SCRIPT)

# def main():
#   #Clean any mess we previously made
#   delete_old_out_dir()
#   # run py2exe
#   run_py2exe()
#   # put the GTK data files in the dist directory
#   unzip_gtk()
#   # build the single file installer
#   run_inno()
#   # prevent the windows command prompt from just closing
#   raw_input('Done..')

# if __name__ == '__main__':
#   main()
<commit_msg>Updating make.py<commit_after># -*- coding: utf-8 -*-
from __future__ import with_statement
import os, sys, shutil, zipfile, platform
from contextlib import closing
from zipfile import ZipFile, ZIP_DEFLATED

## Update odmtools.meta.data whenever creating a release
from odmtools.meta import data

BASE_DIR = os.path.dirname(os.path.realpath(__file__))
SETUP_DIR = os.path.join(BASE_DIR, 'setup')
DIST_DIR = os.path.join(SETUP_DIR, 'Dist/')
WIN_DIR = os.path.join(SETUP_DIR, "Windows")
MAC_DIR = os.path.join(SETUP_DIR, "Mac")

MAC_WORK_DIR = os.path.join(MAC_DIR, "Temp")
WORK_DIR = os.path.join(WIN_DIR, "Temp")

ICON_DIR = os.path.join('odmtools', 'common', "icons")
WIN_ICON_FILE = os.path.join(ICON_DIR, "ODMTools.ico")
MAC_ICON_FILE = os.path.join(ICON_DIR, "ODMTools.icns")

APP_DIR = os.path.join(MAC_DIR, "ODMTools.app")
# Location of Windows files
APP_FILE = os.path.join(BASE_DIR, "ODMTools.py")
MAKE_FILE = os.path.realpath(__file__)
VERSION_FILE = os.path.join(SETUP_DIR, "version.txt")

# Location of Innosetup Installer
INNO_SCRIPT = os.path.join(WIN_DIR, "odmtools_setup.iss")
INNO_EXECUTABLE = '"C:\\Program Files (x86)\\Inno Setup 5\\ISCC.exe"'
ICE_SCRIPT = os.path.join(MAC_DIR, "ODMTools.packproj")
ICE_EXECUTABLE ='freeze'

def check_if_dirs_exist():
    try:
        if sys.platform == 'win32':
            print "Trying to open WIN_DIR: ",
            if not os.path.exists(WIN_DIR):
                os.mkdir(WIN_DIR)
                print "Created: ", WIN_DIR
            print "Success"

            print "Trying to confirm that INNO_SCRIPT exists: ",
            assert os.path.exists(INNO_SCRIPT)
            print "Success: ", INNO_SCRIPT
        elif sys.platform =="darwin":
            print "Trying to open MAC_DIR: "
            assert os.path.exists(MAC_DIR)
            print "Success"

        print "Trying to open WORK_DIR: ",
        if not os.path.exists(WORK_DIR):
            print "Failed... Trying to create the folder...",
            os.mkdir(WORK_DIR)
            print "Created: ", WORK_DIR
        print "Success"

        print "Trying to open DIST_DIR: ",
        if not os.path.exists(DIST_DIR):
            print "Failed... Trying to create the folder...",
            os.mkdir(DIST_DIR)
            print "Created: ", DIST_DIR
        print "Success"

        print "Trying to open ICON_DIR: ",
        assert os.path.exists(ICON_DIR)
        print "Success"

    except Exception as e:
        print e

def zipdir(basedir, archivename):
    assert os.path.isdir(basedir)
    with closing(ZipFile(archivename, "w", ZIP_DEFLATED)) as z:
        for root, dirs, files in os.walk(basedir):
            #NOTE: ignore empty directories
            for fn in files:
                absfn = os.path.join(root, fn)
                zfn = absfn[len(basedir)+len(os.sep):]
                z.write(absfn, zfn)
def printInfo():
    print "============================================================="
    print "=             ODMTools Installer                             "
    print "= Be sure to update odmtools/meta/data with every release    "
    print "= Building release: {version}".format(version=data.version),
    print "\n= Platform: {platform}, {architecture}".format(platform=sys.platform, architecture=platform.architecture()), "\n="
    print "============================================================="
    print "Environment Variables: "
    print ("APP_FILE: ", APP_FILE)
    print ("MAKE_FILE: ", MAKE_FILE)
    print ("BASE_DIR: ", BASE_DIR)
    print ("SETUP_DIR: ", SETUP_DIR)
    print ("DIST_DIR: ", DIST_DIR)

    ## Windows Specific Files/Directories
    print ("WIN_DIR: ", WIN_DIR)
    print ("WIN_ICON_FILE: ", WIN_ICON_FILE)
    print ("WORK_DIR: ", WORK_DIR)
    print ("INNO_SCRIPT: ", INNO_SCRIPT)
    print ("INNO_EXECUTABLE: ", INNO_EXECUTABLE)

    ## OSX Specific Files/Directories
    print ("MAC_DIR: ", MAC_DIR)
    print ("MAC_ICON_FILE: ", MAC_ICON_FILE)
    print ("MAC_WORK_DIR: ", MAC_WORK_DIR)

    print "============================================================="
    print "Checking if the required directories exist"

    check_if_dirs_exist()

def obtain_exe_filename(console=False):
    if console:
        return "{app}_{version}_{os}_{arch}_{type}".format(app=data.app_name,
        version=data.version, os=sys.platform, arch='x86_64', type= "console")
    else:
        return "{app}_{version}_{os}_{arch}".format(app=data.app_name,
        version=data.version, os=sys.platform, arch='x86_64')

def delete_old_out_dir():
    loc_exists = os.path.exists(DIST_DIR)
    isFile = os.path.isfile(DIST_DIR)
    isDir = os.path.isdir(DIST_DIR)
    if loc_exists and isFile:
        print "Removing file DIST_DIR"
        os.remove(DIST_DIR)
    elif loc_exists and isDir:
        print "Removing directory DIST_DIR"
        shutil.rmtree(DIST_DIR)
    else:
        print "Nothing to remove"

def run_pyinstaller(console=False):
    try:
        if console:
            ## Console Version
            os.system('pyinstaller '
                '--clean '
                '--distpath=%s ' % WIN_DIR +
                '--workpath=%s ' % WORK_DIR +
                '--specpath=%s ' % WIN_DIR +
                '--upx-dir=%s ' % BASE_DIR +
                '--icon=%s ' % WIN_ICON_FILE +
                '--version-file=%s ' % VERSION_FILE +
                '--noconfirm ' + APP_FILE)
        else:
            ## Non Console Version
            os.system('pyinstaller '
                '--clean '
                '--distpath=%s ' % WIN_DIR +
                '--workpath=%s ' % WORK_DIR +
                '--specpath=%s ' % WIN_DIR +
                '--upx-dir=%s ' % BASE_DIR +
                '--icon=%s ' % WIN_ICON_FILE +
                '--version-file=%s ' % VERSION_FILE +
                '--noconsole '
                '--noconfirm ' + APP_FILE)

        return True
    except Exception as e:
        print (e)
        return False

def mac_pyinstaller():
    try:
        os.system('pyinstaller '
            '--clean '
            '--distpath=%s ' % DIST_DIR +
            '--workpath=%s ' % MAC_WORK_DIR +
            '--specpath=%s ' % MAC_DIR +
            '--upx-dir=%s ' % BASE_DIR +
            '--icon=%s ' % MAC_ICON_FILE +
            '--version-file=%s ' % VERSION_FILE +
            '--windowed '
            #'--onefile '
            #'--hidden-import="libwx_osx_cocoau-3.0.0.0.0.dylib" '
            '--noconfirm ' + APP_FILE)

        os.system("cp /anaconda/envs/odmtools/lib/libwx_osx_cocoau-3.0.0.0.0.dylib %s" % os.path.join(APP_DIR, "Contents/MacOS/"))
        #copy "libwx_osx_cocoau-3.0.0.0.0.dylib"
        return True
    except Exception as e:
        print (e)
        return False

def move_to_dist(filename):
    assert filename

    if not os.path.isdir(DIST_DIR):
        os.mkdir(DIST_DIR)

    print "Moving {filename} to {dist}".format(filename=os.path.abspath(filename), dist=DIST_DIR),
    try:
        shutil.move(os.path.abspath(filename), DIST_DIR)
        print "Success"
    except shutil.Error as e:
        print (e)

def run_inno():
    os.system(INNO_EXECUTABLE + " " + INNO_SCRIPT)

def run_no_installer():
    # Need to finish, Not functional
    raise ("Not functional yet")
    filename = obtain_exe_filename()

    zipdir(BASE_DIR, filename)
    move_to_dist(filename)

def run_iceberg():
    os.system(ICE_EXECUTABLE + " "+ ICE_SCRIPT)

def main():
    delete_old_out_dir()
    printInfo()

    if sys.platform == 'win32':

        print "Creating Windows Executable..."
        if run_pyinstaller():
            run_inno()

        print "Creating Windows Executable Console..."
        if run_pyinstaller(console=True):
            run_inno()

        print "Create No Installer "

        ## Create Shortcut
        ## Create File
        ## Zip Executable

    elif sys.platform =='darwin':
        if(mac_pyinstaller()):
            run_iceberg()

    # elif sys.platform == 'linux2':
    #     ## Testing, not officially supported
    #     run_no_installer()

if __name__ == '__main__':
    main()

# OUT_DIR = os.path.join(BASE_DIR, 'dist')
# GTK_ZIP = os.path.join('tools', 'windows', 'gtk', 'gtk_to_copy.zip')
# INNO_SCRIPT = os.path.join('tools', 'config-inno.iss')
# INNO_EXECUTABLE = '"c:\\Program Files\\Inno Setup 5\\ISCC.exe"'

# class Unzip(object):
#    def __init__(self, from_file, to_dir, verbose = False):
#        """Removed for brevity, you can find a recipe similar to this in the Cook Book"""

# def run_pyinstaller():
#    # A hack really, but remember setup.py will run on import
#    sys.argv.append('py2exe')
#    import setup

# def unzip_gtk():
#    Unzip(GTK_ZIP, OUT_DIR)

# def run_inno():
#    os.system(INNO_EXECUTABLE + " " + INNO_SCRIPT)

# def main():
#   #Clean any mess we previously made
#   delete_old_out_dir()
#   # run py2exe
#   run_py2exe()
#   # put the GTK data files in the dist directory
#   unzip_gtk()
#   # build the single file installer
#   run_inno()
#   # prevent the windows command prompt from just closing
#   raw_input('Done..')

# if __name__ == '__main__':
#   main()
<|endoftext|>