<commit_before>from __future__ import print_function
import os
import sys
import tinytag
import pickle
import argparse
from tinytag import TinyTag
import shutil

# ------------Function definition section--------------------#

def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

def removeNonAscii(s): return "".join(i for i in s if ord(i)<128 and ord(i)>0)

def saveDB(db,filename):
    with open(filename, 'wb') as f:
        pickle.dump(db, f, pickle.HIGHEST_PROTOCOL)

def loadDB(filename):
    with open(filename, 'rb') as f:
        return pickle.load(f)

def scanToDB(path,db):
	for root, dirs, files in os.walk(path):
		for elem in files:
			if elem.endswith('.m4a') or elem.endswith('.mp3') or elem.endswith('.flac'):
				print('Reading file:', ''.join(['\'',os.path.join(root,elem),'\'']))
				try:
					tag = TinyTag.get(os.path.join(root,elem))
					artist = removeNonAscii(tag.artist).strip(' ,')
					album = removeNonAscii(tag.album).strip(' ,')
					key = ''.join([artist,' - ',album])
					if db.has_key(key):
						db[key].append(os.path.join(root,elem))
					else:
						db[key] = [os.path.join(root,elem)]
				except tinytag.TinyTagException as err:
					eprint('Tag Error:', err)
				except Exception as e:
					eprint('Error:', str(e), '>file:', elem)

#def scanToDB_old(path,filename,db):
#	if filename.endswith('.m4a') or filename.endswith('.mp3') or filename.endswith('.flac'):
#		try:
#			tag = TinyTag.get(os.path.join(path,filename))
#			artist = removeNonAscii(tag.artist).strip(' ,')
#			album = removeNonAscii(tag.album).strip(' ,')
#			key = ''.join([artist,' - ',album])
#			if db.has_key(key):
#				db[key].append(os.path.join(path,filename))
#			else:
#				db[key] = [os.path.join(path,filename)]
#		except tinytag.TinyTagException as err:
#			eprint('Tag Error:', err)
#		except Exception as e:
#			eprint('Error:', str(e), 'file:', elem)

def organizeFiles(db,outpath):
	for key in db.keys():
		dirpath = os.path.join(outpath,key)
		if os.path.exists(dirpath):
			eprint('Warning: directory ', ''.join(['\'',dirpath,'\'']), 'already exists. To ensure that no data is being overwritten, this directory is skipped.')
			continue
		else:
			print('Creating directory', ''.join(['\'',dirpath,'\'.']))
			os.makedirs(dirpath)
			for elem in db[key]:
				# prepare filename
				filename = removeNonAscii(os.path.split(elem)[1])
				print('Copying', ''.join(['\'',elem,'\'']), 'to', ''.join(['\'',os.path.join(dirpath,filename),'\'.']))
				shutil.copy2(elem,os.path.join(dirpath,filename))

def printDB(db):
	for key in db.keys():
		print(key)
		for elem in sorted(db[key]):
			print('\t',elem)

# -------------------Main section----------------------------#

# Setup argument parser
parser = argparse.ArgumentParser(description='Audio file organizer. Recursively scans for audio files (supported formats: .flac, .mp3, .m4a) in the specified folder and organizes them according to the \'artist\' and \'album\' tag.')
parser.add_argument('path',help='Path of the directory to scan for audio files. If the \'-b\' option is used, the list is loaded directly from the binary file (.plk)')
parser.add_argument('-p',action='store_true',help='Print organized audio file list')
parser.add_argument('-b',action='store_true',help='Don\'t scan directory but load the organized list from a previously created (with \'-s\' option) binary file (.plk)')
parser.add_argument('-s',metavar='file',help='Save organized list into a pickle <file>')
parser.add_argument('-c',metavar='dir',help='Create directories in <dir> according to the organized list and copy the appropriate audio files in it. <dir> must exist')
args = parser.parse_args()
#parser.print_help()
#print(args)

# Create empty DB dictionary
db = {}

# Check -b option
if args.b: # Read from .plk file
	if not args.path.endswith('.pkl'):
		eprint('binary input file should be of \'pickle\' format (.plk)')
		exit()
	else:
		db = loadDB(args.path)
else: # Scan given directory
	if os.path.exists(args.path):
		scanToDB(args.path,db)
	else:
		eprint('Specified directory', ''.join(['\'',args.path,'\'']), 'does not exist. Please specify an existing directory and try again.')

# Check -s option
if args.s:
	if not args.s.endswith('.pkl'):
		eprint('binary output file should be of \'pickle\' format (.plk)')
		exit()
	else:
		saveDB(db,args.s)

# Check -p option
if args.p:
	printDB(db)

# Check -c option
if args.c:
	outpath = args.c
	if os.path.exists(outpath):
		organizeFiles(db,outpath)
	else:
		eprint('Specified directory', ''.join(['\'',outpath,'\'']), 'does not exist. Please specify an existing directory and try again.')

<commit_msg>Code cleanup<commit_after>from __future__ import print_function
import os
import sys
import tinytag
import pickle
import argparse
from tinytag import TinyTag
import shutil

# ----------------------- Function definition section -------------------------- #

def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

def removeNonAscii(s): return "".join(i for i in s if ord(i)<128 and ord(i)>0)

def saveDB(db,filename):
    with open(filename, 'wb') as f:
        pickle.dump(db, f, pickle.HIGHEST_PROTOCOL)

def loadDB(filename):
    with open(filename, 'rb') as f:
        return pickle.load(f)

def scanToDB(path,db):
	for root, dirs, files in os.walk(path):
		for elem in files:
			if elem.endswith('.m4a') or elem.endswith('.mp3') or elem.endswith('.flac'):
				print('Reading file:', ''.join(['\'',os.path.join(root,elem),'\'']))
				try:
					tag = TinyTag.get(os.path.join(root,elem))
					artist = removeNonAscii(tag.artist).strip(' ,')
					album = removeNonAscii(tag.album).strip(' ,')
					key = ''.join([artist,' - ',album])
					if db.has_key(key):
						db[key].append(os.path.join(root,elem))
					else:
						db[key] = [os.path.join(root,elem)]
				except tinytag.TinyTagException as err:
					eprint('Tag Error:', err)
				except Exception as e:
					eprint('Error:', str(e), '>file:', elem)

def organizeFiles(db,outpath):
	for key in db.keys():
		dirpath = os.path.join(outpath,key)
		if os.path.exists(dirpath):
			eprint('Warning: directory ', ''.join(['\'',dirpath,'\'']),
				('already exists. To ensure that no data is being overwritten, '
				'this directory is skipped.') )
			continue
		else:
			print('Creating directory', ''.join(['\'',dirpath,'\'.']))
			os.makedirs(dirpath)
			for elem in db[key]:
				# prepare filename
				filename = removeNonAscii(os.path.split(elem)[1])
				fullpath = os.path.join(dirpath,filename)
				print('Copying', ''.join(['\'',elem,'\'']), 'to',
					''.join(['\'',fullpath,'\'.']))
				shutil.copy2(elem,fullpath)

def printDB(db):
	for key in db.keys():
		print(key)
		for elem in sorted(db[key]):
			print('\t',elem)

# ------------------------------ Main section ---------------------------------- #

# Setup argument parser
parser = argparse.ArgumentParser(
	description=('Audio file organizer. Recursively scans for audio files '
	'(supported formats: .flac, .mp3, .m4a) in the specified folder and organizes'
	' them according to the \'artist\' and \'album\' tag.'))
parser.add_argument('path',
	help=('Path of the directory to scan for audio files. If the \'-b\' option is '
	'used, the list is loaded directly from the binary file (.plk)'))
parser.add_argument('-p',
	action='store_true',
	help='Print organized audio file list')
parser.add_argument('-b',
	action='store_true',
	help=('Don\'t scan directory but load the organized list from a previously '
	'created (with \'-s\' option) binary file (.plk)'))
parser.add_argument('-s',
	metavar='file',
	help='Save organized list into a pickle <file>')
parser.add_argument('-c',
	metavar='dir',
	help=('Create directories in <dir> according to the organized list and copy the'
	' appropriate audio files in it. <dir> must exist'))
args = parser.parse_args()

# Create empty DB dictionary
db = {}

# Check -b option
if args.b: # Read from .plk file
	if not args.path.endswith('.pkl'):
		eprint('binary input file should be of \'pickle\' format (.plk)')
		exit()
	else:
		db = loadDB(args.path)
else: # Scan given directory
	if os.path.exists(args.path):
		scanToDB(args.path,db)
	else:
		eprint('Specified directory', ''.join(['\'',args.path,'\'']),
			'does not exist. Please specify an existing directory and try again.')
		exit()

# Check -s option
if args.s:
	if not args.s.endswith('.pkl'):
		eprint('binary output file should be of \'pickle\' format (.plk)')
		exit()
	else:
		saveDB(db,args.s)

# Check -p option
if args.p:
	printDB(db)

# Check -c option
if args.c:
	outpath = args.c
	if os.path.exists(outpath):
		organizeFiles(db,outpath)
	else:
		eprint('Specified directory', ''.join(['\'',outpath,'\'']),
			'does not exist. Please specify an existing directory and try again.')
		exit()

<|endoftext|>


<commit_before>from os import path, listdir
from datetime import datetime

from mercurial import ui, hg, util
from mercurial.node import short
import git

from models import Repo, Commit

def find_git_repos(basepath, baseurl=''):
    """Returns a list of hg Repos in basepath"""
    def _find_repos(_path, name):
        if not path.isdir(_path):
            return []
        ret = []
        subdir = listdir(_path)
        if '.git' not in subdir:
            for r in subdir:
                reponame =  '{0}/{1}'.format(name, r) if name else r
                ret.extend(_find_repos(path.join(_path, r), reponame))
        else:
            ret.append(Repo(
                type='git',
                name=name,
                basepath=basepath,
                baseurl=baseurl,
                commits=[],
            ))
        return ret
    return _find_repos(basepath, '')

def find_git_commits(repo, from_date, until_date):
    """Initializes repo.commits with all commits between the given dates"""
    if from_date > until_date:
        # this should be impossible, maybe we should
        # raise an exception to indicate missuse
        return []

    #TODO: optimize
    gitrepo = git.Repo(repo.path)
    commits = [c for c in gitrepo.iter_commits()]

    # helper function to get branch from git.Commit
    branch = lambda c: gitrepo.git.branch(contains=c.hexsha).split('\n')[0][2:]
    # helper function to get datetime from git.Commit
    when = lambda c: datetime.fromtimestamp(c.committed_date)
    # helper function to get Commit from git.Commit
    com = lambda c: Commit(
        id=c.hexsha,
        repo=repo,
        urlpattern='?p={repo.name};a=commit;h={id}',
        user=c.author.name,
        date=when(c),
        #XXX: in git a commit may be in more than one branch
        # we are choosing the first we find it in.
        branch=branch(c),
        #XXX: I don't exactly know why we need to decode
        message=c.summary.decode('utf-8'),
    )
    # filter on date
    repo.commits.extend([com(c) for c in commits if from_date < when(c) < until_date])

<commit_msg>Fixing bare git repo bug.<commit_after>from os import path, listdir
from datetime import datetime

from mercurial import ui, hg, util
from mercurial.node import short
import git

from models import Repo, Commit

def find_git_repos(basepath, baseurl=''):
    """Returns a list of hg Repos in basepath"""
    def _find_repos(_path, name):
        if not path.isdir(_path):
            return []
        ret = []
        subdir = listdir(_path)
        if '.git' not in subdir and not _path[-4:] == '.git':
            for r in subdir:
                reponame =  '{0}/{1}'.format(name, r) if name else r
                ret.extend(_find_repos(path.join(_path, r), reponame))
        else:
            ret.append(Repo(
                type='git',
                name=name,
                basepath=basepath,
                baseurl=baseurl,
                commits=[],
            ))
        return ret
    return _find_repos(basepath, '')

def find_git_commits(repo, from_date, until_date):
    """Initializes repo.commits with all commits between the given dates"""
    if from_date > until_date:
        # this should be impossible, maybe we should
        # raise an exception to indicate missuse
        return []

    #TODO: optimize
    gitrepo = git.Repo(repo.path)
    commits = [c for c in gitrepo.iter_commits()]

    # helper function to get branch from git.Commit
    branch = lambda c: gitrepo.git.branch(contains=c.hexsha).split('\n')[0][2:]
    # helper function to get datetime from git.Commit
    when = lambda c: datetime.fromtimestamp(c.committed_date)
    # helper function to get Commit from git.Commit
    com = lambda c: Commit(
        id=c.hexsha,
        repo=repo,
        urlpattern='?p={repo.name};a=commit;h={id}',
        user=c.author.name,
        date=when(c),
        #XXX: in git a commit may be in more than one branch
        # we are choosing the first we find it in.
        branch=branch(c),
        #XXX: I don't exactly know why we need to decode
        message=c.summary.decode('utf-8'),
    )
    # filter on date
    repo.commits.extend([com(c) for c in commits if from_date < when(c) < until_date])

<|endoftext|>


<commit_before>import os
import time
import logging
import socketserver
import socket
import threading
from google.protobuf.message import (
    Message
)
from .msgs.packet_pb2 import Packet
from .msgs.publish_pb2 import Publish
from .msgs.publishers_pb2 import Publishers
from .msgs.subscribe_pb2 import Subscribe
from .msgs.time_pb2 import Time
from .msgs.gz_string_pb2 import GzString
from .msgs.gz_string_v_pb2 import GzString_V
from .gz_msgs.bool_pb2 import Bool
from .gz_msgs.float64_pb2 import Float64

logger = logging.getLogger(__name__)

class Connection:
    HEADER_SIZE = 8
    def __init__(self):
        self.socket = None
        self.server = None

    def connect(self, host: str, port: int):
        logger.info("raw connect %s:%s", host, port)
        self.host = host
        self.port = port
        self.socket = socket.create_connection((host, port))
        local_host, local_port = self.socket.getsockname()
        remote_host, remote_port = self.socket.getpeername()
        logger.info("   connect %s:%s -> %s:%s", local_host, local_port, remote_host, remote_port)
        self._make_streams()

    def connect_and_wait(self, host: str, port: int):
        self.host = host
        self.port = port
        while True:
            try:
                self.socket = socket.create_connection((host, port))
                break
            except socket.timeout as ex:
                logger.warning("Cannot connect, retrying in five seconds.", exc_info=1)
        self._make_streams()

    def _make_streams(self):
        self.rfile = self.socket.makefile('b')
        self.wfile = self.socket.makefile('wb')

    def close(self):
        if self.socket is not None:
            self.socket.close()
            self.socket = None
        if self.server is not None:
            self.server.shutdown()
            self.server = None

    def serve(self, callback):
        class Handler(socketserver.StreamRequestHandler):
            def __init__(hself):
                hself.callback = callback

            def handle(self):
                conn = Connection()
                conn.rfile = hself.rfile
                conn.wfile = hself.wfile
                hself.callback(conn)

        def _serve():
            self.server = socketserver.TCPServer(("", 0), Handler)
            x = self.host, self.port = self.server.server_address
            logger.debug("listening on %s:%s" % x)
            self.server.serve_forever()
        thread = threading.Thread(target=_serve)
        thread.start()

    def raw_read(self) -> bytes:
        buff = self.rfile.read(Connection.HEADER_SIZE)
        if len(buff) != 8:
            logger.error("Only read %d bytes instead of 8 for header." % (len(buff),))
            return None
        size = int(buff, 16)
        data = self.rfile.read(size)
        return data

    def read(self) -> Packet:
        a = self.raw_read()
        if a is None:
            return None
        p = Packet()
        p.ParseFromString(a)
        return p

    def write(self, message: Message):
        _bytes = message.SerializeToString()
        self.wfile.write(_bytes)

    def write_packet(self, name: str, message: Message):
        ms = current_time_millis()
        pack = Packet()
        pack.stamp.sec = ms // 1000
        pack.stamp.nsec = (ms % 1000) * 1000
        pack.type = name
        pack.serialized_data = message.SerializeToString()
        self.write(pack)

def get_type_name(message: Message) -> str:
        return message.DESCRIPTOR.full_name

class Publisher(Message):
    def __init__(self, topic, msg_type, host, port):
        self.topic = topic
        self.msg_type = msg_type
        self.host = host
        self.port = port
        self.listeners = []
        self.latching = False
        self.last_msg = None

    def publish(self, msg: Message):
        self.last_msg = msg
        to_remove = []
        for listener in self.listeners:
            try:
                listener.write(msg)
            except Exception:
                # todo: refine exception type
                to_remove.append(listener)

        for listener in to_remove:
            logger.info("Removing listener from topic %s" % (self.topic,))
            try:
                listener.close()
            except Exception:
                # todo: refine exception type
                # boo hoo
                pass
            self.listeners.remove(listener)

    def connect(self, conn: Connection):
        logger.info("Handling subscriber connection for topic: %s" % (self.topic,))
        if self.latching and self.last_msg is not None:
            try:
                conn.write(self.last_msg)
            except Exception:
                # todo: refine exception type
                logger.warning("Writing latched message failed on topic %s" % (self.topic,))
                try:
                    conn.close()
                except Exception:
                    # todo: refine exception type
                    # boo hoo
                    pass
            return

        self.listeners.append(conn)
        self.notify_all()

    def wait_for_connection(timeout_millis: int) -> bool:
        start = current_time_millis()
        while not self.listeners:
            remain = timeout_millis - (current_time_millis() - start)
            if remain <= 0:
                break
            this.wait(remain)

        return bool(self.listeners)

def current_time_millis():
    return int(round(time.time() * 1000))

class RemotePublisherRecord:
    def __init__(self, pub: Publish):
        self.pub  = pub

    @property
    def topic(self) -> str:
        return self.pub.topic

    @property
    def host(self) -> str:
        return self.pub.host

    @property
    def port(self) -> int:
        return self.pub.port

    @property
    def msg_type(self) -> int:
        return self.pub.msg_type

    def __str__(self):
        return "%s (%s) %s:%s" % (self.topic, self.msg_type, self.host, self.port)

    def connect(connection: Connection):
        # not implemented in origin for some reason
        raise Exception("Ha! Psych!")

class Subscriber(Message):
    def __init__(self, topic, msg_type, callback, message_class, host, port):
        self.topic = topic
        self.msg_type = msg_type
        self.host = host
        self.port = port
        self.callback = callback
        self.message_class = message_class
        self.connections = []

    def connect(self, pub):
        def _run():
            self.handle_connect(pub)
        thread = threading.Thread(target=_run)
        thread.start()

    def handle_connect(self, pub):
        logger.info("CONN for %s from %s:%s", self.topic, self.host, self.port)
        conn = Connection()
        try:
            conn.connect(pub.host, pub.port)
            self.connections.append(conn)
            sub = Subscribe()
            sub.topic = self.topic
            sub.msg_type = self.msg_type
            sub.host = self.host
            sub.port = self.port
            sub.latching = False
            conn.write_packet("sub", sub)

            while True:
                data = conn.raw_read()
                if data is None:
                    self.connections.remove(conn)
                    return
                import pdb
                pdb.set_trace()
                msg = self.message_class()
                msg.ParseFromString(data)
                self.callback(msg)
        except Exception:
            # todo: refine exception type
            try:
                conn.close()
            except Exception:
                # todo: refine exception type
                pass
            logger.exception("exception in handle_connect")

class Node:
    def __init__(self, name):
        self.name = name
        self.server = Connection()
        self.master = Connection()
        self.publishers = {}
        self.subscriptions = {}
        self.namespaces = []

    def wait_for_connection(self, host: str, port: int):
        def _handle(conn: Connection):
            self.handle(conn)

        self.server.serve(_handle)
        self.master.connect_and_wait(host, port)
        self.initialize_connection()
        def _run():
            self.run()
        thread = threading.Thread(target=_run)
        thread.start()

    def advertise(self, topic: str, message_class: Message) -> Publisher:
        topic = self._fix_topic(topic)
        logger.info("ADV %s" % (topic,))
        msg_type = get_type_name(message_class)
        pub = Publisher(topic, msg_type, self.server.host, self.server.port)
        self.publishers[topic] = pub

        # todo: find publish.proto, figure out the real way to construct one
        req = Publish()
        req.topic = topic
        req.msg_type = msg_type
        req.host = self.server.host
        req.port = self.server.port

        try:
            self.master.write_packet("advertise", req)
        except Exception as ex:
            # todo: refine exception type
            # todo todo: wait for original authors to think of something more
            logger.exception("something bad")
        return pub

    def subscribe(self, topic: str, message_class: Message, callback: "(Message) -> None"):
        topic = self._fix_topic(topic)
        logger.info("SUB %s", topic)
        if topic in self.subscriptions:
            raise Exception("Multiple subscriptions for %s" % (topic,))

        msg_type = get_type_name(message_class)

        req = Subscribe()
        req.topic = topic
        req.msg_type = msg_type
        req.host = self.server.host
        req.port = self.server.port
        req.latching = False

        try:
            self.master.write_packet("subscribe", req)
        except Exception as ex:
            # todo: refine exception type
            # todo todo: wait for original authors to think of something more
            logger.exception("something bad")

        sub = Subscriber(topic, msg_type, callback, message_class, self.server.host, self.server.port)
        self.subscriptions[topic] = sub
        # waaat? the topic is the dict's key, why would you iterate through everything?
        # unless its possible for the topic to change after its been added to the dict..
        for pub in self.publishers.values():
            if pub.topic == topic:
                sub.connect(pub)
        return sub

    def run(self):
        try:
            while True:
                packet = self.master.read()
                if packet is None:
                    logger.critical("Received null packet, shutting down connection to master.")
                    self.master.close()
                    return
                self.process_packet(packet)
        except Exception:
            # todo: refine exception type
            logger.exception("I/O Error (?)")

    def initialize_connection(self):
        init_data = self.master.read()
        if init_data.type != "version_init":
            raise Exception("Expected 'version_init' packet, got '%s'." % (init_data.type,))
        # todo: figure out the actual api for
        version = GzString()
        version.ParseFromString(init_data.serialized_data)
        logger.info("Version: %s", version.data)

        namespace_data = self.master.read()
        ns = GzString_V()
        ns.ParseFromString(namespace_data.serialized_data)
        self.namespaces.extend(ns.data)
        logger.info(str(self.namespaces))

        publisher_data = self.master.read()
        if publisher_data.type == "publishers_init":
            pubs = Publishers()
            pubs.ParseFromString(publisher_data.serialized_data)
            for pub in pubs.publisher:
                record = RemotePublisherRecord(pub)
                self.publishers[record.topic] = record
            for pub in self.publishers:
                logger.info(self.publishers[pub])
        else:
            logger.error("No publisher data received.")

    def process_packet(self, packet):
        if packet.type == "publisher_add":
            pub1 = Publish()
            pub1.ParseFromString(packet.serialized_data)
            pub = RemotePublisherRecord(pub1)

            if pub.host == self.server.host and pub.port == self.server.port:
                logger.info("ACK %s", pub.topic)
                return

            logger.info("New Publisher: %s", pub.topic)
            logger.info("Publisher: %s", pub1)
            self.publishers[pub.topic] = pub
        elif packet.type in ["publisher_subscribe", "publisher_advertise"]:
            pub1 = Publish()
            pub1.ParseFromString(packet.serialized_data)
            pub = RemotePublisherRecord(pub1)

            if pub.host == self.server.host and pub.port == self.server.port:
                logger.info("Ignoring subscription request on (local) %s", pub.topic)
                return

            logger.info("PUBSUB found for: %s", pub.topic)
            logger.info("Publisher: %s", pub1)
            self.subscriptions[pub.topic].connect(pub)
        elif packet.type == "topic_namespace_add":
            ns = GzString()
            ns.ParseFromString(packet.serialized_data)
            self.namespaces.append(ns.data)
            logger.info("New Namespace: %s", pub.topic)
        elif packet.type == "unsubscribe":
            sub = Subscribe()
            sub.ParseFromString(packet.serialized_data)
            logger.warning("Ignoring unsubscribe: %s", sub);
        else:
            logger.warning("Can't handle %s", packet.type)

    def handle(self, conn: Connection):
        import pdb
        pdb.set_trace()
        logger.info("Handling new connection")
        msg = conn.read()
        if msg is None:
            logger.warning("Didst read null messaged.")
            return

        if msg.type == "sub":
            sub = Subscribe()
            sub.ParseFromString(msg.serialized_data)
            if sub.topic not in self.publishers:
                logger.error("Subscription for unknown topic %s", sub.topic)
                return

            logger.info("New connection for topic %s", sub.topic)

            pub = self.publishers[sub.topic]

            if pub.msg_type != sub.msg_type:
                logger.error("Message type mismatch requested=%d publishing=%s", pub.msg_type, sub.msg_type)
                return

            logger.info("CONN %s", sub.topic)

            pub.connect(conn)
        else:
            logger.warning("Unknown message type: %s", msg.type)

    def _fix_topic(self, topic: str):
        return "/gazebo/%s/%s" % (self.name, topic)

<commit_msg>apparently i suck at socket programming<commit_after>import os
import time
import logging
import socketserver
import socket
import threading
from google.protobuf.message import (
    Message
)
from .msgs.packet_pb2 import Packet
from .msgs.publish_pb2 import Publish
from .msgs.publishers_pb2 import Publishers
from .msgs.subscribe_pb2 import Subscribe
from .msgs.time_pb2 import Time
from .msgs.gz_string_pb2 import GzString
from .msgs.gz_string_v_pb2 import GzString_V
from .gz_msgs.bool_pb2 import Bool
from .gz_msgs.float64_pb2 import Float64

logger = logging.getLogger(__name__)

class Connection:
    HEADER_SIZE = 8
    def __init__(self):
        self.socket = None
        self.server = None

    def connect(self, host: str, port: int):
        logger.info("raw connect %s:%s", host, port)
        self.host = host
        self.port = port
        try:
            self.socket = socket.create_connection((host, port))
        except ConnectionRefusedError:
            logger.exception("connection refused %s:%s", host, port)
            raise
        local_host, local_port = self.socket.getsockname()
        remote_host, remote_port = self.socket.getpeername()
        logger.info("   connect %s:%s -> %s:%s", local_host, local_port, remote_host, remote_port)
        self._make_streams()

    def connect_and_wait(self, host: str, port: int):
        self.host = host
        self.port = port
        while True:
            try:
                self.socket = socket.create_connection((host, port))
                break
            except ConnectionRefusedError as ex:
                logger.warning("Cannot connect, retrying in five seconds.")
                time.sleep(5)
        self._make_streams()

    def _make_streams(self):
        #self.rfile = self.socket.makefile('rb')
        #self.wfile = self.socket.makefile('wb')
        pass

    def close(self):
        if self.socket is not None:
            self.socket.close()
            self.socket = None
        if self.server is not None:
            self.server.shutdown()
            self.server = None

    def serve(self, callback):
        class Handler(socketserver.StreamRequestHandler):
            def __init__(hself):
                hself.callback = callback

            def handle(hself):
                import pdb
                pdb.set_trace()
                conn = Connection()
                conn.rfile = hself.rfile
                conn.wfile = hself.wfile
                hself.callback(conn)

        def _serve():
            try:
                self.server = socketserver.TCPServer(("", 0), Handler)
                x = self.host, self.port = self.server.server_address
                logger.debug("listening on %s:%s" % x)
                self.server.serve_forever()
            except:
                logger.exception("server thread failed")
        thread = threading.Thread(target=_serve)
        thread.start()

    def _raw_read(self, n) -> bytes:
        buf = bytearray(n)
        buf_view = memoryview(buf)
        while n:
            recvd = self.socket.recv_into(buf_view, n)
            if recvd == 0:
                return None
            buf_view = buf_view[recvd:]
            n -= recvd
        return bytes(buf)

    def raw_read(self) -> bytes:
        buff = self._raw_read(Connection.HEADER_SIZE)
        if len(buff) != 8:
            logger.error("Only read %d bytes instead of 8 for header." % (len(buff),))
            return None
        size = int(buff, 16)
        data = self._raw_read(size)
        return data

    def read(self) -> Packet:
        a = self.raw_read()
        if a is None:
            return None
        p = Packet()
        p.ParseFromString(a)
        return p

    def _raw_write(self, buf):
        n = len(buf)
        sumsent = 0
        while sumsent < n:
            sent = self.socket.send(buf[sumsent:])
            if sent == 0:
                raise Exception("socket connection broken")
            sumsent += sent

    def write(self, message: Message):
        _bytes = message.SerializeToString()
        logger.info("sending: %r", _bytes)
        self._raw_write(_bytes)

    def write_packet(self, name: str, message: Message):
        ms = current_time_millis()
        pack = Packet()
        pack.stamp.sec = ms // 1000
        pack.stamp.nsec = (ms % 1000) * 1000
        pack.type = name
        pack.serialized_data = message.SerializeToString()
        self.write(pack)

def get_type_name(message: Message) -> str:
        return message.DESCRIPTOR.full_name

class Publisher(Message):
    def __init__(self, topic, msg_type, host, port):
        self.topic = topic
        self.msg_type = msg_type
        self.host = host
        self.port = port
        self.listeners = []
        self.latching = False
        self.last_msg = None

    def publish(self, msg: Message):
        self.last_msg = msg
        to_remove = []
        for listener in self.listeners:
            try:
                listener.write(msg)
            except Exception:
                # todo: refine exception type
                to_remove.append(listener)

        for listener in to_remove:
            logger.info("Removing listener from topic %s" % (self.topic,))
            try:
                listener.close()
            except Exception:
                # todo: refine exception type
                # boo hoo
                pass
            self.listeners.remove(listener)

    def connect(self, conn: Connection):
        logger.info("Handling subscriber connection for topic: %s" % (self.topic,))
        if self.latching and self.last_msg is not None:
            try:
                conn.write(self.last_msg)
            except Exception:
                # todo: refine exception type
                logger.warning("Writing latched message failed on topic %s" % (self.topic,))
                try:
                    conn.close()
                except Exception:
                    # todo: refine exception type
                    # boo hoo
                    pass
            return

        self.listeners.append(conn)
        self.notify_all()

    def wait_for_connection(timeout_millis: int) -> bool:
        start = current_time_millis()
        while not self.listeners:
            remain = timeout_millis - (current_time_millis() - start)
            if remain <= 0:
                break
            this.wait(remain)

        return bool(self.listeners)

def current_time_millis():
    return int(round(time.time() * 1000))

class RemotePublisherRecord:
    def __init__(self, pub: Publish):
        self.pub  = pub

    @property
    def topic(self) -> str:
        return self.pub.topic

    @property
    def host(self) -> str:
        return self.pub.host

    @property
    def port(self) -> int:
        return self.pub.port

    @property
    def msg_type(self) -> int:
        return self.pub.msg_type

    def __str__(self):
        return "%s (%s) %s:%s" % (self.topic, self.msg_type, self.host, self.port)

    def connect(connection: Connection):
        # not implemented in origin for some reason
        raise Exception("Ha! Psych!")

class Subscriber(Message):
    def __init__(self, topic, msg_type, callback, message_class, host, port):
        self.topic = topic
        self.msg_type = msg_type
        self.local_host = host
        self.local_port = port
        self.callback = callback
        self.message_class = message_class
        self.connections = []

    def connect(self, pub):
        def _run():
            self.handle_connect(pub)
        thread = threading.Thread(target=_run)
        thread.start()

    def handle_connect(self, pub):
        logger.info(
            "CONN for %s from %s:%s",
            self.topic, pub.host, pub.port)
        conn = Connection()
        try:
            conn.connect(pub.host, pub.port)
            self.connections.append(conn)
            sub = Subscribe()
            sub.topic = self.topic
            sub.msg_type = self.msg_type
            sub.host = pub.host
            sub.port = pub.port
            sub.latching = False
            logger.info("hey! talk to me! %s", sub)
            conn.write_packet("sub", sub)

            while True:
                data = conn.raw_read()
                if data is None:
                    self.connections.remove(conn)
                    return
                import pdb
                pdb.set_trace()
                msg = self.message_class()
                msg.ParseFromString(data)
                self.callback(msg)
        except Exception:
            # todo: refine exception type
            try:
                conn.close()
            except Exception:
                # todo: refine exception type
                pass
            logger.exception("exception in handle_connect")

class Node:
    def __init__(self, name):
        self.name = name
        self.server = Connection()
        self.master = Connection()
        self.publishers = {}
        self.subscriptions = {}
        self.namespaces = []

    def wait_for_connection(self, host: str, port: int):
        def _handle(conn: Connection):
            self.handle(conn)

        self.server.serve(_handle)
        self.master.connect_and_wait(host, port)
        self.initialize_connection()
        thread = threading.Thread(target=self.run)
        thread.start()

    def advertise(self, topic: str, message_class: Message) -> Publisher:
        topic = self._fix_topic(topic)
        logger.info("ADV %s" % (topic,))
        msg_type = get_type_name(message_class)
        pub = Publisher(topic, msg_type, self.server.host, self.server.port)
        self.publishers[topic] = pub

        # todo: find publish.proto, figure out the real way to construct one
        req = Publish()
        req.topic = topic
        req.msg_type = msg_type
        req.host = self.server.host
        req.port = self.server.port

        try:
            self.master.write_packet("advertise", req)
        except Exception as ex:
            # todo: refine exception type
            # todo todo: wait for original authors to think of something more
            logger.exception("something bad")
        return pub

    def subscribe(self, topic: str, message_class: Message, callback: "(Message) -> None"):
        topic = self._fix_topic(topic)
        logger.info("SUB %s", topic)
        if topic in self.subscriptions:
            raise Exception("Multiple subscriptions for %s" % (topic,))

        msg_type = get_type_name(message_class)

        req = Subscribe()
        req.topic = topic
        req.msg_type = msg_type
        req.host = self.server.host
        req.port = self.server.port
        req.latching = False

        try:
            self.master.write_packet("subscribe", req)
        except Exception as ex:
            # todo: refine exception type
            # todo todo: wait for original authors to think of something more
            logger.exception("something bad")

        sub = Subscriber(topic, msg_type, callback, message_class, self.server.host, self.server.port)
        self.subscriptions[topic] = sub
        # waaat? the topic is the dict's key, why would you iterate through everything?
        # unless its possible for the topic to change after its been added to the dict..
        for pub in self.publishers.values():
            if pub.topic == topic:
                sub.connect(pub)
        return sub

    def run(self):
        try:
            while True:
                packet = self.master.read()
                if packet is None:
                    logger.critical("Received null packet, shutting down connection to master.")
                    self.master.close()
                    return
                self.process_packet(packet)
        except Exception:
            # todo: refine exception type
            logger.exception("I/O Error (?)")

    def initialize_connection(self):
        init_data = self.master.read()
        if init_data.type != "version_init":
            raise Exception("Expected 'version_init' packet, got '%s'." % (init_data.type,))
        # todo: figure out the actual api for
        version = GzString()
        version.ParseFromString(init_data.serialized_data)
        logger.info("Version: %s", version.data)

        namespace_data = self.master.read()
        ns = GzString_V()
        ns.ParseFromString(namespace_data.serialized_data)
        self.namespaces.extend(ns.data)
        logger.info(str(self.namespaces))

        publisher_data = self.master.read()
        if publisher_data.type == "publishers_init":
            pubs = Publishers()
            pubs.ParseFromString(publisher_data.serialized_data)
            for pub in pubs.publisher:
                record = RemotePublisherRecord(pub)
                self.publishers[record.topic] = record
            for i, pub in enumerate(self.publishers):
                logger.info("init - publisher[%d]: %s", i, self.publishers[pub])
        else:
            logger.error("No publisher data received.")

    def process_packet(self, packet):
        if packet.type == "publisher_add":
            pub1 = Publish()
            pub1.ParseFromString(packet.serialized_data)
            pub = RemotePublisherRecord(pub1)

            if pub.host == self.server.host and pub.port == self.server.port:
                logger.info("ACK %s", pub.topic)
                return

            logger.info("New Publisher: %s", pub.topic)
            logger.info("Publisher: %s", pub1)
            self.publishers[pub.topic] = pub
        elif packet.type in ["publisher_subscribe", "publisher_advertise"]:
            pub1 = Publish()
            pub1.ParseFromString(packet.serialized_data)
            pub = RemotePublisherRecord(pub1)

            if pub.host == self.server.host and pub.port == self.server.port:
                logger.info("Ignoring subscription request on (local) %s", pub.topic)
                return

            logger.info("PUBSUB found for: %s", pub.topic)
            logger.info("Publisher: %s", pub1)
            self.subscriptions[pub.topic].connect(pub)
        elif packet.type == "topic_namespace_add":
            ns = GzString()
            ns.ParseFromString(packet.serialized_data)
            self.namespaces.append(ns.data)
            logger.info("New Namespace: %s", ns.data)
        elif packet.type == "unsubscribe":
            sub = Subscribe()
            sub.ParseFromString(packet.serialized_data)
            logger.warning("Ignoring unsubscribe: %s", sub);
        else:
            logger.warning("Can't handle %s", packet.type)

    def handle(self, conn: Connection):
        import pdb
        pdb.set_trace()
        logger.info("Handling new connection")
        msg = conn.read()
        if msg is None:
            logger.warning("Didst read null messaged.")
            return

        if msg.type == "sub":
            sub = Subscribe()
            sub.ParseFromString(msg.serialized_data)
            if sub.topic not in self.publishers:
                logger.error("Subscription for unknown topic %s", sub.topic)
                return

            logger.info("New connection for topic %s", sub.topic)

            pub = self.publishers[sub.topic]

            if pub.msg_type != sub.msg_type:
                logger.error("Message type mismatch requested=%d publishing=%s", pub.msg_type, sub.msg_type)
                return

            logger.info("CONN %s", sub.topic)

            pub.connect(conn)
        else:
            logger.warning("Unknown message type: %s", msg.type)

    def _fix_topic(self, topic: str):
        return "/gazebo/%s/%s" % (self.name, topic)

<|endoftext|>


<commit_before>from datetime import datetime

from openpyxl import Workbook

from .config import ConfigDefaults
from .utils import format_time

logger_version = "1.0"

logs = []

def save_logs():
    global logs
    if logs is None or len(logs) < 1:
        return

    file_loc = ConfigDefaults.log_file + datetime.now().date().isoformat() + ".txt"
    log_str = ""

    for l in logs:
        log_str += "[{}:{}] {}\n".format(l[0].hour, l[0].minute, l[1])

    with open(file_loc, "a+") as f:
        f.write(log_str)

    logs = []

def log(msg="\n", *args, **kwargs):
    s_msg = str(msg)
    logs.append((datetime.now(), msg))
    print(msg)
    save_logs()

class OnlineLogger:

    def __init__(self, musicbot):
        self.member_data = {}
        self.action_phases = {}
        self.ongoing_online_phases = {}
        self.ongoing_playing_phases = {}
        self.musicbot = musicbot

    def create_output(self):
        alphabet = list("abcdefghijklmnopqrstuvwxyz")

        def index_to_alphabet(ind):
            if ind < len(alphabet):
                return alphabet[ind].upper()

            remainder = ind % len(alphabet)
            return index_to_alphabet(ind - remainder) + alphabet[remainder].upper()

        wb = Workbook()
        all_phases = {}
        for mem in self.action_phases:
            phases = all_phases.get(mem, None)
            if phases is None:
                all_phases[mem] = self.action_phases[mem]
            else:
                all_phases[mem].extend(self.action_phases[mem])
        for mem in self.ongoing_online_phases:
            phases = all_phases.get(mem, None)
            if phases is None:
                all_phases[mem] = self.ongoing_online_phases[mem]
            else:
                all_phases[mem].extend(self.ongoing_online_phases[mem])
        for mem in self.ongoing_playing_phases:
            phases = all_phases.get(mem, None)
            if phases is None:
                all_phases[mem] = self.ongoing_playing_phases[mem]
            else:
                all_phases[mem].extend(self.ongoing_playing_phases[mem])

        for member in all_phases:
            ws = wb.create_sheet(self.musicbot.get_global_user(member).name)
            for dimension in ws.column_dimensions.values():
                dimension.auto_size = True
            index = 1
            for action in all_phases[member]:
                ws["A{}".format(index)] = action.type_string
                ws["B{}".format(index)] = action.detailed_string
                ws["C{}".format(index)] = action.start_string
                ws["D{}".format(index)] = action.end_string
                ws["E{}".format(index)] = action.duration_string
                index += 1

        wb.save("cache/last_survey_data.xlsx")

    def reset(self):
        self.action_phases = {}

    def update_stats(self, user_id, is_online, game_playing):
        # print("looking at " + self.musicbot.get_global_user(user_id).name)
        user_data = self.get_user_data(user_id)
        last_online_phase = self.get_last_online_phase(user_id)
        last_playing_phase = self.get_last_playing_phase(user_id)

        if is_online and not user_data.is_online:
            # came online
            self.push_ongoing_online_phase(
                user_id, OnlinePhase(datetime.now()))
            # print("  -came online")

        if not is_online and user_data.is_online:
            # went offline
            if last_online_phase is not None:
                last_online_phase.set_end(datetime.now())
                self.push_action_phase(user_id, last_online_phase)
            # print("  -went offline")

        if game_playing is not None and user_data.game_playing is None:
            # started playing
            self.push_ongoing_playing_phase(
                user_id, PlayingPhase(game_playing, datetime.now()))
            # print("  -started playing " + game_playing.name)

        if game_playing is None and user_data.game_playing is not None:
            # stopped playing
            if last_playing_phase is not None:
                last_playing_phase.set_end(datetime.now())
                self.push_action_phase(user_id, last_playing_phase)
            # print("  -stopped playing " + game_playing.name)

        self.member_data[user_id] = MemberStaus(is_online, game_playing)

    def get_user_data(self, user_id):
        user_data = self.member_data.get(user_id, None)

        if user_data is None:
            return MemberStaus()

        return user_data

    def push_action_phase(self, user_id, action_phase):
        phases = self.action_phases.get(user_id, None)
        if phases is None:
            self.action_phases[user_id] = [action_phase, ]
        else:
            self.action_phases[user_id].append(action_phase)

        try:
            self.ongoing_online_phases[user_id].pop(action_phase)
        except:
            print("Couldn't remove action phase from ongoing phases")
            return

        try:
            self.ongoing_playing_phases[user_id].pop(action_phase)
        except:
            print("Couldn't remove action phase from ongoing playing phases")

    def push_ongoing_online_phase(self, user_id, phase):
        phases = self.ongoing_online_phases.get(user_id, None)
        if phases is None:
            self.ongoing_online_phases[user_id] = [phase, ]
        else:
            self.ongoing_online_phases[user_id].append(phase)

    def push_ongoing_playing_phase(self, user_id, phase):
        phases = self.ongoing_playing_phases.get(user_id, None)
        if phases is None:
            self.ongoing_playing_phases[user_id] = [phase, ]
        else:
            self.ongoing_playing_phases[user_id].append(phase)

    def get_last_online_phase(self, user_id):
        online_phases = self.ongoing_online_phases.get(user_id, None)
        if online_phases is None or len(online_phases) < 1:
            return None

        return online_phases[-1]

    def get_last_playing_phase(self, user_id):
        playing_phases = self.ongoing_playing_phases.get(user_id, None)
        if playing_phases is None or len(playing_phases) < 1:
            return None

        return playing_phases[-1]

class MemberStaus:

    def __init__(self, is_online=False, game_playing=None):
        self.is_online = is_online
        self.game_playing = game_playing

class PlayingPhase:

    def __init__(self, game, start, end=None):
        self.game = game
        self.start = start
        self.end = end

    def set_end(self, end):
        self.end = end

    def __str__(self):
        if self.end is None:
            return "Started \"{0}\" at {1.month:0>2}-{1.day:0>2} {1.hour:0>2}:{1.minute:0>2}".format(self.game.name, self.start)

        return "Played \"{0}\" from {1.month:0>2}-{1.day:0>2} {1.hour:0>2}:{1.minute:0>2} to {2.month:0>2}-{2.day:0>2} {2.hour:0>2}:{2.minute:0>2}".format(self.game.name, self.start, self.end)

    @property
    def duration_string(self):
        return format_time((self.end - self.start).total_seconds(), round_seconds=True, max_specifications=3) if self.end is not None else "Ongoing"

    @property
    def type_string(self):
        return "PLAYING"

    @property
    def detailed_string(self):
        return self.game.name

    @property
    def start_string(self):
        return "{0.year:0>4}/{0.month:0>2}/{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.start)

    @property
    def end_string(self):
        return "{0.year:0>4}/{0.month:0>2}/{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.end) if self.end is not None else "Until now"

class OnlinePhase:

    def __init__(self, start, end=None):
        self.start = start
        self.end = end

    def set_end(self, end):
        self.end = end

    def __str__(self):
        if self.end is None:
            return "Came online at {0.month:0>2}-{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.start)

        return "Was online from {0.month:0>2}-{0.day:0>2} {0.hour:0>2}:{0.minute:0>2} to {1.month:0>2}-{1.day:0>2} {1.hour:0>2}:{1.minute:0>2}".format(self.start, self.end)

    @property
    def duration_string(self):
        return format_time((self.end - self.start).total_seconds(), round_seconds=True, max_specifications=3) if self.end is not None else "Ongoing"

    @property
    def type_string(self):
        return "ONLINE"

    @property
    def detailed_string(self):
        return ""

    @property
    def start_string(self):
        return "{0.year:0>4}/{0.month:0>2}/{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.start)

    @property
    def end_string(self):
        return "{0.year:0>4}/{0.month:0>2}/{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.end) if self.end is not None else "Until now"
<commit_msg>made this less stupid xD<commit_after>from datetime import datetime

from openpyxl import Workbook

from .config import ConfigDefaults
from .utils import format_time

logger_version = "1.0"

logs = []

def save_logs():
    global logs
    if logs is None or len(logs) < 1:
        return

    file_loc = ConfigDefaults.log_file + datetime.now().date().isoformat() + ".txt"
    log_str = ""

    for l in logs:
        log_str += "[{}:{}] {}\n".format(l[0].hour, l[0].minute, l[1])

    with open(file_loc, "a+") as f:
        f.write(log_str)

    logs = []

def log(msg="\n", *args, **kwargs):
    s_msg = str(msg)
    logs.append((datetime.now(), msg))
    print(msg)
    save_logs()

class OnlineLogger:

    def __init__(self, musicbot):
        self.member_data = {}
        self.ongoing_online_phases = {}
        self.ongoing_playing_phases = {}
        self.musicbot = musicbot

    def create_output(self):
        alphabet = list("abcdefghijklmnopqrstuvwxyz")

        def index_to_alphabet(ind):
            if ind < len(alphabet):
                return alphabet[ind].upper()

            remainder = ind % len(alphabet)
            return index_to_alphabet(ind - remainder) + alphabet[remainder].upper()

        wb = Workbook()
        all_phases = {}
        # for mem in self.action_phases:
        #     phases = all_phases.get(mem, None)
        #     if phases is None:
        #         all_phases[mem] = self.action_phases[mem]
        #     else:
        #         all_phases[mem].extend(self.action_phases[mem])
        for mem in self.ongoing_online_phases:
            phases = all_phases.get(mem, None)
            if phases is None:
                all_phases[mem] = self.ongoing_online_phases[mem]
            else:
                all_phases[mem].extend(self.ongoing_online_phases[mem])
        for mem in self.ongoing_playing_phases:
            phases = all_phases.get(mem, None)
            if phases is None:
                all_phases[mem] = self.ongoing_playing_phases[mem]
            else:
                all_phases[mem].extend(self.ongoing_playing_phases[mem])

        for member in all_phases:
            ws = wb.create_sheet(self.musicbot.get_global_user(member).name)
            for dimension in ws.column_dimensions.values():
                dimension.auto_size = True
            index = 1
            for action in sorted(all_phases[member], key=lambda phase: phase.start):
                ws["A{}".format(index)] = action.type_string
                ws["B{}".format(index)] = action.detailed_string
                ws["C{}".format(index)] = action.start_string
                ws["D{}".format(index)] = action.end_string
                ws["E{}".format(index)] = action.duration_string
                index += 1

        wb.save("cache/last_survey_data.xlsx")

    def reset(self):
        self.action_phases = {}

    def update_stats(self, user_id, is_online, game_playing):
        # print("looking at " + self.musicbot.get_global_user(user_id).name)
        user_data = self.get_user_data(user_id)
        last_online_phase = self.get_last_online_phase(user_id)
        last_playing_phase = self.get_last_playing_phase(user_id)

        if is_online and not user_data.is_online:
            # came online
            self.push_ongoing_online_phase(
                user_id, OnlinePhase(datetime.now()))
            # print("  -came online")

        if not is_online and user_data.is_online:
            # went offline
            if last_online_phase is not None:
                last_online_phase.set_end(datetime.now())
                # self.push_action_phase(user_id, last_online_phase)
            # print("  -went offline")

        if game_playing is not None and user_data.game_playing is None:
            # started playing
            self.push_ongoing_playing_phase(
                user_id, PlayingPhase(game_playing, datetime.now()))
            # print("  -started playing " + game_playing.name)

        if game_playing is None and user_data.game_playing is not None:
            # stopped playing
            if last_playing_phase is not None:
                last_playing_phase.set_end(datetime.now())
                # self.push_action_phase(user_id, last_playing_phase)
            # print("  -stopped playing " + game_playing.name)

        self.member_data[user_id] = MemberStaus(is_online, game_playing)

    def get_user_data(self, user_id):
        user_data = self.member_data.get(user_id, None)

        if user_data is None:
            return MemberStaus()

        return user_data

    # def push_action_phase(self, user_id, action_phase):
        phases = self.action_phases.get(user_id, None)
        if phases is None:
            self.action_phases[user_id] = [action_phase, ]
        else:
            self.action_phases[user_id].append(action_phase)

        try:
            self.ongoing_online_phases[user_id].pop(action_phase)
        except:
            print("Couldn't remove action phase from ongoing phases")
            return

        try:
            self.ongoing_playing_phases[user_id].pop(action_phase)
        except:
            print("Couldn't remove action phase from ongoing playing phases")

    def push_ongoing_online_phase(self, user_id, phase):
        phases = self.ongoing_online_phases.get(user_id, None)
        if phases is None:
            self.ongoing_online_phases[user_id] = [phase, ]
        else:
            self.ongoing_online_phases[user_id].append(phase)

    def push_ongoing_playing_phase(self, user_id, phase):
        phases = self.ongoing_playing_phases.get(user_id, None)
        if phases is None:
            self.ongoing_playing_phases[user_id] = [phase, ]
        else:
            self.ongoing_playing_phases[user_id].append(phase)

    def get_last_online_phase(self, user_id):
        online_phases = self.ongoing_online_phases.get(user_id, None)
        if online_phases is None or len(online_phases) < 1:
            return None

        return online_phases[-1]

    def get_last_playing_phase(self, user_id):
        playing_phases = self.ongoing_playing_phases.get(user_id, None)
        if playing_phases is None or len(playing_phases) < 1:
            return None

        return playing_phases[-1]

class MemberStaus:

    def __init__(self, is_online=False, game_playing=None):
        self.is_online = is_online
        self.game_playing = game_playing

class PlayingPhase:

    def __init__(self, game, start, end=None):
        self.game = game
        self.start = start
        self.end = end

    def set_end(self, end):
        self.end = end

    def __str__(self):
        if self.end is None:
            return "Started \"{0}\" at {1.month:0>2}-{1.day:0>2} {1.hour:0>2}:{1.minute:0>2}".format(self.game.name, self.start)

        return "Played \"{0}\" from {1.month:0>2}-{1.day:0>2} {1.hour:0>2}:{1.minute:0>2} to {2.month:0>2}-{2.day:0>2} {2.hour:0>2}:{2.minute:0>2}".format(self.game.name, self.start, self.end)

    @property
    def duration_string(self):
        return format_time((self.end - self.start).total_seconds(), round_seconds=True, max_specifications=3) if self.end is not None else "Ongoing"

    @property
    def type_string(self):
        return "PLAYING"

    @property
    def detailed_string(self):
        return self.game.name

    @property
    def start_string(self):
        return "{0.year:0>4}/{0.month:0>2}/{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.start)

    @property
    def end_string(self):
        return "{0.year:0>4}/{0.month:0>2}/{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.end) if self.end is not None else "Until now"

class OnlinePhase:

    def __init__(self, start, end=None):
        self.start = start
        self.end = end

    def set_end(self, end):
        self.end = end

    def __str__(self):
        if self.end is None:
            return "Came online at {0.month:0>2}-{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.start)

        return "Was online from {0.month:0>2}-{0.day:0>2} {0.hour:0>2}:{0.minute:0>2} to {1.month:0>2}-{1.day:0>2} {1.hour:0>2}:{1.minute:0>2}".format(self.start, self.end)

    @property
    def duration_string(self):
        return format_time((self.end - self.start).total_seconds(), round_seconds=True, max_specifications=3) if self.end is not None else "Ongoing"

    @property
    def type_string(self):
        return "ONLINE"

    @property
    def detailed_string(self):
        return ""

    @property
    def start_string(self):
        return "{0.year:0>4}/{0.month:0>2}/{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.start)

    @property
    def end_string(self):
        return "{0.year:0>4}/{0.month:0>2}/{0.day:0>2} {0.hour:0>2}:{0.minute:0>2}".format(self.end) if self.end is not None else "Until now"
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import tensorflow as tf

tf.set_random_seed(777)

class BatchNorm(object):

    def __init__(self, epsilon=1e-6, momentum=0.9, name="batch_norm"):
        with tf.variable_scope(name) as scope:
            self.eps = epsilon
            self.momentum = momentum
            self.name = name

    def __call__(self, x, train=True):
        return tf.layers.batch_normalization(inputs=x,
                                             momentum=self.momentum,
                                             epsilon=self.eps,
                                             scale=True,
                                             training=train)

def conv2d(input_, output_dim, k_h=3, k_w=3, d_h=2, d_w=2, name="Conv2D"):
    with tf.variable_scope(name):
        w = tf.get_variable('weights', shape=[k_h, k_w, input_.get_shape()[-1], output_dim],
                            initializer=tf.contrib.layers.variance_scaling_initializer())
        b = tf.get_variable('biases', [output_dim],
                            initializer=tf.constant_initializer(0.))

        conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME')
        conv = tf.reshape(tf.nn.bias_add(conv, b), conv.get_shape())

        return conv

def deconv2d(input_, output_shape, k_h=3, k_w=3, d_h=2, d_w=2, name="DeConv2D"):
    with tf.variable_scope(name):
        w = tf.get_variable('weights', shape=[k_h, k_w, output_shape[-1], input_.get_shape()[-1]],
                            initializer=tf.contrib.layers.variance_scaling_initializer())
        b = tf.get_variable('biases', [output_shape[-1]],
                            initializer=tf.constant_initializer(0.))

        deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, strides=[1, d_h, d_w, 1])
        deconv = tf.reshape(tf.nn.bias_add(deconv, b), deconv.get_shape())

        return deconv

def linear(input_, output_size, scope=None, bias_start=0.):
    shape = input_.get_shape().as_list()

    with tf.variable_scope(scope or "Linear"):
        matrix = tf.get_variable("matrix", shape=[shape[1], output_size],
                                 initializer=tf.contrib.layers.variance_scaling_initializer())

        bias_term = tf.get_variable("bias", shape=[output_size],
                                    initializer=tf.constant_initializer(bias_start))

        layer = tf.nn.bias_add(tf.matmul(input_, matrix), bias_term)

        return layer

class WGAN:

    def __init__(self, s, batch_size=64, input_height=28, input_width=28, input_channel=1, n_classes=10,
                 sample_num=64, sample_size=8, n_input=784,
                 z_dim=128, gf_dim=64, df_dim=64, epsilon=1e-12,
                 enable_bn=False, enable_selu=False, enable_adam=False, enable_gp=False):

        """
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 32
        :param input_height: input image height, default 28
        :param input_width: input image width, default 28
        :param input_channel: input image channel, default 1 (gray-scale)
        - in case of MNIST, image size is 28x28x1(HWC).
        :param n_classes: input dataset's classes
        - in case of MNIST, 10 (0 ~ 9)

        # Output Settings
        :param sample_num: the number of output images, default 64
        :param sample_size: sample image size, default 8

        # For model
        :param n_input: input image size, default 784(28x28)

        # Training Option
        :param z_dim: z dimension (kinda noise), default 128
        :param gf_dim: the number of generator filters, default 64
        :param df_dim: the number of discriminator filters, default 64
        :param epsilon: epsilon, default 1e-9
        :param enable_bn: enabling batch normalization, default False
        :param enable_selu: enabling selu activation func, default False
        :param enable_adam: enabling adam optimizer, default False
        :param enable_gp: enabling gradient penaly, default False
        """

        self.s = s
        self.batch_size = batch_size
        self.input_height = input_height
        self.input_width = input_width
        self.input_channel = input_channel
        self.n_classes = n_classes

        self.sample_size = sample_size
        self.sample_num = sample_num
        self.n_input = n_input

        self.image_shape = [self.input_height, self.input_width, self.input_channel]

        self.z_dim = z_dim
        self.gf_dim = gf_dim
        self.df_dim = df_dim
        self.eps = epsilon

        # Custom Batch Normalization
        self.d_bn1 = BatchNorm(self.batch_size, name='d_bn1')
        self.d_bn2 = BatchNorm(self.batch_size, name='d_bn2')
        self.d_bn3 = BatchNorm(self.batch_size, name='d_bn3')

        self.g_bn1 = BatchNorm(self.batch_size, name='g_bn1')
        self.g_bn2 = BatchNorm(self.batch_size, name='g_bn2')
        self.g_bn3 = BatchNorm(self.batch_size, name='g_bn3')

        # Placeholders
        self.x = tf.placeholder(tf.float32, shape=[self.batch_size] + self.image_shape, name='x-images')
        self.z = tf.placeholder(tf.float32, shape=[self.batch_size, self.z_dim], name='z-noise')

        # Training Options - based on the WGAN paper
        self.learning_rate = 5e-5  # very slow
        self.critic = 5
        self.clip = 0.01
        self.d_clip = []  # (-0.01 ~ 0.01)
        self.d_lambda = 0.25
        self.decay = 0.90

        self.EnableBN = enable_bn
        self.EnableSELU = enable_selu
        self.EnableAdam = enable_adam
        self.EnableGP = enable_gp

        self.d_op = None
        self.g_op = None

        self.build_wgan()  # build WGAN model

    def discriminator(self, x, reuse=None):
        with tf.variable_scope('discriminator', reuse=reuse):
            h0 = conv2d(x, self.df_dim, name='d_h0_conv')
            if self.EnableBN:
                h0 = self.d_bn1(h0)
            if self.EnableSELU:
                h0 = tf.nn.selu(h0)
            else:
                h0 = tf.nn.leaky_relu(h0, alpha=0.2)

            h1 = conv2d(h0, self.df_dim * 2, name='d_h1_conv')
            if self.EnableBN:
                h1 = self.d_bn2(h1)
            if self.EnableSELU:
                h1 = tf.nn.selu(h1)
            else:
                h1 = tf.nn.leaky_relu(h1, alpha=0.2)

            h2 = conv2d(h1, self.df_dim * 4, name='d_h2_conv')
            if self.EnableBN:
                h2 = self.d_bn3(h2)
            if self.EnableSELU:
                h2 = tf.nn.selu(h2)
            else:
                h2 = tf.nn.leaky_relu(h2, alpha=0.2)

            h3 = linear(tf.reshape(h2, [self.batch_size, -1]), 1, 'd_h3_linear')

            return tf.nn.sigmoid(h3)

    def generator(self, z, reuse=None):
        with tf.variable_scope('generator', reuse=reuse):
            h0 = tf.reshape(linear(z, self.gf_dim * 8 * 4 * 4, 'g_h0_lin'), [-1, 4, 4, self.gf_dim * 8])
            if self.EnableBN:
                h0 = self.d_bn1(h0)
            if self.EnableSELU:
                h0 = tf.nn.selu(h0)
            else:
                h0 = tf.nn.leaky_relu(h0, alpha=0.2)

            h1 = deconv2d(h0, [self.batch_size, 7, 7, self.gf_dim * 4], name='g_h1')
            if self.EnableBN:
                h1 = self.d_bn2(h1)
            if self.EnableSELU:
                h1 = tf.nn.selu(h1)
            else:
                h1 = tf.nn.leaky_relu(h1, alpha=0.2)

            h2 = deconv2d(h1, [self.batch_size, 14, 14, self.gf_dim * 2], name='g_h2')
            if self.EnableBN:
                h2 = self.d_bn3(h2)
            if self.EnableSELU:
                h2 = tf.nn.selu(h2)
            else:
                h2 = tf.nn.leaky_relu(h2, alpha=0.2)

            h3 = deconv2d(h2, [self.batch_size,  # output shape is same as input shape
                               self.input_height, self.input_width, self.input_channel], name='g_h3')

            return tf.nn.tanh(h3)

    def build_wgan(self):
        # Using DCGAN D/G Model

        # Generator
        self.g = self.generator(self.z)

        # Discriminator
        d_real = self.discriminator(self.x)
        d_fake = self.discriminator(self.g, reuse=True)

        # The WGAN losses
        # maximize log(D(G(z)))
        # maximize log(D(x)) + log(1 - D(G(z)))

        log = lambda x: tf.log(x + self.eps)

        self.g_loss = -tf.reduce_mean(log(d_fake))
        d_real_loss = -tf.reduce_mean(log(d_real))
        d_fake_loss = -tf.reduce_mean(log(1. - d_fake))
        self.d_loss = d_real_loss + d_fake_loss

        # The gradient penalty loss
        if self.EnableGP:
            alpha = tf.random_uniform(shape=self.x.get_shape(),
                                      minval=0., maxval=1., name='alpha')
            diff = self.g - self.x  # fake data - real data
            interpolates = self.x + alpha * diff
            d_interp = self.discriminator(interpolates, reuse=True)
            gradients = tf.gradients(d_interp, [interpolates])[0]
            slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients), reduction_indices=[1]))
            gradient_penalty = tf.reduce_mean((slopes - 1.) ** 2)

            # Update D loss
            self.d_loss += self.d_lambda * gradient_penalty

        # Summary
        z_sum = tf.summary.histogram("z", self.z)
        g = tf.reshape(self.g, shape=[-1] + self.image_shape)
        g_sum = tf.summary.image("g", g)  # generated image from G model
        d_real_sum = tf.summary.histogram("d_real", d_real)
        d_fake_sum = tf.summary.histogram("d_fake", d_fake)

        d_real_loss_sum = tf.summary.scalar("d_real_loss", d_real_loss)
        d_fake_loss_sum = tf.summary.scalar("d_fake_loss", d_fake_loss)
        d_loss_sum = tf.summary.scalar("d_loss", self.d_loss)
        g_loss_sum = tf.summary.scalar("g_loss", self.g_loss)

        # Collect trainer values
        vars = tf.trainable_variables()
        d_params = [v for v in vars if v.name.startswith('discriminator')]
        g_params = [v for v in vars if v.name.startswith('generator')]

        self.d_clip = [v.assign(tf.clip_by_value(v, -self.clip, self.clip)) for v in d_params]

        # Model Saver
        self.saver = tf.train.Saver()

        # Optimizer
        if self.EnableAdam:
            self.d_op = tf.train.AdamOptimizer(learning_rate=2e-4,
                                               beta1=0.5).minimize(self.d_loss, var_list=d_params)
            self.g_op = tf.train.AdamOptimizer(learning_rate=2e-4,
                                               beta1=0.5).minimize(self.g_loss, var_list=g_params)
        else:
            self.d_op = tf.train.RMSPropOptimizer(learning_rate=self.learning_rate,
                                                  decay=self.decay).minimize(self.d_loss, var_list=d_params)
            self.g_op = tf.train.RMSPropOptimizer(learning_rate=self.learning_rate,
                                                  decay=self.decay).minimize(self.g_loss, var_list=g_params)

        # Merge summary
        self.g_sum = tf.summary.merge([z_sum, d_fake_sum, g_sum, d_fake_loss_sum, g_loss_sum])
        self.d_sum = tf.summary.merge([z_sum, d_real_sum, d_real_loss_sum, d_loss_sum])
        self.merged = tf.summary.merge_all()
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<commit_msg>change some...<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import tensorflow as tf

tf.set_random_seed(777)

class BatchNorm(object):

    def __init__(self, epsilon=1e-6, momentum=0.9, name="batch_norm"):
        with tf.variable_scope(name) as scope:
            self.eps = epsilon
            self.momentum = momentum
            self.name = name

    def __call__(self, x, train=True):
        return tf.layers.batch_normalization(inputs=x,
                                             momentum=self.momentum,
                                             epsilon=self.eps,
                                             scale=True,
                                             training=train)

def conv2d(input_, output_dim, k_h=3, k_w=3, d_h=2, d_w=2, name="Conv2D"):
    with tf.variable_scope(name):
        w = tf.get_variable('weights', shape=[k_h, k_w, input_.get_shape()[-1], output_dim],
                            initializer=tf.contrib.layers.variance_scaling_initializer())
        b = tf.get_variable('biases', [output_dim],
                            initializer=tf.constant_initializer(0.))

        conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME')
        conv = tf.reshape(tf.nn.bias_add(conv, b), conv.get_shape())

        return conv

def deconv2d(input_, output_shape, k_h=3, k_w=3, d_h=2, d_w=2, name="DeConv2D"):
    with tf.variable_scope(name):
        w = tf.get_variable('weights', shape=[k_h, k_w, output_shape[-1], input_.get_shape()[-1]],
                            initializer=tf.contrib.layers.variance_scaling_initializer())
        b = tf.get_variable('biases', [output_shape[-1]],
                            initializer=tf.constant_initializer(0.))

        deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape, strides=[1, d_h, d_w, 1])
        deconv = tf.reshape(tf.nn.bias_add(deconv, b), deconv.get_shape())

        return deconv

def linear(input_, output_size, scope=None, bias_start=0.):
    shape = input_.get_shape().as_list()

    with tf.variable_scope(scope or "Linear"):
        matrix = tf.get_variable("matrix", shape=[shape[1], output_size],
                                 initializer=tf.contrib.layers.variance_scaling_initializer())

        bias_term = tf.get_variable("bias", shape=[output_size],
                                    initializer=tf.constant_initializer(bias_start))

        layer = tf.nn.bias_add(tf.matmul(input_, matrix), bias_term)

        return layer

class WGAN:

    def __init__(self, s, batch_size=64, input_height=28, input_width=28, input_channel=1, n_classes=10,
                 sample_num=64, sample_size=8, n_input=784,
                 z_dim=128, gf_dim=64, df_dim=64, epsilon=1e-12,
                 enable_bn=False, enable_selu=False, enable_adam=False, enable_gp=False):

        """
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 32
        :param input_height: input image height, default 28
        :param input_width: input image width, default 28
        :param input_channel: input image channel, default 1 (gray-scale)
        - in case of MNIST, image size is 28x28x1(HWC).
        :param n_classes: input dataset's classes
        - in case of MNIST, 10 (0 ~ 9)

        # Output Settings
        :param sample_num: the number of output images, default 64
        :param sample_size: sample image size, default 8

        # For model
        :param n_input: input image size, default 784(28x28)

        # Training Option
        :param z_dim: z dimension (kinda noise), default 128
        :param gf_dim: the number of generator filters, default 64
        :param df_dim: the number of discriminator filters, default 64
        :param epsilon: epsilon, default 1e-9
        :param enable_bn: enabling batch normalization, default False
        :param enable_selu: enabling selu activation func, default False
        :param enable_adam: enabling adam optimizer, default False
        :param enable_gp: enabling gradient penaly, default False
        """

        self.s = s
        self.batch_size = batch_size
        self.input_height = input_height
        self.input_width = input_width
        self.input_channel = input_channel
        self.n_classes = n_classes

        self.sample_size = sample_size
        self.sample_num = sample_num
        self.n_input = n_input

        self.image_shape = [self.input_height, self.input_width, self.input_channel]

        self.z_dim = z_dim
        self.gf_dim = gf_dim
        self.df_dim = df_dim
        self.eps = epsilon

        # Custom Batch Normalization
        self.d_bn1 = BatchNorm(self.batch_size, name='d_bn1')
        self.d_bn2 = BatchNorm(self.batch_size, name='d_bn2')
        self.d_bn3 = BatchNorm(self.batch_size, name='d_bn3')

        self.g_bn1 = BatchNorm(self.batch_size, name='g_bn1')
        self.g_bn2 = BatchNorm(self.batch_size, name='g_bn2')
        self.g_bn3 = BatchNorm(self.batch_size, name='g_bn3')

        # Placeholders
        self.x = tf.placeholder(tf.float32, shape=[self.batch_size] + self.image_shape, name='x-images')
        self.z = tf.placeholder(tf.float32, shape=[self.batch_size, self.z_dim], name='z-noise')

        # Training Options - based on the WGAN paper
        self.learning_rate = 5e-5  # very slow
        self.critic = 5
        self.clip = 0.01
        self.d_clip = []  # (-0.01 ~ 0.01)
        self.d_lambda = 0.25
        self.decay = 0.90

        self.EnableBN = enable_bn
        self.EnableSELU = enable_selu
        self.EnableAdam = enable_adam
        self.EnableGP = enable_gp

        self.d_op = None
        self.g_op = None

        self.build_wgan()  # build WGAN model

    def discriminator(self, x, reuse=None):
        with tf.variable_scope('discriminator', reuse=reuse):
            h0 = conv2d(x, self.df_dim, name='d_h0_conv')
            if self.EnableBN:
                h0 = self.d_bn1(h0)
            if self.EnableSELU:
                h0 = tf.nn.selu(h0)
            else:
                h0 = tf.nn.leaky_relu(h0, alpha=0.2)

            h1 = conv2d(h0, self.df_dim * 2, name='d_h1_conv')
            if self.EnableBN:
                h1 = self.d_bn2(h1)
            if self.EnableSELU:
                h1 = tf.nn.selu(h1)
            else:
                h1 = tf.nn.leaky_relu(h1, alpha=0.2)

            h2 = conv2d(h1, self.df_dim * 4, name='d_h2_conv')
            if self.EnableBN:
                h2 = self.d_bn3(h2)
            if self.EnableSELU:
                h2 = tf.nn.selu(h2)
            else:
                h2 = tf.nn.leaky_relu(h2, alpha=0.2)

            h3 = linear(tf.reshape(h2, [self.batch_size, -1]), 1, 'd_h3_linear')

            return tf.nn.sigmoid(h3)

    def generator(self, z, reuse=None):
        with tf.variable_scope('generator', reuse=reuse):
            h0 = tf.reshape(linear(z, self.gf_dim * 8 * 4 * 4, 'g_h0_lin'), [-1, 4, 4, self.gf_dim * 8])
            if self.EnableBN:
                h0 = self.d_bn1(h0)
            if self.EnableSELU:
                h0 = tf.nn.selu(h0)
            else:
                h0 = tf.nn.leaky_relu(h0, alpha=0.2)

            h1 = deconv2d(h0, [self.batch_size, 7, 7, self.gf_dim * 4], name='g_h1')
            if self.EnableBN:
                h1 = self.d_bn2(h1)
            if self.EnableSELU:
                h1 = tf.nn.selu(h1)
            else:
                h1 = tf.nn.leaky_relu(h1, alpha=0.2)

            h2 = deconv2d(h1, [self.batch_size, 14, 14, self.gf_dim * 2], name='g_h2')
            if self.EnableBN:
                h2 = self.d_bn3(h2)
            if self.EnableSELU:
                h2 = tf.nn.selu(h2)
            else:
                h2 = tf.nn.leaky_relu(h2, alpha=0.2)

            h3 = deconv2d(h2, [self.batch_size,  # output shape is same as input shape
                               self.input_height, self.input_width, self.input_channel], name='g_h3')

            return tf.nn.tanh(h3)

    def build_wgan(self):
        def log(x, eps=self.eps):
            return tf.log(x + eps)

        # Generator
        self.g = self.generator(self.z)

        # Discriminator
        d_real = self.discriminator(self.x)
        d_fake = self.discriminator(self.g, reuse=True)

        # The WGAN losses
        d_real_loss = -tf.reduce_mean(log(d_real))
        d_fake_loss = -tf.reduce_mean(log(1. - d_fake))
        self.d_loss = d_real_loss + d_fake_loss
        self.g_loss = -tf.reduce_mean(log(d_fake))

        # The gradient penalty loss
        if self.EnableGP:
            alpha = tf.random_uniform(shape=self.x.get_shape(),
                                      minval=0., maxval=1., name='alpha')
            diff = self.g - self.x  # fake data - real data
            interpolates = self.x + alpha * diff
            d_interp = self.discriminator(interpolates, reuse=True)
            gradients = tf.gradients(d_interp, [interpolates])[0]
            slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients), reduction_indices=[1]))
            gradient_penalty = tf.reduce_mean((slopes - 1.) ** 2)

            # Update D loss
            self.d_loss += self.d_lambda * gradient_penalty

        # Summary
        tf.summary.histogram("z", self.z)

        g = tf.reshape(self.g, shape=[-1] + self.image_shape)
        tf.summary.image("g", g)  # generated image from G model
        tf.summary.histogram("d_real", d_real)
        tf.summary.histogram("d_fake", d_fake)

        tf.summary.scalar("d_real_loss", d_real_loss)
        tf.summary.scalar("d_fake_loss", d_fake_loss)
        tf.summary.scalar("d_loss", self.d_loss)
        tf.summary.scalar("g_loss", self.g_loss)

        # Collect trainer values
        t_vars = tf.trainable_variables()
        d_params = [v for v in t_vars if v.name.startswith('discriminator')]
        g_params = [v for v in t_vars if v.name.startswith('generator')]

        self.d_clip = [v.assign(tf.clip_by_value(v, -self.clip, self.clip)) for v in d_params]

        # Optimizer
        if self.EnableAdam:
            self.d_op = tf.train.AdamOptimizer(learning_rate=2e-4,
                                               beta1=0.5).minimize(self.d_loss, var_list=d_params)
            self.g_op = tf.train.AdamOptimizer(learning_rate=2e-4,
                                               beta1=0.5).minimize(self.g_loss, var_list=g_params)
        else:
            self.d_op = tf.train.RMSPropOptimizer(learning_rate=self.learning_rate,
                                                  decay=self.decay).minimize(self.d_loss, var_list=d_params)
            self.g_op = tf.train.RMSPropOptimizer(learning_rate=self.learning_rate,
                                                  decay=self.decay).minimize(self.g_loss, var_list=g_params)

        # Merge summary
        self.merged = tf.summary.merge_all()

        # Model Saver
        self.saver = tf.train.Saver(max_to_keep=1)
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pygments.formatters.other
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    Other formatters: NullFormatter, RawTokenFormatter.

    :copyright: 2006-2007 by Georg Brandl, Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

from pygments.formatter import Formatter
from pygments.util import get_choice_opt

__all__ = ['NullFormatter', 'RawTokenFormatter']

class NullFormatter(Formatter):
    """
    Output the text unchanged without any formatting.
    """
    name = 'Text only'
    aliases = ['text', 'null']
    filenames = ['*.txt']

    def format(self, tokensource, outfile):
        enc = self.encoding
        for ttype, value in tokensource:
            if enc:
                outfile.write(value.encode(enc))
            else:
                outfile.write(value)

class RawTokenFormatter(Formatter):
    r"""
    Format tokens as a raw representation for storing token streams.

    The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later
    be converted to a token stream with the `RawTokenLexer`, described in the
    `lexer list <lexers.txt>`_.

    Only one option is accepted:

    `compress`
        If set to ``'gz'`` or ``'bz2'``, compress the output with the given
        compression algorithm after encoding (default: ``''``).
    """
    name = 'Raw tokens'
    aliases = ['raw', 'tokens']
    filenames = ['*.raw']

    unicodeoutput = False

    def __init__(self, **options):
        Formatter.__init__(self, **options)
        self.compress = get_choice_opt(options, 'compress',
                                       ['', 'none', 'gz', 'bz2'], '')

    def format(self, tokensource, outfile):
        if self.compress == 'gz':
            import gzip
            outfile = gzip.GzipFile('', 'wb', 9, outfile)
            write = outfile.write
            flush = outfile.flush
        elif self.compress == 'bz2':
            import bz2
            compressor = bz2.BZ2Compressor(9)
            def write(text):
                outfile.write(compressor.compress(text))
            def flush():
                outfile.write(compressor.flush())
                outfile.flush()
        else:
            write = outfile.write
            flush = outfile.flush

        lasttype = None
        lastval = u''
        for ttype, value in tokensource:
            write("%s\t%r\n" % (ttype, value))
        flush()
<commit_msg>Add optional error token colorizing to the !RawTokenFormatter to close #333.<commit_after># -*- coding: utf-8 -*-
"""
    pygments.formatters.other
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    Other formatters: NullFormatter, RawTokenFormatter.

    :copyright: 2006-2007 by Georg Brandl, Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

from pygments.formatter import Formatter
from pygments.util import get_choice_opt
from pygments.token import Token
from pygments.console import colorize

__all__ = ['NullFormatter', 'RawTokenFormatter']

class NullFormatter(Formatter):
    """
    Output the text unchanged without any formatting.
    """
    name = 'Text only'
    aliases = ['text', 'null']
    filenames = ['*.txt']

    def format(self, tokensource, outfile):
        enc = self.encoding
        for ttype, value in tokensource:
            if enc:
                outfile.write(value.encode(enc))
            else:
                outfile.write(value)

class RawTokenFormatter(Formatter):
    r"""
    Format tokens as a raw representation for storing token streams.

    The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later
    be converted to a token stream with the `RawTokenLexer`, described in the
    `lexer list <lexers.txt>`_.

    Only one option is accepted:

    `compress`
        If set to ``'gz'`` or ``'bz2'``, compress the output with the given
        compression algorithm after encoding (default: ``''``).
    `error_color`
        If set to a color name, highlight error tokens using that color.  If
        set but with no value, defaults to ``'red'``.
        *New in Pygments 0.11.*

    """
    name = 'Raw tokens'
    aliases = ['raw', 'tokens']
    filenames = ['*.raw']

    unicodeoutput = False

    def __init__(self, **options):
        Formatter.__init__(self, **options)
        self.compress = get_choice_opt(options, 'compress',
                                       ['', 'none', 'gz', 'bz2'], '')
        self.error_color = options.get('error_color', None)
        if self.error_color is True:
            self.error_color = 'red'
        if self.error_color is not None:
            try:
                colorize(self.error_color, '')
            except KeyError:
                raise ValueError("Invalid color %r specified" %
                                 self.error_color)

    def format(self, tokensource, outfile):
        if self.compress == 'gz':
            import gzip
            outfile = gzip.GzipFile('', 'wb', 9, outfile)
            write = outfile.write
            flush = outfile.flush
        elif self.compress == 'bz2':
            import bz2
            compressor = bz2.BZ2Compressor(9)
            def write(text):
                outfile.write(compressor.compress(text))
            def flush():
                outfile.write(compressor.flush())
                outfile.flush()
        else:
            write = outfile.write
            flush = outfile.flush

        lasttype = None
        lastval = u''
        if self.error_color:
            for ttype, value in tokensource:
                line = "%s\t%r\n" % (ttype, value)
                if ttype is Token.Error:
                    write(colorize(self.error_color, line))
                else:
                    write(line)
        else:
            for ttype, value in tokensource:
                write("%s\t%r\n" % (ttype, value))
        flush()
<|endoftext|>


<commit_before>from kraconvert import kra
import os
import argparse
from PIL import Image, ImageCms

__author__ = 'pierre'

parser = argparse.ArgumentParser(prog='kraconvert', description='Krita .kra file batch processing')

parser.add_argument(
    'kras', metavar='kra', type=str, nargs='+', help='list of .kra files to convert'
)

parser.add_argument(
    '-i', '--icc', action='store_true', help='Extract icc colour profile'
)

parser.add_argument(
    '-o', '--output', type=str, nargs=1, required=False,
    help='output directory, if you convert multiple kra files, each'
         'gets it\'s own directory', default=os.curdir
)

parser.add_argument(
    '-p', '--png', action='store_true', required=False, help='extract the merged PNG file'
)

jpeg = parser.add_argument_group('JPEG Options')

jpeg.add_argument(
    '-j', '--jpeg', type=int, default=90, required=False
)

jpeg.add_argument(
    '-s', '--sizes', type=int, nargs='*', default=[1600, 1000, 800, 600], help='Desired sizes for your JPEG files'
)

jpeg.add_argument(
    '-w', '--webready', action='store_true', help='Try to convert jpeg files to sRGB-builtin'
)

args = parser.parse_args()

def extract_png(kras, outdir):
    for kra in kras:
        png = kra.get_merged_image()
        png_name = kra.get_basename() + '.png'

        png_dir = os.path.join(outdir, kra.get_basename(), 'png')
        os.makedirs(png_dir, exist_ok=True)
        kra.merged_image_path = os.path.join(png_dir, png_name)

        with open(os.path.join(png_dir, png_name), 'w+b') as f:
            f.write(png)
            f.close()

def extract_icc(kras, outdir):
    for kra in kras:
        icc = kra.get_icc()
        icc_dir = os.path.join(outdir, kra.get_basename(), 'icc')
        os.makedirs(icc_dir, exist_ok=True)
        kra.icc_path = os.path.join(icc_dir, icc['name'])

        with open(os.path.join(icc_dir, icc['name']), 'w+b') as f:
            f.write(icc['data'])
            f.close()

def export_as_jpegs(kras, outdir):
    for kra in kras:
        for size in args.sizes:
            jpeg_dir = os.path.join(outdir, kra.get_basename(), 'jpeg')
            os.makedirs(jpeg_dir, exist_ok=True)

            im = Image.open(kra.merged_image_path)

            jpeg_name = kra.get_basename() + '{0}.jpeg'.format(size)

            new_im = im.thumbnail((size, size), Image.ANTIALIAS)
            srgb = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'files', 'sRGB.icc')

            if args.webready:
                im = ImageCms.profileToProfile(im, kra.icc_path, srgb)
                im.save(os.path.join(jpeg_dir, jpeg_name), quality=args.jpeg)
            else:
                im.save(os.path.join(jpeg_dir, jpeg_name), quality=args.jpeg, icc_profile=kra.icc)

def main():
    kras = []

    outdir = args.output

    depend_on_png = (args.jpeg,)

    if args.webready and not args.icc:
        raise SystemExit('I need -i to get the source icc profile')

    if not args.png and any(depend_on_png):
        raise SystemExit('Some options work only if you specify -p')

    if args.kras:
        for krafile in args.kras:
            kras.append(kra.Kra(krafile))

    if args.png:
        extract_png(kras, outdir)

    if args.icc:
        extract_icc(kras, outdir)

    if args.jpeg:
        export_as_jpegs(kras, outdir)

if __name__ == '__main__':
    main()
<commit_msg>Possible to save a webversion of the png with sRGB Profile.<commit_after>from kraconvert import kra
import os
import argparse
from PIL import Image, ImageCms

__author__ = 'pierre'

parser = argparse.ArgumentParser(prog='kraconvert', description='Krita .kra file batch processing')

parser.add_argument(
    'kras', metavar='kra', type=str, nargs='+', help='list of .kra files to convert'
)

parser.add_argument(
    '-i', '--icc', action='store_true', help='Extract icc colour profile'
)

parser.add_argument(
    '-o', '--output', type=str, nargs=1, required=False,
    help='output directory, if you convert multiple kra files, each'
         'gets it\'s own directory', default=os.curdir
)

parser.add_argument(
    '-p', '--png', action='store_true', required=False, help='extract the merged PNG file'
)

jpeg = parser.add_argument_group('JPEG Options')

jpeg.add_argument(
    '-j', '--jpeg', type=int, default=90, required=False
)

jpeg.add_argument(
    '-s', '--sizes', type=int, nargs='*', default=[1600, 1000, 800, 600], help='Desired sizes for your JPEG files'
)

jpeg.add_argument(
    '-w', '--webready', action='store_true', help='Try to convert jpeg files to sRGB-builtin'
)

args = parser.parse_args()

srgb = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'files', 'sRGB.icc')

def extract_png(kras, outdir):
    for kra in kras:
        png = kra.get_merged_image()
        png_name = kra.get_basename() + '.png'
        png_name_web = kra.get_basename() + '_srgb.png'

        png_dir = os.path.join(outdir, kra.get_basename(), 'png')
        os.makedirs(png_dir, exist_ok=True)
        kra.merged_image_path = os.path.join(png_dir, png_name)

        with open(kra.merged_image_path, 'w+b') as f:
            f.write(png)
            f.close()

        if args.webready:
            im = Image.open(kra.merged_image_path)

            # BUG: I don't know why but I need to use inPlace here or else
            # my test png was just transparent
            ImageCms.profileToProfile(im, kra.icc_path, srgb, inPlace=True)
            im.save(os.path.join(png_dir, png_name_web), optimize=True)

def extract_icc(kras, outdir):
    for kra in kras:
        icc = kra.get_icc()
        icc_dir = os.path.join(outdir, kra.get_basename(), 'icc')
        os.makedirs(icc_dir, exist_ok=True)
        kra.icc_path = os.path.join(icc_dir, icc['name'])

        with open(kra.icc_path, 'w+b') as f:
            f.write(icc['data'])
            f.close()

def export_as_jpegs(kras, outdir):
    for kra in kras:
        for size in args.sizes:
            jpeg_dir = os.path.join(outdir, kra.get_basename(), 'jpeg')
            os.makedirs(jpeg_dir, exist_ok=True)

            im = Image.open(kra.merged_image_path)

            jpeg_name = kra.get_basename() + '{0}.jpeg'.format(size)

            new_im = im.thumbnail((size, size), Image.ANTIALIAS)

            if args.webready:
                im = ImageCms.profileToProfile(im, kra.icc_path, srgb)
                im.save(os.path.join(jpeg_dir, jpeg_name), quality=args.jpeg, optimize=True)
            else:
                im.save(os.path.join(jpeg_dir, jpeg_name), quality=args.jpeg, optimize=True, icc_profile=kra.icc)

def main():
    kras = []

    outdir = args.output

    depend_on_png = (args.jpeg,)

    if args.webready and not args.icc:
        raise SystemExit('I need -i to get the source icc profile')

    if not args.png and any(depend_on_png):
        raise SystemExit('Some options work only if you specify -p')

    if args.kras:
        for krafile in args.kras:
            kras.append(kra.Kra(krafile))

    if args.icc:
        extract_icc(kras, outdir)

    if args.png:
        extract_png(kras, outdir)

    if args.jpeg:
        export_as_jpegs(kras, outdir)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from django.contrib import admin

from kuma.core.models import IPBan

admin.site.register(IPBan, admin.ModelAdmin)
<commit_msg>bug 1122658 - custom admin list_display for IPBan<commit_after>from django.contrib import admin

from kuma.core.models import IPBan

class IPBanAdmin(admin.ModelAdmin):
    list_display = ('ip', 'created', 'deleted')

admin.site.register(IPBan, IPBanAdmin)
<|endoftext|>


<commit_before>from django.contrib import admin

from kuma.core.models import IPBan

class IPBanAdmin(admin.ModelAdmin):
    list_display = ('ip', 'created', 'deleted')

admin.site.register(IPBan, IPBanAdmin)
<commit_msg>bug 1113260 - enforce soft model deletes in admin<commit_after>from django.contrib import admin

from kuma.core.models import IPBan

class IPBanAdmin(admin.ModelAdmin):
    # Remove list delete action to enforce model soft delete in admin site
    actions = None
    readonly_fields = ('deleted',)
    list_display = ('ip', 'created', 'deleted')

admin.site.register(IPBan, IPBanAdmin)
<|endoftext|>