<commit_before>from logs import logDecorator as lD
import json, os
import matplotlib.pyplot as plt # This comes before networkx
import networkx as nx
from networkx.drawing.nx_pydot import graphviz_layout
from datetime import datetime as dt
from lib.databaseIO import pgIO

config = json.load(open('../config/config.json'))
logBase = config['logging']['logBase'] + '.lib.resultGraph.graphLib'

@lD.log(logBase + '.generateGraph')
def generateGraph(logger):
    '''generate a directed graph from the modules config

    generate a networkX.Graph object by reading the contents
    of the ``config/modules/`` folder.

    Parameters
    ----------
    logger : {logging.logger}
        logging element

    Returns
    -------
    [type]
        [description]
    '''

    try:

        graph = nx.DiGraph()

        folder = '../config/modules'
        files  = [f for f in os.listdir(folder) if f.endswith('.json')]

        for f in files:
            data = json.load(open(os.path.join(folder, f)))
            inp  = list(data['inputs'].keys())
            out  = list(data['outputs'].keys())
            f = f.replace('.json', '')

            graph.add_node( f, type='module', summary='' )

            # Add the incoming edges
            for n in inp:
                if n not in graph.nodes:
                    summary = json.dumps(data['inputs'][n])
                    graph.add_node( n,
                        type    = data['inputs'][n]['type'],
                        summary = summary)

                graph.add_edge(n, f)

            # Add the outgoing edges
            for n in out:
                if n not in graph.nodes:
                    summary = json.dumps(data['outputs'][n])
                    graph.add_node( n,
                        type    = data['outputs'][n]['type'],
                        summary = summary)

                graph.add_edge(f, n)

    except Exception as e:
        logger.error('Unable to generate the graph: {}'.format(e))

    return graph

@lD.log(logBase + '.plotGraph')
def plotGraph(logger, graph, fileName=None):
    '''[summary]

    [description]

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    graph : {[type]}
        [description]
    fileName : {[type]}, optional
        [description] (the default is None, which [default_description])
    '''

    try:
        plt.figure()

        moduleNodes = [m for m, d in graph.nodes(data=True) if ('module' == d['type'])]
        otherNodes  = [m for m, d in graph.nodes(data=True) if ('module' != d['type'])]
        lables      = {m:m for m in graph.nodes}

        pos = graphviz_layout(graph, prog='dot')
        nx.draw_networkx_nodes(graph, pos, nodelist=moduleNodes, node_color='orange', node_size=500)
        nx.draw_networkx_nodes(graph, pos, nodelist=otherNodes, node_color='cyan', node_size=500)
        nx.draw_networkx_edges(graph, pos,  arrows=True)
        nx.draw_networkx_labels(graph, pos, lables, font_size=10)

        if fileName is not None:
            plt.savefig(fileName)

        plt.close()
        print('Graph saved ...')
    except Exception as e:
        logger.error('Unable to plot the graph: {}'.format(e))

    return

@lD.log(logBase + '.generateSubGraph')
def generateSubGraph(logger, graph, keyNode):
    '''[summary]

    [description]

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    graph : {[type]}
        [description]
    keyNode : {[type]}
        [description]

    Returns
    -------
    [type]
        [description]
    '''

    try:
        newGraph = nx.DiGraph()

        nodes = list(nx.ancestors(graph, keyNode))
        nodes.append(keyNode)

        for n, d in graph.nodes(data=True):
            if (n in nodes):
                newGraph.add_node( n, **d )

        for n1, n2 in graph.edges:
            if (n1 in nodes) and (n2 in nodes):
                newGraph.add_edge(n1, n2)

    except Exception as e:
        logger.error('Unable to generate the right subgraph: {}'.format(e))

    return newGraph

@lD.log(logBase + '.graphToSerialized')
def graphToSerialized(logger, graph):
    '''serializes a graph

    Takes a networkX.Graph object and converts it into a serialized
    set of nodes and edges.

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    graph : {networkX.Graph object}
        A networkX graph object that is to be serialized

    Returns
    -------
    tuple of serialized lists
        This is a tuple of nodes and edges in a serialized format
        that can be later directly inserted into a database.
    '''

    progName = config['logging']['logBase']

    now = dt.now()

    nodes = []
    for n, d in graph.nodes(data=True):
            nodes.append([
                progName,  # program name
                now,   # current datetime
                n,         # node name
                d['type'], # 'module', 'csv'. ...
                d['summary']])

    edges = []
    for n1, n2 in graph.edges:
        edges.append([progName, now, n1, n2])

    return nodes, edges

@lD.log(logBase + '.serializedToGraph')
def serializedToGraph(logger, nodes, edges):
    '''deserialize a graph serialized earlier

    Take serialized versions of the nodes and edges which is
    produced by the function ``graphToSerialized`` and convert
    that into a normal ``networkX``.

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    nodes : {list}
        serialized versions of the nodes of a graph
    edges : {list}
        A list of edges in a serialized form

    Returns
    -------
    networkX.Graph object
        Takes a list of serialized nodes and edges and converts it
        into a networkX.Graph object
    '''

    graph = nx.DiGraph()

    for _, _, n, t, s in nodes:
        graph.add_node( n, type = t, summary = s)

    for _, _, e1, e2 in edges:
        graph.add_edge( e1, e2)

    return graph

@lD.log(logBase + '.uploadGraph')
def uploadGraph(logger, graph, dbName=None):
    '''upload the supplied graph to a database

    Given a graph, this function is going to upload the graph into
    a particular database. In case a database is not specified, this
    will try to upload the data into the default database.

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    graph : {networkX graph}
        The graph that needs to be uploaded into the database
    dbName : {str}, optional
        The name of the database in which to upload the data into. This
        is the identifier within the ``db.json`` configuration file. (the
        default is ``None``, which would use the default database specified
        within the same file)
    '''

    try:

        nodes, edges graphToSerialized(graph)

        queryNodes = '''insert into graph.nodes values %s'''
        queryEdges = '''insert into graph.edges values %s'''

        pgIO.commitDataList(queryNodes, nodes, dbName=dbName)
        pgIO.commitDataList(queryEdges, edges, dbName=dbName)

    except Exception as e:
        logger.error()

    return

<commit_msg>changes in the graph<commit_after>from logs import logDecorator as lD
import json, os
import matplotlib.pyplot as plt # This comes before networkx
import networkx as nx
from networkx.drawing.nx_pydot import graphviz_layout
from datetime import datetime as dt
from lib.databaseIO import pgIO

config = json.load(open('../config/config.json'))
logBase = config['logging']['logBase'] + '.lib.resultGraph.graphLib'

@lD.log(logBase + '.generateGraph')
def generateGraph(logger):
    '''generate a directed graph from the modules config

    generate a networkX.Graph object by reading the contents
    of the ``config/modules/`` folder.

    Parameters
    ----------
    logger : {logging.logger}
        logging element

    Returns
    -------
    networkX.Graph object
        Graph of which object is created when
    '''

    try:

        graph = nx.DiGraph()

        folder = '../config/modules'
        files  = [f for f in os.listdir(folder) if f.endswith('.json')]

        for f in files:
            data = json.load(open(os.path.join(folder, f)))
            inp  = list(data['inputs'].keys())
            out  = list(data['outputs'].keys())
            f = f.replace('.json', '')

            graph.add_node( f, type='module', summary='' )

            # Add the incoming edges
            for n in inp:
                if n not in graph.nodes:
                    summary = json.dumps(data['inputs'][n])
                    graph.add_node( n,
                        type    = data['inputs'][n]['type'],
                        summary = summary)

                graph.add_edge(n, f)

            # Add the outgoing edges
            for n in out:
                if n not in graph.nodes:
                    summary = json.dumps(data['outputs'][n])
                    graph.add_node( n,
                        type    = data['outputs'][n]['type'],
                        summary = summary)

                graph.add_edge(f, n)

    except Exception as e:
        logger.error('Unable to generate the graph: {}'.format(e))

    return graph

@lD.log(logBase + '.plotGraph')
def plotGraph(logger, graph, fileName=None):
    '''plot the graph

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    graph : {networkX.Graph object}
        The graph that needs to be plotted
    fileName : {str}, optional
        name of the file where to save the graph (the default is None, which
        results in no graph being generated)
    '''

    try:
        plt.figure()

        moduleNodes = [m for m, d in graph.nodes(data=True) if ('module' == d['type'])]
        otherNodes  = [m for m, d in graph.nodes(data=True) if ('module' != d['type'])]
        lables      = {m:m for m in graph.nodes}

        pos = graphviz_layout(graph, prog='dot')
        nx.draw_networkx_nodes(graph, pos, nodelist=moduleNodes, node_color='orange', node_size=500)
        nx.draw_networkx_nodes(graph, pos, nodelist=otherNodes, node_color='cyan', node_size=500)
        nx.draw_networkx_edges(graph, pos,  arrows=True)
        nx.draw_networkx_labels(graph, pos, lables, font_size=10)

        if fileName is not None:
            plt.savefig(fileName)

        plt.close()
        print('Graph saved ...')
    except Exception as e:
        logger.error('Unable to plot the graph: {}'.format(e))

    return

@lD.log(logBase + '.generateSubGraph')
def generateSubGraph(logger, graph, keyNode):
    '''generate a subgraph that contains all prior nodes

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    graph : {networkX.Graph object}
        [description]
    keyNode : {str}
        Name of the node whose ancestors need to be geenrated.

    Returns
    -------
    networkX.Graph object
        graph containing the particular node and its ancistors.
    '''

    try:
        newGraph = nx.DiGraph()

        nodes = list(nx.ancestors(graph, keyNode))
        nodes.append(keyNode)

        for n, d in graph.nodes(data=True):
            if (n in nodes):
                newGraph.add_node( n, **d )

        for n1, n2 in graph.edges:
            if (n1 in nodes) and (n2 in nodes):
                newGraph.add_edge(n1, n2)

    except Exception as e:
        logger.error('Unable to generate the right subgraph: {}'.format(e))

    return newGraph

@lD.log(logBase + '.graphToSerialized')
def graphToSerialized(logger, graph):
    '''serializes a graph

    Takes a networkX.Graph object and converts it into a serialized
    set of nodes and edges.

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    graph : {networkX.Graph object}
        A networkX graph object that is to be serialized

    Returns
    -------
    tuple of serialized lists
        This is a tuple of nodes and edges in a serialized format
        that can be later directly inserted into a database.
    '''

    progName = config['logging']['logBase']

    now = dt.now()

    nodes = []
    for n, d in graph.nodes(data=True):
            nodes.append([
                progName,  # program name
                now,   # current datetime
                n,         # node name
                d['type'], # 'module', 'csv'. ...
                d['summary']])

    edges = []
    for n1, n2 in graph.edges:
        edges.append([progName, now, n1, n2])

    return nodes, edges

@lD.log(logBase + '.serializedToGraph')
def serializedToGraph(logger, nodes, edges):
    '''deserialize a graph serialized earlier

    Take serialized versions of the nodes and edges which is
    produced by the function ``graphToSerialized`` and convert
    that into a normal ``networkX``.

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    nodes : {list}
        serialized versions of the nodes of a graph
    edges : {list}
        A list of edges in a serialized form

    Returns
    -------
    networkX.Graph object
        Takes a list of serialized nodes and edges and converts it
        into a networkX.Graph object
    '''

    graph = nx.DiGraph()

    for _, _, n, t, s in nodes:
        graph.add_node( n, type = t, summary = s)

    for _, _, e1, e2 in edges:
        graph.add_edge( e1, e2)

    return graph

@lD.log(logBase + '.uploadGraph')
def uploadGraph(logger, graph, dbName=None):
    '''upload the supplied graph to a database

    Given a graph, this function is going to upload the graph into
    a particular database. In case a database is not specified, this
    will try to upload the data into the default database.

    Parameters
    ----------
    logger : {logging.logger}
        logging element
    graph : {networkX graph}
        The graph that needs to be uploaded into the database
    dbName : {str}, optional
        The name of the database in which to upload the data into. This
        is the identifier within the ``db.json`` configuration file. (the
        default is ``None``, which would use the default database specified
        within the same file)
    '''

    try:

        nodes, edges = graphToSerialized(graph)

        queryNodes = '''insert into graph.nodes values %s'''
        queryEdges = '''insert into graph.edges values %s'''

        pgIO.commitDataList(queryNodes, nodes, dbName=dbName)
        pgIO.commitDataList(queryEdges, edges, dbName=dbName)

    except Exception as e:
        logger.error()

    return

<|endoftext|>


<commit_before>
"""
    fom.version
    ~~~~~~~~~~~

    The version of Fom

    :copyright: 2010 Fom authors
    :license: MIT, see LICENSE for more information.
"""

version = '0.9.8'

<commit_msg>Bumped version number for PyPi<commit_after>
"""
    fom.version
    ~~~~~~~~~~~

    The version of Fom

    :copyright: 2010 Fom authors
    :license: MIT, see LICENSE for more information.
"""

version = '0.9.9'

<|endoftext|>


<commit_before>"""
Forever.fm Server
by @psobot, Nov 3 2012
"""

import logging
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
import config
import apikeys
import database

import os
import sys
import json
import lame
import copy
import time
import info
import random
import datetime
import threading
import traceback
import tornado.web
import tornado.ioloop
import tornado.template
import tornadio2.server
import multiprocessing
import pyechonest.config

from mixer import Mixer
from daemon import Daemon
from utils import daemonize
from listeners import Listeners
from assetcompiler import compiled
from sockethandler import SocketHandler
from bufferedqueue import BufferedReadQueue
from monitor import MonitorHandler, MonitorSocket, monitordaemon

#   API Key setup
pyechonest.config.ECHO_NEST_API_KEY = apikeys.ECHO_NEST_API_KEY

started_at_timestamp = time.time()
started_at = datetime.datetime.utcnow()

test = 'test' in sys.argv
SECONDS_PER_FRAME = lame.SAMPLES_PER_FRAME / 44100.0

templates = tornado.template.Loader(config.template_dir)
templates.autoescape = None
first_frame = threading.Semaphore(0)

class MainHandler(tornado.web.RequestHandler):
	mtime = 0
	template = 'index.html'

	def __gen(self):
		debug = self.get_argument('__debug', None)
		if debug is None:
			debug = self.request.host.startswith("localhost")
		else:
			debug = debug in ["True", 1, "on"]
		kwargs = {
			'debug': debug,
			'compiled': compiled,
			'open': len(StreamHandler.relays) > 0 or debug,
			'endpoint': StreamHandler.relay_url() if not debug else "/all.mp3"
		}
		try:
			if os.path.getmtime(config.template_dir + self.template) > self.mtime:
				templates.reset()
				self.mtime = time.time()
			return templates.load(self.template).generate(**kwargs)
		except Exception, e:
			log.error(e)
			tornado.web.RequestHandler.send_error(self, 500)
			return

	def head(self):
		self.__gen()
		self.finish()

	def get(self):
		self.finish(self.__gen())

class InfoHandler(tornado.web.RequestHandler):
	actions = []
	started = None
	samples = 0L
	duration = 0.

	@classmethod
	def add(self, data):
		if not self.actions:
			self.started = time.time()
		self.samples += data['samples']
		self.duration += data['duration']
		self.clean()
		log.info("Adding track info. Currently holding info for %d tracks.",
				 len(self.actions))
		self.actions.append(data)
		SocketHandler.on_segment(data)

	@classmethod
	def clean(cls):
			now = time.time()
			while cls.actions and cls.actions[0]['time'] \
			   + cls.actions[0]['duration'] + config.past_played_buffer < now:
				cls.actions.pop(0)

	@classmethod
	def stats(cls):
		return {
			"started": cls.started,
			"samples": cls.samples,
			"duration": cls.duration,
		}

	def get(self):
		self.set_header("Content-Type", "application/json")
		try:
			now = self.get_argument('now', None)
			if now:
				now = time.time()
				for _action in self.actions:
					if _action['time'] < now \
					and _action['time'] + _action['duration'] > now:
						action = copy.copy(_action)
						self.write(json.dumps({'frame': action, 'now': now},
								   ensure_ascii=False).encode('utf-8'))
						return
				self.write(json.dumps([]))
			else:
				self.write(json.dumps(self.actions, ensure_ascii=False).encode('utf-8'))
		except Exception:
			log.error("Could not send info burst:\n%s", traceback.format_exc())
			log.error("Data:\n%s", self.actions)
			self.write(json.dumps([]))

class TimingHandler(tornado.web.RequestHandler):
	def get(self):
		self.set_header("Content-Type", "application/json")
		self.write(json.dumps({"time": time.time() * 1000}, ensure_ascii=False).encode('utf-8'))

class StreamHandler(tornado.web.RequestHandler):
	relays = []
	listeners = []

	@classmethod
	def relay_url(cls):
		if len(cls.relays) == 1:
			return cls.relays[0].url
		elif len(cls.relays) > 1:
			choices = [relay for relay in cls.relays for _ in xrange(0, relay.weight)]
			return random.choice(choices).url
		else:
			return ""

	@classmethod
	def stream_frames(cls):
		try:
			cls.relays.broadcast()
		except Exception:
			log.error("Could not broadcast due to: \n%s", traceback.format_exc())

	@classmethod
	def check(cls):
		#   TODO: This should do HTTP requests to ensure that all relays are
		#   still up
		pass

	def head(self):
		try:
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			ua = self.request.headers.get('User-Agent', None)
			if ua == config.relay_ua or len(self.relays) == 0:
				log.info("Got HEAD for relay at %s.", ip)
				self.set_header("Content-Type", "audio/mpeg")
				self.finish()
			else:
				if not self.relays:
					tornado.web.RequestHandler.send_error(self, 503)
				else:
					relay = self.relay_url()
					log.info("Redirected new listener %s to %s", ip, relay)
					self.redirect(relay)
		except Exception:
			log.error("Error in stream.head:\n%s", traceback.format_exc())
			tornado.web.RequestHandler.send_error(self, 500)

	@tornado.web.asynchronous
	def get(self):
		try:
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			ua = self.request.headers.get('User-Agent', None)
			if ua == config.relay_ua:
				url = self.request.headers['X-Relay-Addr']
				if not url.startswith('http://'):
					url = "http://" + url
				port = self.request.headers['X-Relay-Port']
				self.weight = int(self.request.headers.get('X-Relay-Weight', 1))
				log.info("Added new relay at %s:%s with weight %d.", url, port, self.weight)
				self.set_header("Content-Type", "audio/mpeg")
				self.url = "%s:%s/all.mp3" % (url, port)
				self.relays.append(self)
			else:
				if self.request.host.startswith("localhost:8193"):
					log.info("Added new debug listener at %s.", ip)
					self.set_header("Content-Type", "audio/mpeg")
					self.relays.append(self)
				elif not self.relays:
					tornado.web.RequestHandler.send_error(self, 503)
				else:
					relay = self.relay_url()
					log.info("Redirected new listener %s to %s", ip, relay)
					self.redirect(relay)
		except Exception:
			log.error("Error in stream.get:\n%s", traceback.format_exc())
			tornado.web.RequestHandler.send_error(self, 500)

	def on_finish(self):
		if self in self.relays:
			self.relays.remove(self)
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			log.info("Removed relay at %s with weight %d.", ip, self.weight)

class SocketConnection(tornadio2.conn.SocketConnection):
	__endpoints__ = {
		"/info.websocket": SocketHandler,   #TODO: Rename
		"/monitor.websocket": MonitorSocket
	}

if __name__ == "__main__":
	Daemon()

	log.info("Starting %s...", config.app_name)

	track_queue = multiprocessing.Queue(1)
	log.info("Initializing read queue to hold %2.2f seconds of audio.",
			 config.frontend_buffer)
	v2_queue = BufferedReadQueue(int(config.frontend_buffer / SECONDS_PER_FRAME))
	info_queue = multiprocessing.Queue()

	mixer = Mixer(iqueue=track_queue,
				  oqueue=v2_queue.raw,
				  infoqueue=info_queue)
	mixer.start()

	daemonize(database.enqueue_tracks, track_queue)
	daemonize(info.generate, info_queue, first_frame, InfoHandler)
	StreamHandler.relays = Listeners(v2_queue, "All", first_frame)
	daemonize(monitordaemon,StreamHandler.relays,InfoHandler.stats,{"mp3_queue":v2_queue})

	tornado.ioloop.PeriodicCallback(InfoHandler.clean, 5 * 1000).start()
	tornado.ioloop.PeriodicCallback(StreamHandler.check, 10 * 1000).start()

	application = tornado.web.Application(
		tornadio2.TornadioRouter(SocketConnection).apply_routes([
			# Static assets for local development
			(r"/(favicon.ico)", tornado.web.StaticFileHandler, {"path": "static/img/"}),
			(r"/static/(.*)", tornado.web.StaticFileHandler, {"path": "static/"}),

			(r"/timing.json", TimingHandler),

			(r"/all.json", InfoHandler),
			(r"/all.mp3", StreamHandler),

			(r"/monitor", MonitorHandler),
			(r"/", MainHandler),
		]),
		socket_io_port=config.socket_port,
		enabled_protocols=['websocket', 'xhr-multipart', 'xhr-polling', 'jsonp-polling']
	)

	frame_sender = tornado.ioloop.PeriodicCallback(
		StreamHandler.stream_frames, SECONDS_PER_FRAME * 1000
	)
	frame_sender.start()

	application.listen(config.http_port)
	tornadio2.server.SocketServer(application)
<commit_msg>Reword Mixer() instantiation for consistency with the other lines<commit_after>"""
Forever.fm Server
by @psobot, Nov 3 2012
"""

import logging
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
import config
import apikeys
import database

import os
import sys
import json
import lame
import copy
import time
import info
import random
import datetime
import threading
import traceback
import tornado.web
import tornado.ioloop
import tornado.template
import tornadio2.server
import multiprocessing
import pyechonest.config

from mixer import Mixer
from daemon import Daemon
from utils import daemonize
from listeners import Listeners
from assetcompiler import compiled
from sockethandler import SocketHandler
from bufferedqueue import BufferedReadQueue
from monitor import MonitorHandler, MonitorSocket, monitordaemon

#   API Key setup
pyechonest.config.ECHO_NEST_API_KEY = apikeys.ECHO_NEST_API_KEY

started_at_timestamp = time.time()
started_at = datetime.datetime.utcnow()

test = 'test' in sys.argv
SECONDS_PER_FRAME = lame.SAMPLES_PER_FRAME / 44100.0

templates = tornado.template.Loader(config.template_dir)
templates.autoescape = None
first_frame = threading.Semaphore(0)

class MainHandler(tornado.web.RequestHandler):
	mtime = 0
	template = 'index.html'

	def __gen(self):
		debug = self.get_argument('__debug', None)
		if debug is None:
			debug = self.request.host.startswith("localhost")
		else:
			debug = debug in ["True", 1, "on"]
		kwargs = {
			'debug': debug,
			'compiled': compiled,
			'open': len(StreamHandler.relays) > 0 or debug,
			'endpoint': StreamHandler.relay_url() if not debug else "/all.mp3"
		}
		try:
			if os.path.getmtime(config.template_dir + self.template) > self.mtime:
				templates.reset()
				self.mtime = time.time()
			return templates.load(self.template).generate(**kwargs)
		except Exception, e:
			log.error(e)
			tornado.web.RequestHandler.send_error(self, 500)
			return

	def head(self):
		self.__gen()
		self.finish()

	def get(self):
		self.finish(self.__gen())

class InfoHandler(tornado.web.RequestHandler):
	actions = []
	started = None
	samples = 0L
	duration = 0.

	@classmethod
	def add(self, data):
		if not self.actions:
			self.started = time.time()
		self.samples += data['samples']
		self.duration += data['duration']
		self.clean()
		log.info("Adding track info. Currently holding info for %d tracks.",
				 len(self.actions))
		self.actions.append(data)
		SocketHandler.on_segment(data)

	@classmethod
	def clean(cls):
			now = time.time()
			while cls.actions and cls.actions[0]['time'] \
			   + cls.actions[0]['duration'] + config.past_played_buffer < now:
				cls.actions.pop(0)

	@classmethod
	def stats(cls):
		return {
			"started": cls.started,
			"samples": cls.samples,
			"duration": cls.duration,
		}

	def get(self):
		self.set_header("Content-Type", "application/json")
		try:
			now = self.get_argument('now', None)
			if now:
				now = time.time()
				for _action in self.actions:
					if _action['time'] < now \
					and _action['time'] + _action['duration'] > now:
						action = copy.copy(_action)
						self.write(json.dumps({'frame': action, 'now': now},
								   ensure_ascii=False).encode('utf-8'))
						return
				self.write(json.dumps([]))
			else:
				self.write(json.dumps(self.actions, ensure_ascii=False).encode('utf-8'))
		except Exception:
			log.error("Could not send info burst:\n%s", traceback.format_exc())
			log.error("Data:\n%s", self.actions)
			self.write(json.dumps([]))

class TimingHandler(tornado.web.RequestHandler):
	def get(self):
		self.set_header("Content-Type", "application/json")
		self.write(json.dumps({"time": time.time() * 1000}, ensure_ascii=False).encode('utf-8'))

class StreamHandler(tornado.web.RequestHandler):
	relays = []
	listeners = []

	@classmethod
	def relay_url(cls):
		if len(cls.relays) == 1:
			return cls.relays[0].url
		elif len(cls.relays) > 1:
			choices = [relay for relay in cls.relays for _ in xrange(0, relay.weight)]
			return random.choice(choices).url
		else:
			return ""

	@classmethod
	def stream_frames(cls):
		try:
			cls.relays.broadcast()
		except Exception:
			log.error("Could not broadcast due to: \n%s", traceback.format_exc())

	@classmethod
	def check(cls):
		#   TODO: This should do HTTP requests to ensure that all relays are
		#   still up
		pass

	def head(self):
		try:
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			ua = self.request.headers.get('User-Agent', None)
			if ua == config.relay_ua or len(self.relays) == 0:
				log.info("Got HEAD for relay at %s.", ip)
				self.set_header("Content-Type", "audio/mpeg")
				self.finish()
			else:
				if not self.relays:
					tornado.web.RequestHandler.send_error(self, 503)
				else:
					relay = self.relay_url()
					log.info("Redirected new listener %s to %s", ip, relay)
					self.redirect(relay)
		except Exception:
			log.error("Error in stream.head:\n%s", traceback.format_exc())
			tornado.web.RequestHandler.send_error(self, 500)

	@tornado.web.asynchronous
	def get(self):
		try:
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			ua = self.request.headers.get('User-Agent', None)
			if ua == config.relay_ua:
				url = self.request.headers['X-Relay-Addr']
				if not url.startswith('http://'):
					url = "http://" + url
				port = self.request.headers['X-Relay-Port']
				self.weight = int(self.request.headers.get('X-Relay-Weight', 1))
				log.info("Added new relay at %s:%s with weight %d.", url, port, self.weight)
				self.set_header("Content-Type", "audio/mpeg")
				self.url = "%s:%s/all.mp3" % (url, port)
				self.relays.append(self)
			else:
				if self.request.host.startswith("localhost:8193"):
					log.info("Added new debug listener at %s.", ip)
					self.set_header("Content-Type", "audio/mpeg")
					self.relays.append(self)
				elif not self.relays:
					tornado.web.RequestHandler.send_error(self, 503)
				else:
					relay = self.relay_url()
					log.info("Redirected new listener %s to %s", ip, relay)
					self.redirect(relay)
		except Exception:
			log.error("Error in stream.get:\n%s", traceback.format_exc())
			tornado.web.RequestHandler.send_error(self, 500)

	def on_finish(self):
		if self in self.relays:
			self.relays.remove(self)
			ip = self.request.headers.get('X-Real-Ip', self.request.remote_ip)
			log.info("Removed relay at %s with weight %d.", ip, self.weight)

class SocketConnection(tornadio2.conn.SocketConnection):
	__endpoints__ = {
		"/info.websocket": SocketHandler,   #TODO: Rename
		"/monitor.websocket": MonitorSocket
	}

if __name__ == "__main__":
	Daemon()

	log.info("Starting %s...", config.app_name)

	track_queue = multiprocessing.Queue(1)
	log.info("Initializing read queue to hold %2.2f seconds of audio.",
			 config.frontend_buffer)
	v2_queue = BufferedReadQueue(int(config.frontend_buffer / SECONDS_PER_FRAME))
	info_queue = multiprocessing.Queue()

	mixer = Mixer(track_queue,v2_queue.raw,info_queue)
	mixer.start()

	daemonize(database.enqueue_tracks, track_queue)
	daemonize(info.generate, info_queue, first_frame, InfoHandler)
	StreamHandler.relays = Listeners(v2_queue, "All", first_frame)
	daemonize(monitordaemon,StreamHandler.relays,InfoHandler.stats,{"mp3_queue":v2_queue})

	tornado.ioloop.PeriodicCallback(InfoHandler.clean, 5 * 1000).start()
	tornado.ioloop.PeriodicCallback(StreamHandler.check, 10 * 1000).start()

	application = tornado.web.Application(
		tornadio2.TornadioRouter(SocketConnection).apply_routes([
			# Static assets for local development
			(r"/(favicon.ico)", tornado.web.StaticFileHandler, {"path": "static/img/"}),
			(r"/static/(.*)", tornado.web.StaticFileHandler, {"path": "static/"}),

			(r"/timing.json", TimingHandler),

			(r"/all.json", InfoHandler),
			(r"/all.mp3", StreamHandler),

			(r"/monitor", MonitorHandler),
			(r"/", MainHandler),
		]),
		socket_io_port=config.socket_port,
		enabled_protocols=['websocket', 'xhr-multipart', 'xhr-polling', 'jsonp-polling']
	)

	frame_sender = tornado.ioloop.PeriodicCallback(
		StreamHandler.stream_frames, SECONDS_PER_FRAME * 1000
	)
	frame_sender.start()

	application.listen(config.http_port)
	tornadio2.server.SocketServer(application)
<|endoftext|>


<commit_before>'''
Modular Input Script

Copyright (C) 2012 Splunk, Inc.
All Rights Reserved

'''

import sys,logging
import xml.dom.minidom, xml.sax.saxutils

#set up logging
logging.root
logging.root.setLevel(logging.ERROR)
formatter = logging.Formatter('%(levelname)s %(message)s')
#with zero args , should go to STD ERR
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.root.addHandler(handler)

SCHEME = """<scheme>
    <title>xxx</title>
    <description>xxx</description>
    <use_external_validation>true</use_external_validation>
    <streaming_mode>xml</streaming_mode>
    <use_single_instance>false</use_single_instance>

    <endpoint>
        <args>
            <arg name="name">
                <title>xxx</title>
                <description>Name of this input</description>
            </arg>

            <arg name="xxx">
                <title>xxx</title>
                <description>xxx</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>

        </args>
    </endpoint>
</scheme>
"""

def do_validate():
    config = get_validation_config()
    #TODO
    #if error , print_validation_error & sys.exit(2)

def do_run():
    config = get_input_config()
    #TODO , poll for data and print output to STD OUT
    #if error , logger.error & sys.exit(2)

# prints validation error data to be consumed by Splunk
def print_validation_error(s):
    print "<error><message>%s</message></error>" % xml.sax.saxutils.escape(s)

# prints XML stream
def print_xml_single_instance_mode(s):
    print "<stream><event><data>%s</data></event></stream>" % xml.sax.saxutils.escape(s)

# prints XML stream
def print_xml_multi_instance_mode(s,stanza):
    print "<stream><event stanza=""%s""><data>%s</data></event></stream>" % stanza,xml.sax.saxutils.escape(s)

# prints simple stream
def print_simple(s):
    print "%s\n" % s

def usage():
    print "usage: %s [--scheme|--validate-arguments]"
    logging.error("Incorrect Program usaae")
    sys.exit(2)

def do_scheme():
    print SCHEME

#read XML configuration passed from splunkd, need to refactor to support single instance mode
def get_input_config():
    config = {}

    try:
        # read everything from stdin
        config_str = sys.stdin.read()

        # parse the config XML
        doc = xml.dom.minidom.parseString(config_str)
        root = doc.documentElement
        conf_node = root.getElementsByTagName("configuration")[0]
        if conf_node:
            logging.debug("XML: found configuration")
            stanza = conf_node.getElementsByTagName("stanza")[0]
            if stanza:
                stanza_name = stanza.getAttribute("name")
                if stanza_name:
                    logging.debug("XML: found stanza " + stanza_name)
                    config["name"] = stanza_name

                    params = stanza.getElementsByTagName("param")
                    for param in params:
                        param_name = param.getAttribute("name")
                        logging.debug("XML: found param '%s'" % param_name)
                        if param_name and param.firstChild and \
                           param.firstChild.nodeType == param.firstChild.TEXT_NODE:
                            data = param.firstChild.data
                            config[param_name] = data
                            logging.debug("XML: '%s' -> '%s'" % (param_name, data))

        checkpnt_node = root.getElementsByTagName("checkpoint_dir")[0]
        if checkpnt_node and checkpnt_node.firstChild and \
           checkpnt_node.firstChild.nodeType == checkpnt_node.firstChild.TEXT_NODE:
            config["checkpoint_dir"] = checkpnt_node.firstChild.data

        if not config:
            raise Exception, "Invalid configuration received from Splunk."

    except Exception, e:
        raise Exception, "Error getting Splunk configuration via STDIN: %s" % str(e)

    return config

#read XML configuration passed from splunkd, need to refactor to support single instance mode
def get_validation_config():
    val_data = {}

    # read everything from stdin
    val_str = sys.stdin.read()

    # parse the validation XML
    doc = xml.dom.minidom.parseString(val_str)
    root = doc.documentElement

    logging.debug("XML: found items")
    item_node = root.getElementsByTagName("item")[0]
    if item_node:
        logging.debug("XML: found item")

        name = item_node.getAttribute("name")
        val_data["stanza"] = name

        params_node = item_node.getElementsByTagName("param")
        for param in params_node:
            name = param.getAttribute("name")
            logging.debug("Found param %s" % name)
            if name and param.firstChild and \
               param.firstChild.nodeType == param.firstChild.TEXT_NODE:
                val_data[name] = param.firstChild.data

    return val_data

if __name__ == '__main__':

    if len(sys.argv) > 1:
        if sys.argv[1] == "--scheme":
            do_scheme()
        elif sys.argv[1] == "--validate-arguments":
            do_validate()
        else:
            usage()
    else:
        do_run()

    sys.exit(0)<commit_msg>typo in logging message<commit_after>'''
Modular Input Script

Copyright (C) 2012 Splunk, Inc.
All Rights Reserved

'''

import sys,logging
import xml.dom.minidom, xml.sax.saxutils

#set up logging
logging.root
logging.root.setLevel(logging.ERROR)
formatter = logging.Formatter('%(levelname)s %(message)s')
#with zero args , should go to STD ERR
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.root.addHandler(handler)

SCHEME = """<scheme>
    <title>xxx</title>
    <description>xxx</description>
    <use_external_validation>true</use_external_validation>
    <streaming_mode>xml</streaming_mode>
    <use_single_instance>false</use_single_instance>

    <endpoint>
        <args>
            <arg name="name">
                <title>xxx</title>
                <description>Name of this input</description>
            </arg>

            <arg name="xxx">
                <title>xxx</title>
                <description>xxx</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>

        </args>
    </endpoint>
</scheme>
"""

def do_validate():
    config = get_validation_config()
    #TODO
    #if error , print_validation_error & sys.exit(2)

def do_run():
    config = get_input_config()
    #TODO , poll for data and print output to STD OUT
    #if error , logger.error & sys.exit(2)

# prints validation error data to be consumed by Splunk
def print_validation_error(s):
    print "<error><message>%s</message></error>" % xml.sax.saxutils.escape(s)

# prints XML stream
def print_xml_single_instance_mode(s):
    print "<stream><event><data>%s</data></event></stream>" % xml.sax.saxutils.escape(s)

# prints XML stream
def print_xml_multi_instance_mode(s,stanza):
    print "<stream><event stanza=""%s""><data>%s</data></event></stream>" % stanza,xml.sax.saxutils.escape(s)

# prints simple stream
def print_simple(s):
    print "%s\n" % s

def usage():
    print "usage: %s [--scheme|--validate-arguments]"
    logging.error("Incorrect Program Usage")
    sys.exit(2)

def do_scheme():
    print SCHEME

#read XML configuration passed from splunkd, need to refactor to support single instance mode
def get_input_config():
    config = {}

    try:
        # read everything from stdin
        config_str = sys.stdin.read()

        # parse the config XML
        doc = xml.dom.minidom.parseString(config_str)
        root = doc.documentElement
        conf_node = root.getElementsByTagName("configuration")[0]
        if conf_node:
            logging.debug("XML: found configuration")
            stanza = conf_node.getElementsByTagName("stanza")[0]
            if stanza:
                stanza_name = stanza.getAttribute("name")
                if stanza_name:
                    logging.debug("XML: found stanza " + stanza_name)
                    config["name"] = stanza_name

                    params = stanza.getElementsByTagName("param")
                    for param in params:
                        param_name = param.getAttribute("name")
                        logging.debug("XML: found param '%s'" % param_name)
                        if param_name and param.firstChild and \
                           param.firstChild.nodeType == param.firstChild.TEXT_NODE:
                            data = param.firstChild.data
                            config[param_name] = data
                            logging.debug("XML: '%s' -> '%s'" % (param_name, data))

        checkpnt_node = root.getElementsByTagName("checkpoint_dir")[0]
        if checkpnt_node and checkpnt_node.firstChild and \
           checkpnt_node.firstChild.nodeType == checkpnt_node.firstChild.TEXT_NODE:
            config["checkpoint_dir"] = checkpnt_node.firstChild.data

        if not config:
            raise Exception, "Invalid configuration received from Splunk."

    except Exception, e:
        raise Exception, "Error getting Splunk configuration via STDIN: %s" % str(e)

    return config

#read XML configuration passed from splunkd, need to refactor to support single instance mode
def get_validation_config():
    val_data = {}

    # read everything from stdin
    val_str = sys.stdin.read()

    # parse the validation XML
    doc = xml.dom.minidom.parseString(val_str)
    root = doc.documentElement

    logging.debug("XML: found items")
    item_node = root.getElementsByTagName("item")[0]
    if item_node:
        logging.debug("XML: found item")

        name = item_node.getAttribute("name")
        val_data["stanza"] = name

        params_node = item_node.getElementsByTagName("param")
        for param in params_node:
            name = param.getAttribute("name")
            logging.debug("Found param %s" % name)
            if name and param.firstChild and \
               param.firstChild.nodeType == param.firstChild.TEXT_NODE:
                val_data[name] = param.firstChild.data

    return val_data

if __name__ == '__main__':

    if len(sys.argv) > 1:
        if sys.argv[1] == "--scheme":
            do_scheme()
        elif sys.argv[1] == "--validate-arguments":
            do_validate()
        else:
            usage()
    else:
        do_run()

    sys.exit(0)<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
################################################################################
#
#
################################################################################

# =============================================================================
#
# =============================================================================

import warnings

from fate_arch.session import computing_session as session
from fate_arch.common import log
from fate_arch.federation import segment_transfer_enabled
from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.criterion import XgboostCriterion
from federatedml.util import consts

LOGGER = log.getLogger()

class SplitInfo(object):
    def __init__(self, sitename=consts.GUEST, best_fid=None, best_bid=None,
                 sum_grad=0, sum_hess=0, gain=None, missing_dir=1, left_sample_count=0):
        self.sitename = sitename
        self.best_fid = best_fid
        self.best_bid = best_bid
        self.sum_grad = sum_grad
        self.sum_hess = sum_hess
        self.gain = gain
        self.missing_dir = missing_dir

    def __str__(self):
        return '**fid {}, bid {}, sum_grad{}, sum_hess{}, gain{}**'.format(self.best_fid, self.best_bid, \
                self.sum_grad, self.sum_hess, self.gain)

class Splitter(object):

    def __init__(self, criterion_method, criterion_params=[0, 1], min_impurity_split=1e-2, min_sample_split=2,
                 min_leaf_node=1):
        LOGGER.info("splitter init!")
        if not isinstance(criterion_method, str):
            raise TypeError("criterion_method type should be str, but %s find" % (type(criterion_method).__name__))

        if criterion_method == "xgboost":
            if not criterion_params:
                self.criterion = XgboostCriterion()
            else:
                try:
                    reg_lambda = float(criterion_params[0])
                    self.criterion = XgboostCriterion(reg_lambda)
                except:
                    warnings.warn("criterion_params' first criterion_params should be numeric")
                    self.criterion = XgboostCriterion()

        self.min_impurity_split = min_impurity_split
        self.min_sample_split = min_sample_split
        self.min_leaf_node = min_leaf_node

    def find_split_single_histogram_guest(self, histogram, valid_features, sitename, use_missing, zero_as_missing):

        best_fid = None
        best_gain = self.min_impurity_split - consts.FLOAT_ZERO
        best_bid = None
        best_sum_grad_l = None
        best_sum_hess_l = None
        missing_bin = 0
        if use_missing:
            missing_bin = 1

        # in default, missing value going to right
        missing_dir = 1

        for fid in range(len(histogram)):
            if valid_features[fid] is False:
                continue
            bin_num = len(histogram[fid])
            if bin_num == 0 + missing_bin:
                continue
            sum_grad = histogram[fid][bin_num - 1][0]
            sum_hess = histogram[fid][bin_num - 1][1]
            node_cnt = histogram[fid][bin_num - 1][2]

            if node_cnt < self.min_sample_split:
                break

            for bid in range(bin_num - missing_bin - 1):
                sum_grad_l = histogram[fid][bid][0]
                sum_hess_l = histogram[fid][bid][1]
                node_cnt_l = histogram[fid][bid][2]

                sum_grad_r = sum_grad - sum_grad_l
                sum_hess_r = sum_hess - sum_hess_l
                node_cnt_r = node_cnt - node_cnt_l

                if node_cnt_l >= self.min_leaf_node and node_cnt_r >= self.min_leaf_node:
                    gain = self.criterion.split_gain([sum_grad, sum_hess],
                                                     [sum_grad_l, sum_hess_l], [sum_grad_r, sum_hess_r])

                    if gain > self.min_impurity_split and gain > best_gain:

                        best_gain = gain
                        best_fid = fid
                        best_bid = bid
                        best_sum_grad_l = sum_grad_l
                        best_sum_hess_l = sum_hess_l
                        missing_dir = 1

                """ missing value handle: dispatch to left child"""
                if use_missing:
                    sum_grad_l += histogram[fid][-1][0] - histogram[fid][-2][0]
                    sum_hess_l += histogram[fid][-1][1] - histogram[fid][-2][1]
                    node_cnt_l += histogram[fid][-1][2] - histogram[fid][-2][2]

                    sum_grad_r -= histogram[fid][-1][0] - histogram[fid][-2][0]
                    sum_hess_r -= histogram[fid][-1][1] - histogram[fid][-2][1]
                    node_cnt_r -= histogram[fid][-1][2] - histogram[fid][-2][2]

                    if node_cnt_l >= self.min_leaf_node and node_cnt_r >= self.min_leaf_node:
                        gain = self.criterion.split_gain([sum_grad, sum_hess],
                                                         [sum_grad_l, sum_hess_l], [sum_grad_r, sum_hess_r])

                        if gain > self.min_impurity_split and gain > best_gain:
                            best_gain = gain
                            best_fid = fid
                            best_bid = bid
                            best_sum_grad_l = sum_grad_l
                            best_sum_hess_l = sum_hess_l
                            missing_dir = -1

        splitinfo = SplitInfo(sitename=sitename, best_fid=best_fid, best_bid=best_bid,
                              gain=best_gain, sum_grad=best_sum_grad_l, sum_hess=best_sum_hess_l,
                              missing_dir=missing_dir)

        return splitinfo

    def find_split(self, histograms, valid_features, partitions=1, sitename=consts.GUEST,
                   use_missing=False, zero_as_missing=False):
        LOGGER.info("splitter find split of raw data")
        histogram_table = session.parallelize(histograms, include_key=False, partition=partitions)
        splitinfo_table = histogram_table.mapValues(lambda sub_hist:
                                                    self.find_split_single_histogram_guest(sub_hist,
                                                                                valid_features,
                                                                                sitename,
                                                                                use_missing,
                                                                                zero_as_missing))

        tree_node_splitinfo = [None for i in range(len(histograms))]
        for id, splitinfo in splitinfo_table.collect():
            tree_node_splitinfo[id] = splitinfo

        # tree_node_splitinfo = [splitinfo[1] for splitinfo in splitinfo_table.collect()]

        return tree_node_splitinfo

    def find_split_single_histogram_host(self, fid_with_histogram, valid_features, sitename, use_missing=False,
                                         zero_as_missing=False):
        node_splitinfo = []
        node_grad_hess = []

        missing_bin = 0
        if use_missing:
            missing_bin = 1

        fid, histogram = fid_with_histogram
        if valid_features[fid] is False:
            return [], []
        bin_num = len(histogram)
        if bin_num == 0:
            return [], []

        node_cnt = histogram[bin_num - 1][2]

        if node_cnt < self.min_sample_split:
            return [], []

        for bid in range(bin_num - missing_bin - 1):
            sum_grad_l = histogram[bid][0]
            sum_hess_l = histogram[bid][1]
            node_cnt_l = histogram[bid][2]

            node_cnt_r = node_cnt - node_cnt_l

            if node_cnt_l >= self.min_leaf_node and node_cnt_r >= self.min_leaf_node:
                splitinfo = SplitInfo(sitename=sitename, best_fid=fid,
                                      best_bid=bid, sum_grad=sum_grad_l, sum_hess=sum_hess_l,
                                      missing_dir=1)

                node_splitinfo.append(splitinfo)
                node_grad_hess.append((sum_grad_l, sum_hess_l))

            if use_missing:
                sum_grad_l += histogram[-1][0] - histogram[-2][0]
                sum_hess_l += histogram[-1][1] - histogram[-2][1]
                node_cnt_l += histogram[-1][2] - histogram[-2][2]

                splitinfo = SplitInfo(sitename=sitename, best_fid=fid,
                                      best_bid=bid, sum_grad=sum_grad_l, sum_hess=sum_hess_l,
                                      missing_dir=-1)

                node_splitinfo.append(splitinfo)
                node_grad_hess.append((sum_grad_l, sum_hess_l))

        return node_splitinfo, node_grad_hess

    def find_split_host(self, histograms, valid_features, node_map, sitename=consts.HOST,
                        use_missing=False, zero_as_missing=False):
        LOGGER.info("splitter find split of host")
        tree_node_splitinfo = [[] for i in range(len(node_map))]
        encrypted_node_grad_hess = [[] for i in range(len(node_map))]
        host_splitinfo_table = histograms.mapValues(lambda fid_with_hist:
                                                    self.find_split_single_histogram_host(fid_with_hist, valid_features,
                                                                                sitename,
                                                                                use_missing,
                                                                                zero_as_missing))

        for (nid, fid), splitinfo in host_splitinfo_table.collect():
            tree_node_splitinfo[nid].extend(splitinfo[0])
            encrypted_node_grad_hess[nid].extend(splitinfo[1])

        return tree_node_splitinfo, BigObjectTransfer(encrypted_node_grad_hess)

    def node_gain(self, grad, hess):
        return self.criterion.node_gain(grad, hess)

    def node_weight(self, grad, hess):
        return self.criterion.node_weight(grad, hess)

    def split_gain(self, sum_grad, sum_hess, sum_grad_l, sum_hess_l, sum_grad_r, sum_hess_r):
        gain = self.criterion.split_gain([sum_grad, sum_hess], \
                                         [sum_grad_l, sum_hess_l], [sum_grad_r, sum_hess_r])
        return gain

class BigObjectTransfer(metaclass=segment_transfer_enabled()):
    def __init__(self, data):
        self._obj = data

    def get_data(self):
        return self._obj
<commit_msg>restore<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
################################################################################
#
#
################################################################################

# =============================================================================
#
# =============================================================================

import warnings

from fate_arch.session import computing_session as session
from fate_arch.common import log
from fate_arch.federation import segment_transfer_enabled
from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.criterion import XgboostCriterion
from federatedml.util import consts

LOGGER = log.getLogger()

class SplitInfo(object):
    def __init__(self, sitename=consts.GUEST, best_fid=None, best_bid=None,
                 sum_grad=0, sum_hess=0, gain=None, missing_dir=1, left_sample_count=0):
        self.sitename = sitename
        self.best_fid = best_fid
        self.best_bid = best_bid
        self.sum_grad = sum_grad
        self.sum_hess = sum_hess
        self.gain = gain
        self.missing_dir = missing_dir

    def __str__(self):
        return '**fid {}, bid {}, sum_grad{}, sum_hess{}, gain{}**'.format(self.best_fid, self.best_bid, \
                self.sum_grad, self.sum_hess, self.gain)

class Splitter(object):

    def __init__(self, criterion_method, criterion_params=[0, 1], min_impurity_split=1e-2, min_sample_split=2,
                 min_leaf_node=1):
        LOGGER.info("splitter init!")
        if not isinstance(criterion_method, str):
            raise TypeError("criterion_method type should be str, but %s find" % (type(criterion_method).__name__))

        if criterion_method == "xgboost":
            if not criterion_params:
                self.criterion = XgboostCriterion()
            else:
                try:
                    reg_lambda = float(criterion_params[0])
                    self.criterion = XgboostCriterion(reg_lambda)
                except:
                    warnings.warn("criterion_params' first criterion_params should be numeric")
                    self.criterion = XgboostCriterion()

        self.min_impurity_split = min_impurity_split
        self.min_sample_split = min_sample_split
        self.min_leaf_node = min_leaf_node

    def find_split_single_histogram_guest(self, histogram, valid_features, sitename, use_missing, zero_as_missing):

        best_fid = None
        best_gain = self.min_impurity_split - consts.FLOAT_ZERO
        best_bid = None
        best_sum_grad_l = None
        best_sum_hess_l = None
        missing_bin = 0
        if use_missing:
            missing_bin = 1

        # in default, missing value going to right
        missing_dir = 1

        for fid in range(len(histogram)):
            if valid_features[fid] is False:
                continue
            bin_num = len(histogram[fid])
            if bin_num == 0 + missing_bin:
                continue
            sum_grad = histogram[fid][bin_num - 1][0]
            sum_hess = histogram[fid][bin_num - 1][1]
            node_cnt = histogram[fid][bin_num - 1][2]

            if node_cnt < self.min_sample_split:
                break

            for bid in range(bin_num - missing_bin - 1):
                sum_grad_l = histogram[fid][bid][0]
                sum_hess_l = histogram[fid][bid][1]
                node_cnt_l = histogram[fid][bid][2]

                sum_grad_r = sum_grad - sum_grad_l
                sum_hess_r = sum_hess - sum_hess_l
                node_cnt_r = node_cnt - node_cnt_l

                if node_cnt_l >= self.min_leaf_node and node_cnt_r >= self.min_leaf_node:
                    gain = self.criterion.split_gain([sum_grad, sum_hess],
                                                     [sum_grad_l, sum_hess_l], [sum_grad_r, sum_hess_r])

                    if gain > self.min_impurity_split and gain > best_gain + consts.FLOAT_ZERO:

                        best_gain = gain
                        best_fid = fid
                        best_bid = bid
                        best_sum_grad_l = sum_grad_l
                        best_sum_hess_l = sum_hess_l
                        missing_dir = 1

                """ missing value handle: dispatch to left child"""
                if use_missing:
                    sum_grad_l += histogram[fid][-1][0] - histogram[fid][-2][0]
                    sum_hess_l += histogram[fid][-1][1] - histogram[fid][-2][1]
                    node_cnt_l += histogram[fid][-1][2] - histogram[fid][-2][2]

                    sum_grad_r -= histogram[fid][-1][0] - histogram[fid][-2][0]
                    sum_hess_r -= histogram[fid][-1][1] - histogram[fid][-2][1]
                    node_cnt_r -= histogram[fid][-1][2] - histogram[fid][-2][2]

                    if node_cnt_l >= self.min_leaf_node and node_cnt_r >= self.min_leaf_node:
                        gain = self.criterion.split_gain([sum_grad, sum_hess],
                                                         [sum_grad_l, sum_hess_l], [sum_grad_r, sum_hess_r])

                        if gain > self.min_impurity_split and gain > best_gain:
                            best_gain = gain
                            best_fid = fid
                            best_bid = bid
                            best_sum_grad_l = sum_grad_l
                            best_sum_hess_l = sum_hess_l
                            missing_dir = -1

        splitinfo = SplitInfo(sitename=sitename, best_fid=best_fid, best_bid=best_bid,
                              gain=best_gain, sum_grad=best_sum_grad_l, sum_hess=best_sum_hess_l,
                              missing_dir=missing_dir)

        return splitinfo

    def find_split(self, histograms, valid_features, partitions=1, sitename=consts.GUEST,
                   use_missing=False, zero_as_missing=False):
        LOGGER.info("splitter find split of raw data")
        histogram_table = session.parallelize(histograms, include_key=False, partition=partitions)
        splitinfo_table = histogram_table.mapValues(lambda sub_hist:
                                                    self.find_split_single_histogram_guest(sub_hist,
                                                                                valid_features,
                                                                                sitename,
                                                                                use_missing,
                                                                                zero_as_missing))

        tree_node_splitinfo = [None for i in range(len(histograms))]
        for id, splitinfo in splitinfo_table.collect():
            tree_node_splitinfo[id] = splitinfo

        # tree_node_splitinfo = [splitinfo[1] for splitinfo in splitinfo_table.collect()]

        return tree_node_splitinfo

    def find_split_single_histogram_host(self, fid_with_histogram, valid_features, sitename, use_missing=False,
                                         zero_as_missing=False):
        node_splitinfo = []
        node_grad_hess = []

        missing_bin = 0
        if use_missing:
            missing_bin = 1

        fid, histogram = fid_with_histogram
        if valid_features[fid] is False:
            return [], []
        bin_num = len(histogram)
        if bin_num == 0:
            return [], []

        node_cnt = histogram[bin_num - 1][2]

        if node_cnt < self.min_sample_split:
            return [], []

        for bid in range(bin_num - missing_bin - 1):
            sum_grad_l = histogram[bid][0]
            sum_hess_l = histogram[bid][1]
            node_cnt_l = histogram[bid][2]

            node_cnt_r = node_cnt - node_cnt_l

            if node_cnt_l >= self.min_leaf_node and node_cnt_r >= self.min_leaf_node:
                splitinfo = SplitInfo(sitename=sitename, best_fid=fid,
                                      best_bid=bid, sum_grad=sum_grad_l, sum_hess=sum_hess_l,
                                      missing_dir=1)

                node_splitinfo.append(splitinfo)
                node_grad_hess.append((sum_grad_l, sum_hess_l))

            if use_missing:
                sum_grad_l += histogram[-1][0] - histogram[-2][0]
                sum_hess_l += histogram[-1][1] - histogram[-2][1]
                node_cnt_l += histogram[-1][2] - histogram[-2][2]

                splitinfo = SplitInfo(sitename=sitename, best_fid=fid,
                                      best_bid=bid, sum_grad=sum_grad_l, sum_hess=sum_hess_l,
                                      missing_dir=-1)

                node_splitinfo.append(splitinfo)
                node_grad_hess.append((sum_grad_l, sum_hess_l))

        return node_splitinfo, node_grad_hess

    def find_split_host(self, histograms, valid_features, node_map, sitename=consts.HOST,
                        use_missing=False, zero_as_missing=False):
        LOGGER.info("splitter find split of host")
        tree_node_splitinfo = [[] for i in range(len(node_map))]
        encrypted_node_grad_hess = [[] for i in range(len(node_map))]
        host_splitinfo_table = histograms.mapValues(lambda fid_with_hist:
                                                    self.find_split_single_histogram_host(fid_with_hist, valid_features,
                                                                                sitename,
                                                                                use_missing,
                                                                                zero_as_missing))

        for (nid, fid), splitinfo in host_splitinfo_table.collect():
            tree_node_splitinfo[nid].extend(splitinfo[0])
            encrypted_node_grad_hess[nid].extend(splitinfo[1])

        return tree_node_splitinfo, BigObjectTransfer(encrypted_node_grad_hess)

    def node_gain(self, grad, hess):
        return self.criterion.node_gain(grad, hess)

    def node_weight(self, grad, hess):
        return self.criterion.node_weight(grad, hess)

    def split_gain(self, sum_grad, sum_hess, sum_grad_l, sum_hess_l, sum_grad_r, sum_hess_r):
        gain = self.criterion.split_gain([sum_grad, sum_hess], \
                                         [sum_grad_l, sum_hess_l], [sum_grad_r, sum_hess_r])
        return gain

class BigObjectTransfer(metaclass=segment_transfer_enabled()):
    def __init__(self, data):
        self._obj = data

    def get_data(self):
        return self._obj
<|endoftext|>


<commit_before>__author__ = 'moonkey'

import sys, argparse, logging

import numpy as np
from PIL import Image
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)

from model.model import Model
import exp_config

def process_args(args, defaults):
    parser = argparse.ArgumentParser()

    parser.add_argument('--gpu-id', dest="gpu_id",
                        type=int, default=defaults.GPU_ID)

    parser.add_argument('--use-gru', dest='use_gru', action='store_true')

    parser.add_argument('--phase', dest="phase",
                        type=str, default=defaults.PHASE,
                        choices=['train', 'test'],
                        help=('Phase of experiment, can be either'
                            ' train or test, default=%s'%(defaults.PHASE)))
    parser.add_argument('--data-path', dest="data_path",
                        type=str, default=defaults.DATA_PATH,
                        help=('Path of file containing the path and labels'
                            ' of training or testing data, default=%s'
                            %(defaults.DATA_PATH)))
    parser.add_argument('--data-base-dir', dest="data_base_dir",
                        type=str, default=defaults.DATA_BASE_DIR,
                        help=('The base directory of the paths in the file '
                            'containing the path and labels, default=%s'
                            %(defaults.DATA_PATH)))
    parser.add_argument('--visualize', dest='visualize', action='store_true',
                        help=('Visualize attentions or not'
                            ', default=%s' %(defaults.VISUALIZE)))
    parser.add_argument('--no-visualize', dest='visualize', action='store_false')
    parser.set_defaults(visualize=defaults.VISUALIZE)
    parser.add_argument('--batch-size', dest="batch_size",
                        type=int, default=defaults.BATCH_SIZE,
                        help=('Batch size, default = %s'
                            %(defaults.BATCH_SIZE)))
    parser.add_argument('--initial-learning-rate', dest="initial_learning_rate",
                        type=float, default=defaults.INITIAL_LEARNING_RATE,
                        help=('Initial learning rate, default = %s'
                            %(defaults.INITIAL_LEARNING_RATE)))
    parser.add_argument('--num-epoch', dest="num_epoch",
                        type=int, default=defaults.NUM_EPOCH,
                        help=('Number of epochs, default = %s'
                            %(defaults.NUM_EPOCH)))
    parser.add_argument('--steps-per-checkpoint', dest="steps_per_checkpoint",
                        type=int, default=defaults.STEPS_PER_CHECKPOINT,
                        help=('Checkpointing (print perplexity, save model) per'
                            ' how many steps, default = %s'
                            %(defaults.STEPS_PER_CHECKPOINT)))
    parser.add_argument('--target-vocab-size', dest="target_vocab_size",
                        type=int, default=defaults.TARGET_VOCAB_SIZE,
                        help=('Target vocabulary size, default=%s'
                            %(defaults.TARGET_VOCAB_SIZE)))
    parser.add_argument('--model-dir', dest="model_dir",
                        type=str, default=defaults.MODEL_DIR,
                        help=('The directory for saving and loading model '
                            '(structure is not stored), '
                            'default=%s' %(defaults.MODEL_DIR)))
    parser.add_argument('--target-embedding-size', dest="target_embedding_size",
                        type=int, default=defaults.TARGET_EMBEDDING_SIZE,
                        help=('Embedding dimension for each target, default=%s'
                            %(defaults.TARGET_EMBEDDING_SIZE)))
    parser.add_argument('--attn-num-hidden', dest="attn_num_hidden",
                        type=int, default=defaults.ATTN_NUM_HIDDEN,
                        help=('number of hidden units in attention decoder cell'
                            ', default=%s'
                            %(defaults.ATTN_NUM_HIDDEN)))
    parser.add_argument('--attn-num-layers', dest="attn_num_layers",
                        type=int, default=defaults.ATTN_NUM_LAYERS,
                        help=('number of hidden layers in attention decoder cell'
                            ', default=%s'
                            %(defaults.ATTN_NUM_LAYERS)))
    parser.add_argument('--load-model', dest='load_model', action='store_true',
                        help=('Load model from model-dir or not'
                            ', default=%s' %(defaults.LOAD_MODEL)))
    parser.add_argument('--no-load-model', dest='load_model', action='store_false')
    parser.set_defaults(load_model=defaults.LOAD_MODEL)
    parser.add_argument('--log-path', dest="log_path",
                        type=str, default=defaults.LOG_PATH,
                        help=('Log file path, default=%s'
                            %(defaults.LOG_PATH)))
    parser.add_argument('--output-dir', dest="output_dir",
                        type=str, default=defaults.OUTPUT_DIR,
                        help=('Output directory, default=%s'
                            %(defaults.OUTPUT_DIR)))
    parser.add_argument('--reg-val', dest="reg_val",
                        type=float, default=defaults.REG_VAL,
                        help=('Use L2 regularization losses.'
                              ', default=%s'
                              % (defaults.REG_VAL)))
    parser.add_argument('--max-gradient-norm', dest="max_gradient_norm",
                        type=int, default=defaults.MAX_GRADIENT_NORM,
                        help=('Clip gradients to this norm.'
                              ', default=%s'
                              % (defaults.MAX_GRADIENT_NORM)))
    parser.add_argument('--augmentation', dest='augmentation',
                        type=float, default=defaults.AUGMENTATION,
                        help=('P(image augmentation).'
                              ', default=%s'
                              % (defaults.AUGMENTATION)))
    parser.set_defaults(augmentation=defaults.AUGMENTATION)
    parser.add_argument('--opt-attn', dest="opt_attn",
                        type=str, default=defaults.OPT_ATTN,
                        choices=['softmax','log_softmax', 'sigmoid', 'no_attn'],
                        help=('Attention mechanism used for decoder, default=%s'
                            %(defaults.OPT_ATTN)))
    parser.set_defaults(opt_attn=defaults.OPT_ATTN)
    parser.add_argument('--no-gradient_clipping', dest='clip_gradients', action='store_false',
                        help=('Do not perform gradient clipping, default for clip_gradients is %s' %
                              (defaults.CLIP_GRADIENTS)))
    parser.set_defaults(clip_gradients=defaults.CLIP_GRADIENTS)

    parameters = parser.parse_args(args)
    return parameters

def main(args, defaults):
    parameters = process_args(args, defaults)
    logging.basicConfig(
        level=logging.DEBUG,
        format='%(asctime)-15s %(name)-5s %(levelname)-8s %(message)s',
        filename=parameters.log_path)
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)-15s %(name)-5s %(levelname)-8s %(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)

    with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
        model = Model(
                phase = parameters.phase,
                visualize = parameters.visualize,
                data_path = parameters.data_path,
                data_base_dir = parameters.data_base_dir,
                output_dir = parameters.output_dir,
                batch_size = parameters.batch_size,
                initial_learning_rate = parameters.initial_learning_rate,
                num_epoch = parameters.num_epoch,
                steps_per_checkpoint = parameters.steps_per_checkpoint,
                target_vocab_size = parameters.target_vocab_size,
                model_dir = parameters.model_dir,
                target_embedding_size = parameters.target_embedding_size,
                attn_num_hidden = parameters.attn_num_hidden,
                attn_num_layers = parameters.attn_num_layers,
                clip_gradients = parameters.clip_gradients,
                reg_val = parameters.reg_val,
                max_gradient_norm = parameters.max_gradient_norm,
                opt_attn = parameters.opt_attn,
                load_model = parameters.load_model,
                valid_target_length = float('inf'),
                gpu_id=parameters.gpu_id,
                use_gru=parameters.use_gru,
                augmentation=parameters.augmentation,
                session = sess)
        model.launch()

if __name__ == "__main__":
    main(sys.argv[1:], exp_config.ExpConfig)

<commit_msg>tf 2 comp<commit_after>__author__ = 'moonkey'

import sys, argparse, logging

import numpy as np
from PIL import Image
import tensorflow as tf
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

from model.model import Model
import exp_config

def process_args(args, defaults):
    parser = argparse.ArgumentParser()

    parser.add_argument('--gpu-id', dest="gpu_id",
                        type=int, default=defaults.GPU_ID)

    parser.add_argument('--use-gru', dest='use_gru', action='store_true')

    parser.add_argument('--phase', dest="phase",
                        type=str, default=defaults.PHASE,
                        choices=['train', 'test'],
                        help=('Phase of experiment, can be either'
                            ' train or test, default=%s'%(defaults.PHASE)))
    parser.add_argument('--data-path', dest="data_path",
                        type=str, default=defaults.DATA_PATH,
                        help=('Path of file containing the path and labels'
                            ' of training or testing data, default=%s'
                            %(defaults.DATA_PATH)))
    parser.add_argument('--data-base-dir', dest="data_base_dir",
                        type=str, default=defaults.DATA_BASE_DIR,
                        help=('The base directory of the paths in the file '
                            'containing the path and labels, default=%s'
                            %(defaults.DATA_PATH)))
    parser.add_argument('--visualize', dest='visualize', action='store_true',
                        help=('Visualize attentions or not'
                            ', default=%s' %(defaults.VISUALIZE)))
    parser.add_argument('--no-visualize', dest='visualize', action='store_false')
    parser.set_defaults(visualize=defaults.VISUALIZE)
    parser.add_argument('--batch-size', dest="batch_size",
                        type=int, default=defaults.BATCH_SIZE,
                        help=('Batch size, default = %s'
                            %(defaults.BATCH_SIZE)))
    parser.add_argument('--initial-learning-rate', dest="initial_learning_rate",
                        type=float, default=defaults.INITIAL_LEARNING_RATE,
                        help=('Initial learning rate, default = %s'
                            %(defaults.INITIAL_LEARNING_RATE)))
    parser.add_argument('--num-epoch', dest="num_epoch",
                        type=int, default=defaults.NUM_EPOCH,
                        help=('Number of epochs, default = %s'
                            %(defaults.NUM_EPOCH)))
    parser.add_argument('--steps-per-checkpoint', dest="steps_per_checkpoint",
                        type=int, default=defaults.STEPS_PER_CHECKPOINT,
                        help=('Checkpointing (print perplexity, save model) per'
                            ' how many steps, default = %s'
                            %(defaults.STEPS_PER_CHECKPOINT)))
    parser.add_argument('--target-vocab-size', dest="target_vocab_size",
                        type=int, default=defaults.TARGET_VOCAB_SIZE,
                        help=('Target vocabulary size, default=%s'
                            %(defaults.TARGET_VOCAB_SIZE)))
    parser.add_argument('--model-dir', dest="model_dir",
                        type=str, default=defaults.MODEL_DIR,
                        help=('The directory for saving and loading model '
                            '(structure is not stored), '
                            'default=%s' %(defaults.MODEL_DIR)))
    parser.add_argument('--target-embedding-size', dest="target_embedding_size",
                        type=int, default=defaults.TARGET_EMBEDDING_SIZE,
                        help=('Embedding dimension for each target, default=%s'
                            %(defaults.TARGET_EMBEDDING_SIZE)))
    parser.add_argument('--attn-num-hidden', dest="attn_num_hidden",
                        type=int, default=defaults.ATTN_NUM_HIDDEN,
                        help=('number of hidden units in attention decoder cell'
                            ', default=%s'
                            %(defaults.ATTN_NUM_HIDDEN)))
    parser.add_argument('--attn-num-layers', dest="attn_num_layers",
                        type=int, default=defaults.ATTN_NUM_LAYERS,
                        help=('number of hidden layers in attention decoder cell'
                            ', default=%s'
                            %(defaults.ATTN_NUM_LAYERS)))
    parser.add_argument('--load-model', dest='load_model', action='store_true',
                        help=('Load model from model-dir or not'
                            ', default=%s' %(defaults.LOAD_MODEL)))
    parser.add_argument('--no-load-model', dest='load_model', action='store_false')
    parser.set_defaults(load_model=defaults.LOAD_MODEL)
    parser.add_argument('--log-path', dest="log_path",
                        type=str, default=defaults.LOG_PATH,
                        help=('Log file path, default=%s'
                            %(defaults.LOG_PATH)))
    parser.add_argument('--output-dir', dest="output_dir",
                        type=str, default=defaults.OUTPUT_DIR,
                        help=('Output directory, default=%s'
                            %(defaults.OUTPUT_DIR)))
    parser.add_argument('--reg-val', dest="reg_val",
                        type=float, default=defaults.REG_VAL,
                        help=('Use L2 regularization losses.'
                              ', default=%s'
                              % (defaults.REG_VAL)))
    parser.add_argument('--max-gradient-norm', dest="max_gradient_norm",
                        type=int, default=defaults.MAX_GRADIENT_NORM,
                        help=('Clip gradients to this norm.'
                              ', default=%s'
                              % (defaults.MAX_GRADIENT_NORM)))
    parser.add_argument('--augmentation', dest='augmentation',
                        type=float, default=defaults.AUGMENTATION,
                        help=('P(image augmentation).'
                              ', default=%s'
                              % (defaults.AUGMENTATION)))
    parser.set_defaults(augmentation=defaults.AUGMENTATION)
    parser.add_argument('--opt-attn', dest="opt_attn",
                        type=str, default=defaults.OPT_ATTN,
                        choices=['softmax','log_softmax', 'sigmoid', 'no_attn'],
                        help=('Attention mechanism used for decoder, default=%s'
                            %(defaults.OPT_ATTN)))
    parser.set_defaults(opt_attn=defaults.OPT_ATTN)
    parser.add_argument('--no-gradient_clipping', dest='clip_gradients', action='store_false',
                        help=('Do not perform gradient clipping, default for clip_gradients is %s' %
                              (defaults.CLIP_GRADIENTS)))
    parser.set_defaults(clip_gradients=defaults.CLIP_GRADIENTS)

    parameters = parser.parse_args(args)
    return parameters

def main(args, defaults):
    parameters = process_args(args, defaults)
    logging.basicConfig(
        level=logging.DEBUG,
        format='%(asctime)-15s %(name)-5s %(levelname)-8s %(message)s',
        filename=parameters.log_path)
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)-15s %(name)-5s %(levelname)-8s %(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)

    with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
        model = Model(
                phase = parameters.phase,
                visualize = parameters.visualize,
                data_path = parameters.data_path,
                data_base_dir = parameters.data_base_dir,
                output_dir = parameters.output_dir,
                batch_size = parameters.batch_size,
                initial_learning_rate = parameters.initial_learning_rate,
                num_epoch = parameters.num_epoch,
                steps_per_checkpoint = parameters.steps_per_checkpoint,
                target_vocab_size = parameters.target_vocab_size,
                model_dir = parameters.model_dir,
                target_embedding_size = parameters.target_embedding_size,
                attn_num_hidden = parameters.attn_num_hidden,
                attn_num_layers = parameters.attn_num_layers,
                clip_gradients = parameters.clip_gradients,
                reg_val = parameters.reg_val,
                max_gradient_norm = parameters.max_gradient_norm,
                opt_attn = parameters.opt_attn,
                load_model = parameters.load_model,
                valid_target_length = float('inf'),
                gpu_id=parameters.gpu_id,
                use_gru=parameters.use_gru,
                augmentation=parameters.augmentation,
                session = sess)
        model.launch()

if __name__ == "__main__":
    main(sys.argv[1:], exp_config.ExpConfig)

<|endoftext|>


<commit_before>"""Utils for foxpy."""

def extractNifPhrases(json_ld):
    graph = json_ld["@graph"]
    return list(filter(lambda x: x["@type"] == "nif:Phrase", graph))

def insertCharacterAtPosition(string, char, position):
    return string[:position] + char + string[position:]
<commit_msg>fix bug when there are no nif phrases<commit_after>"""Utils for foxpy."""

def extractNifPhrases(json_ld):
    if "@graph" in json_ld:
        graph = json_ld["@graph"]
        return list(filter(lambda x: x["@type"] == "nif:Phrase", graph))
    else:
        return []

def insertCharacterAtPosition(string, char, position):
    return string[:position] + char + string[position:]
<|endoftext|>