<commit_before>import os
import sys
from twisted.internet import reactor
from Hubbot import HubbotFactory
from IRCMessage import IRCChannel

class BotHandler:
    botfactories = {}
    connections = {"irc.chatspike:6667":["#DesertBusBunker"], "applejack.me:6667":["#survivors"]}

    def __init__(self):
        for (server_with_port, channels) in self.connections.items():
            server = server_with_port.split(":")[0]
            port = int(server_with_port.split(":")[1])
            self.startBotFactory(server, port, channels)
        reactor.run()

    def startBotFactory(self, server, port, channels):
        if server in self.botfactories:
            print "Already on server '{}'.".format(server)
            return False
        if type(channels) == list:
            chanObjects = {}
            for channel in channels:
                chanObjects[channel] = IRCChannel(channel)
            botfactory = HubbotFactory(server, port, chanObjects, self)
        else:
            botfactory = HubbotFactory(server, port, channels, self)
        self.botfactories[server] = botfactory
        return True

    def stopBotFactory(self, server, quitmessage=None):
        if quitmessage is None:
            self.quitmessage = "ohok".encode("utf-8")
        else:
            self.quitmessage = quitmessage
        if server not in self.botfactories:
            print "ERROR: Bot for '{}' does not exist yet was asked to stop.".format(server)
        else:
            print "Shutting down bot for server '{}'".format(server)
            self.botfactories[server].protocol.Quitting = True
            try:
                self.botfactories[server].protocol.quit(quitmessage)
                for (name, module) in self.botfactories[server].protocol.moduleHandler.modules.items():
                    module.onUnload()
            except:
                self.botfactories[server].stopTrying()
            self.unregisterFactory(server)
            print "Successfully shut down bot for server '{}'".format(server)

    def unregisterFactory(self, server):
        if server in self.botfactories:
            del self.botfactories[server]

            if len(self.botfactories) == 0:
                print "No more running bots, shutting down."
                reactor.callLater(2.0, reactor.stop)

    def shutdown(self, quitmessage="Shutting down..."):
        quitmessage = quitmessage.encode("utf-8")
        for server, botfactory in self.botfactories.iteritems():
            botfactory.protocol.Quitting = True
            botfactory.protocol.quit(quitmessage)
            for (name, module) in botfactory.protocol.moduleHandler.modules.items():
                module.onUnload()
        self.botfactories = {}
        reactor.callLater(4.0, reactor.stop)

    def restart(self, quitmessage="Restarting..."):
        self.quitmessage = quitmessage.encode("utf-8")
        for server, botfactory in self.botfactories.iteritems():
            botfactory.protocol.Quitting = True
            botfactory.protocol.quit(quitmessage)
            for (name, module) in botfactory.protocol.moduleHandler.modules.items():
                module.onUnload()
        self.botfactories = {}
        reactor.callLater(2.0, self.replaceInstance)

    def replaceInstance(self):
        reactor.stop()
        python = sys.executable
        os.execl(python, python, "BotHandler.py")

if __name__ == "__main__":
    bothandler = BotHandler()
<commit_msg>[Core] I swear I know how to words.<commit_after>import os
import sys
from twisted.internet import reactor
from Hubbot import HubbotFactory
from IRCMessage import IRCChannel

class BotHandler:
    botfactories = {}
    connections = {"irc.chatspike.net:6667":["#DesertBusBunker"], "applejack.me:6667":["#survivors"]}

    def __init__(self):
        for (server_with_port, channels) in self.connections.items():
            server = server_with_port.split(":")[0]
            port = int(server_with_port.split(":")[1])
            self.startBotFactory(server, port, channels)
        reactor.run()

    def startBotFactory(self, server, port, channels):
        if server in self.botfactories:
            print "Already on server '{}'.".format(server)
            return False
        if type(channels) == list:
            chanObjects = {}
            for channel in channels:
                chanObjects[channel] = IRCChannel(channel)
            botfactory = HubbotFactory(server, port, chanObjects, self)
        else:
            botfactory = HubbotFactory(server, port, channels, self)
        self.botfactories[server] = botfactory
        return True

    def stopBotFactory(self, server, quitmessage=None):
        if quitmessage is None:
            self.quitmessage = "ohok".encode("utf-8")
        else:
            self.quitmessage = quitmessage
        if server not in self.botfactories:
            print "ERROR: Bot for '{}' does not exist yet was asked to stop.".format(server)
        else:
            print "Shutting down bot for server '{}'".format(server)
            self.botfactories[server].protocol.Quitting = True
            try:
                self.botfactories[server].protocol.quit(quitmessage)
                for (name, module) in self.botfactories[server].protocol.moduleHandler.modules.items():
                    module.onUnload()
            except:
                self.botfactories[server].stopTrying()
            self.unregisterFactory(server)
            print "Successfully shut down bot for server '{}'".format(server)

    def unregisterFactory(self, server):
        if server in self.botfactories:
            del self.botfactories[server]

            if len(self.botfactories) == 0:
                print "No more running bots, shutting down."
                reactor.callLater(2.0, reactor.stop)

    def shutdown(self, quitmessage="Shutting down..."):
        quitmessage = quitmessage.encode("utf-8")
        for server, botfactory in self.botfactories.iteritems():
            botfactory.protocol.Quitting = True
            botfactory.protocol.quit(quitmessage)
            for (name, module) in botfactory.protocol.moduleHandler.modules.items():
                module.onUnload()
        self.botfactories = {}
        reactor.callLater(4.0, reactor.stop)

    def restart(self, quitmessage="Restarting..."):
        self.quitmessage = quitmessage.encode("utf-8")
        for server, botfactory in self.botfactories.iteritems():
            botfactory.protocol.Quitting = True
            botfactory.protocol.quit(quitmessage)
            for (name, module) in botfactory.protocol.moduleHandler.modules.items():
                module.onUnload()
        self.botfactories = {}
        reactor.callLater(2.0, self.replaceInstance)

    def replaceInstance(self):
        reactor.stop()
        python = sys.executable
        os.execl(python, python, "BotHandler.py")

if __name__ == "__main__":
    bothandler = BotHandler()
<|endoftext|>


<commit_before>import logging, os

from DideRobot import DideRobot
import GlobalStore

class BotHandler:
	bots = {}  #Keys are serverfolder names, value is the bot instance for that serverfolder

	def __init__(self, serverfolderList):
		self.logger = logging.getLogger('DideRobot')
		GlobalStore.bothandler = self

		#Since a lot of modules save stuff to the 'data' subfolder, make sure it exists to save all of them some checking time
		if not os.path.exists(os.path.join(GlobalStore.scriptfolder, 'data')):
			os.mkdir(os.path.join(GlobalStore.scriptfolder, 'data'))

		if not os.path.exists(os.path.join(GlobalStore.scriptfolder, 'serverSettings', 'globalsettings.json')):
			self.logger.critical("'globalsettings.json' file not found in 'serverSettings' folder! Shutting down")
			self.shutdown()
		else:
			for serverfolder in serverfolderList:
				self.startBot(serverfolder)

	def startBot(self, serverfolder):
		if serverfolder in self.bots:
			self.logger.warning("BotHandler got command to join server which I'm already on, '{}'".format(serverfolder))
			return False
		if not os.path.exists(os.path.join(GlobalStore.scriptfolder, 'serverSettings', serverfolder)):
			self.logger.error("BotHandler got command to join server '{}', which I don't have settings for".format(serverfolder))
			return False
		#Start the bot, woo!
		self.bots[serverfolder] = DideRobot(serverfolder)
		return True

	def stopBot(self, serverfolder, quitmessage="Quitting..."):
		if serverfolder not in self.bots:
			self.logger.warning("Asked to stop unknown bot '{}'!".format(serverfolder))
			return False
		quitmessage = quitmessage.encode('utf-8')
		self.bots[serverfolder].quit(quitmessage)
		return True

	def unregisterBot(self, serverfolder):
		if serverfolder not in self.bots:
			self.logger.warning("Asked to unregister non-registered bot '{}'".format(serverfolder))
			return
		del self.bots[serverfolder]
		self.logger.info("Successfully unregistered bot '{}'".format(serverfolder))
		#If there's no more bots running, there's no need to hang about
		if len(self.bots) == 0:
			self.logger.info("Out of bots, shutting down!")
			#Unload all commands. This will mean there won't be any greenlets left running, so the main loop will quit
			GlobalStore.commandhandler.unloadAllCommands()

	def shutdown(self, quitmessage='Shutting down...'):
		#Give all bots the same quit message
		quitmessage = quitmessage.encode('utf-8')
		for serverfolder in self.bots.keys():
			self.stopBot(serverfolder, quitmessage)
<commit_msg>Create bots asynchronously<commit_after>import logging, os

import gevent

from DideRobot import DideRobot
import GlobalStore

class BotHandler:
	bots = {}  #Keys are serverfolder names, value is the bot instance for that serverfolder

	def __init__(self, serverfolderList):
		self.logger = logging.getLogger('DideRobot')
		GlobalStore.bothandler = self

		#Since a lot of modules save stuff to the 'data' subfolder, make sure it exists to save all of them some checking time
		if not os.path.exists(os.path.join(GlobalStore.scriptfolder, 'data')):
			os.mkdir(os.path.join(GlobalStore.scriptfolder, 'data'))

		if not os.path.exists(os.path.join(GlobalStore.scriptfolder, 'serverSettings', 'globalsettings.json')):
			self.logger.critical("'globalsettings.json' file not found in 'serverSettings' folder! Shutting down")
			self.shutdown()
		else:
			for serverfolder in serverfolderList:
				gevent.spawn(self.startBot, serverfolder)

	def startBot(self, serverfolder):
		if serverfolder in self.bots:
			self.logger.warning("BotHandler got command to join server which I'm already on, '{}'".format(serverfolder))
			return False
		if not os.path.exists(os.path.join(GlobalStore.scriptfolder, 'serverSettings', serverfolder)):
			self.logger.error("BotHandler got command to join server '{}', which I don't have settings for".format(serverfolder))
			return False
		#Start the bot, woo!
		self.bots[serverfolder] = DideRobot(serverfolder)
		return True

	def stopBot(self, serverfolder, quitmessage="Quitting..."):
		if serverfolder not in self.bots:
			self.logger.warning("Asked to stop unknown bot '{}'!".format(serverfolder))
			return False
		quitmessage = quitmessage.encode('utf-8')
		self.bots[serverfolder].quit(quitmessage)
		return True

	def unregisterBot(self, serverfolder):
		if serverfolder not in self.bots:
			self.logger.warning("Asked to unregister non-registered bot '{}'".format(serverfolder))
			return
		del self.bots[serverfolder]
		self.logger.info("Successfully unregistered bot '{}'".format(serverfolder))
		#If there's no more bots running, there's no need to hang about
		if len(self.bots) == 0:
			self.logger.info("Out of bots, shutting down!")
			#Unload all commands. This will mean there won't be any greenlets left running, so the main loop will quit
			GlobalStore.commandhandler.unloadAllCommands()

	def shutdown(self, quitmessage='Shutting down...'):
		#Give all bots the same quit message
		quitmessage = quitmessage.encode('utf-8')
		for serverfolder in self.bots.keys():
			self.stopBot(serverfolder, quitmessage)
<|endoftext|>


<commit_before>import os
import pdb
import functools
from core import models
from core.ali.oss import BUCKET, oss2key, is_object_exists, is_size_differ_and_newer
from core.misc import *
from colorMessage import dyeWARNING, dyeFAIL
from sqlalchemy import create_engine, UniqueConstraint
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError

class DB(object):
    def __init__(self, db_path, pipe_path, apps, parameters, dependencies, overwrite=False):
        super(DB, self).__init__()
        self.db_path = db_path
        self.proj_path = os.path.dirname(db_path)
        self.pipe_path = pipe_path
        self.apps = apps
        self.parameters = parameters
        self.modules = self.trim_parameters(parameters)
        self.dependencies = dependencies
        self.engine = create_engine('sqlite:///{db_path}'.format(db_path=self.db_path))
        self.proj = None
        if overwrite and os.path.exists(self.db_path):
            os.remove(self.db_path)
        if not os.path.exists(self.db_path):
            models.Base.metadata.create_all(self.engine)
        Session = sessionmaker(bind=self.engine)
        self.session = Session()

    def trim_parameters(self, parameters):
        skips = ['Inputs', 'Property', 'Parameters', 'CommonData', 'Samples', 'Outputs', 'Conditions', 'CommonParameters']
        return {k:v for k, v in parameters.iteritems() if k not in skips}

    def format(self):
        self.mkProj()
        self.mkInstance()
        # Full Module
        map(self.mkModule, self.modules.keys())
        self.mkDepends()
        self.session.commit()

    def add(self):
        snap_db_list = os.path.expanduser("~/.snap/db.yaml")
        db_list = {}
        if os.path.exists(snap_db_list):
            db_list = loadYaml(snap_db_list)
        if db_list is None:
            db_list = {}
        contract_id = self.parameters['CommonParameters']['ContractID']
        db_list[contract_id] = self.db_path
        dumpYaml(snap_db_list, db_list)

    def mkProj(self):
        commom_parameters = self.parameters['CommonParameters']
        self.proj = models.Project(
            name = commom_parameters['ContractID'],
            description = commom_parameters['project_description'],
            type = commom_parameters.get('BACKEND', models.BCS),
            pipe = self.pipe_path,
            path = commom_parameters.get('WORKSPACE', './'),
            max_job = commom_parameters.get('MAX_JOB', 50),
            mns = commom_parameters.get('MNS') )
        self.session.add(self.proj)
        self.session.commit()

    def mkModule(self, module_name):
        module = models.Module(name = module_name)
        self.session.commit()
        for appname in self.modules[module_name].keys():
            if appname in  self.apps:
                self.mkApp(self.apps[appname], module)

    def mkInstance(self):
        instance_list = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'instance.txt')
        instances = []
        with open(instance_list, 'r') as instance_file:
            for line in instance_file:
                (Name, CPU, MEM, DiskType, DiskSize, Price) = line.strip().split('\t')
                instances.append(models.Instance(
                name=Name, cpu=CPU, mem=MEM, price=Price,
                disk_type=DiskType, disk_size=DiskSize) )

        self.session.add_all(instances)
        self.session.commit()

    def chooseInstance(self, app):
        instance_id = getAppConfig(app, ['requirements', 'instance', 'id'])
        (cpu, mem, disk_size, disk_type) = map(functools.partial(getResourceConfig, app=app), ['cpu', 'mem', 'disk', 'disk_type'])

        if instance_id is None:
            instance = self.session.query(models.Instance). \
            filter( models.Instance.cpu >= cpu ). \
            filter( models.Instance.mem >= unifyUnit(mem) ). \
            order_by( models.Instance.price ).first()
        else:
            instance = self.session.query(models.Instance).filter_by(name = instance_id).one()

        if instance is None:
            raise LookupError("No proper instance found!")

        return instance

    def mkApp(self, app, module):
        def mkTask(script):
            def mkMapping(mapping):
                m = models.Mapping(
                    name = mapping['name'],
                    source = mapping['source'],
                    destination = mapping['destination'],
                    is_write = mapping['is_write'],
                    is_immediate = mapping['is_immediate'],
                    is_required = mapping['is_required'])
                try:
                    self.session.add(m)
                    self.session.commit()
                except IntegrityError:
                    self.session.rollback()
                    m = self.session.query(models.Mapping).filter_by(
                        name = mapping['name'],
                        source = mapping['source'],
                        destination = mapping['destination'],
                        is_write = mapping['is_write'],
                        is_immediate = mapping['is_immediate']).one()
                return m

            script['task'] = models.Task(
                    shell = os.path.abspath(script['filename']),
                    cpu = cpu,
                    mem = unifyUnit(mem),
                    docker_image = app.docker_image,
                    disk_size = unifyUnit(disk_size),
                    disk_type = disk_type,
                    project = self.proj,
                    module = module,
                    app = app,
                    mapping = map(mkMapping, script['mappings']),
                    instance = instance)
            try:
                self.session.add(script['task'])
                self.session.commit()
            except IntegrityError:
                self.session.rollback()
                print dyeWARNING("'{sh}' not unique".format(sh=script['filename']))

        mem = getAppConfig(app, ['requirements', 'resources', 'mem'])
        (cpu, mem, disk_size, disk_type) = map(functools.partial(getResourceConfig, app=app), ['cpu', 'mem', 'disk', 'disk_type'])
        instance = self.chooseInstance(app)
        scripts = [s for s in app.scripts if s['module'] == module.name]

        app = models.App(
            name = app.appname,
            alias = getAppConfig(app, ['name']),
            docker_image = getAppConfig(app, ['requirements', 'container', 'image']),
            instance_image = getAppConfig(app, ['requirements', 'instance', 'image']),
            yaml = app.config_file,
            cpu = cpu,
            mem = unifyUnit(mem),
            disk_size = unifyUnit(disk_size),
            disk_type = disk_type,
            module = module,
            instance = instance)
        self.session.add(app)
        self.session.commit()

        map(mkTask, scripts)

    def mkDepends(self):
        def mkCombTaskDepends(tasks, dep_tasks):
            for task in tasks:
                for dep_task in dep_tasks:
                    task.depend_on.append(dep_task)
            self.session.commit()

        def mkSampleTaskDepends(app, module, dep_app, dep_module):
            def mkEachSampleTaskDepends(sample_name):
                tasks = getSampleTask(app, module, sample_name)
                dep_tasks = getSampleTask(dep_app, dep_module, sample_name)
                mkCombTaskDepends(tasks, dep_tasks)

            map(mkEachSampleTaskDepends, [sample['sample_name'] for sample in self.parameters['Samples']])

        def mkAppDepends(app, module_name, depends):
            for dep_appname in depends[app.name]['depends']:
                if dep_appname in depends:
                    dep_module_name = module_name
                    dep_module = self.session.query(models.Module).filter_by(name = dep_module_name).one()
                    dep_app = self.session.query(models.App).filter_by(name = dep_appname).filter_by(module_id = dep_module.id).one()
                else:
                    dep_module_name = getDepModule(dep_appname)
                    dep_module = self.session.query(models.Module).filter_by(name = dep_module_name).one()
                    dep_app = self.session.query(models.App).filter_by(name = dep_appname).filter_by(module_id = dep_module.id).one()

                if hasSampleName(module_name, app.name) and hasSampleName(dep_module_name, dep_app.name):
                    mkSampleTaskDepends(app, module_name, dep_app, dep_module_name)
                else:
                    tasks = getModuleAppTask(app, module_name)
                    dep_tasks = getModuleAppTask(dep_app, dep_module_name)
                    mkCombTaskDepends(tasks, dep_tasks)

        def getDepModule(dep_appname):
            dep_modules = [k for k, v in self.dependencies.iteritems() if dep_appname in v]
            if len(dep_modules) == 0:
                msg = '{dep_appname} not in any module'.format(dep_appname=dep_appname)
                print dyeFAIL(msg)
                raise KeyError(msg)
            elif len(dep_modules) > 1:
                msg = '{dep_appname} has more than one module: {modules}'.format(dep_appname=dep_appname, modules=dep_modules)
                print dyeFAIL(msg)
                raise KeyError(msg)
            elif len(dep_modules) == 1:
                dep_module = dep_modules[0]
            return dep_module

        def hasSampleName(module, appname):
            return self.dependencies[module][appname]['sh_file'].count('sample_name}}') > 0

        def getModuleAppTask(app, module):
            return [t for t in app.task if t.module.name == module]

        def getSampleTask(app, module, sample_name):
            return [s['task'] for s in self.apps[app.name].scripts if s['task'].module.name == module and s['extra']['sample_name'] == sample_name]

        def mkModuleDepend(name, depends):
            module = self.session.query(models.Module).filter_by(name = name).one()
            for app in module.app:
                mkAppDepends(app, module.name, depends)

        for name, depends in self.dependencies.iteritems():
            mkModuleDepend(name, depends)

    def mkOSSuploadSH(self):
        def addSource(source, destination):
            if source in file_size:
                return
            if not is_object_exists(destination):
                file_size[source] = os.path.getsize(source)
                cmd.append("ossutil cp -f %s %s" % (source, destination))
            elif is_size_differ_and_newer(source, destination):
                cmd.append("ossutil cp -f %s %s" % (source, destination))

        def tryAddSourceWithPrefix(source, destination):
            for each_source in glob.glob(source+'*'):
                each_destination = os.path.join(os.path.dirname(destination), os.path.basename(each_source))
                addSource(each_source, each_destination)

        def mkDataUpload():
            for m in self.session.query(models.Mapping). \
                    filter_by(is_write = 0, is_immediate = 0). \
                    filter(models.Mapping.name != 'sh').all():
               if os.path.exists(m.source):
                   addSource(m.source, m.destination)
               else:
                   msg = "{name}:{source} not exist.".format(name = m.name, source = m.source)
                   print dyeFAIL(msg)
                   tryAddSourceWithPrefix(m.source, m.destination)

            content = "\n".join(['set -ex'] + list(set(cmd)))
            print "uploadData2OSS.sh: %d files(%d GB) to upload" % (len(file_size), sum(file_size.values())/2**30)
            script_file = os.path.join(self.proj_path, 'uploadData2OSS.sh')
            write(script_file, content)

        def mkScriptUpload():
            for m in self.session.query(models.Mapping).filter_by(name = 'sh').all():
                addSource(m.source, m.destination)

            content = "\n".join(['set -ex'] + list(set(cmd)))
            print "uploadScripts2OSS.sh: %d files to upload" % len(cmd)
            script_file = os.path.join(self.proj_path, 'uploadScript2OSS.sh')
            write(script_file, content)

        cmd = []
        file_size = {}
        mkDataUpload()
        cmd = []
        mkScriptUpload()

def unifyUnit(size):
    if isinstance(size, int) or isinstance(size, float):
        return float(size)
    elif size.upper().endswith("M"):
        return float(size.upper().strip('M')) / 1024
    elif size.upper().endswith("G"):
        return float(size.upper().strip('G'))
    else:
        raise ValueError("Unkown Unit: {size}".format(size=size))

def getConfig(appconfig, keys):
    for key in keys:
        appconfig = appconfig.get(key)
        if appconfig is None:
            break
    return appconfig

def getAppConfig(app, keys):
    appconfig = app.config['app']
    return getConfig(appconfig, keys)

def getResourceConfig(key, app):
    keys = ['requirements', 'resources']
    keys.append(key)
    return getAppConfig(app, keys)
<commit_msg>skip mkDepends when db_path is :memory:<commit_after>import os
import pdb
import functools
from core import models
from core.ali.oss import BUCKET, oss2key, is_object_exists, is_size_differ_and_newer
from core.misc import *
from colorMessage import dyeWARNING, dyeFAIL
from sqlalchemy import create_engine, UniqueConstraint
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError

class DB(object):
    def __init__(self, db_path, pipe_path, apps, parameters, dependencies, overwrite=False):
        super(DB, self).__init__()
        self.db_path = db_path
        self.proj_path = os.path.dirname(db_path)
        self.pipe_path = pipe_path
        self.apps = apps
        self.parameters = parameters
        self.modules = self.trim_parameters(parameters)
        self.dependencies = dependencies
        self.engine = create_engine('sqlite:///{db_path}'.format(db_path=self.db_path))
        self.proj = None
        if overwrite and os.path.exists(self.db_path):
            os.remove(self.db_path)
        if not os.path.exists(self.db_path):
            models.Base.metadata.create_all(self.engine)
        Session = sessionmaker(bind=self.engine)
        self.session = Session()

    def trim_parameters(self, parameters):
        skips = ['Inputs', 'Property', 'Parameters', 'CommonData', 'Samples', 'Outputs', 'Conditions', 'CommonParameters']
        return {k:v for k, v in parameters.iteritems() if k not in skips}

    def format(self):
        self.mkProj()
        self.mkInstance()
        map(self.mkModule, self.modules.keys())
        if self.db_path != ':memory:':
            self.mkDepends()
        self.session.commit()

    def add(self):
        snap_db_list = os.path.expanduser("~/.snap/db.yaml")
        db_list = {}
        if os.path.exists(snap_db_list):
            db_list = loadYaml(snap_db_list)
        if db_list is None:
            db_list = {}
        contract_id = self.parameters['CommonParameters']['ContractID']
        db_list[contract_id] = self.db_path
        dumpYaml(snap_db_list, db_list)

    def mkProj(self):
        commom_parameters = self.parameters['CommonParameters']
        self.proj = models.Project(
            name = commom_parameters['ContractID'],
            description = commom_parameters['project_description'],
            type = commom_parameters.get('BACKEND', models.BCS),
            pipe = self.pipe_path,
            path = commom_parameters.get('WORKSPACE', './'),
            max_job = commom_parameters.get('MAX_JOB', 50),
            mns = commom_parameters.get('MNS') )
        self.session.add(self.proj)
        self.session.commit()

    def mkModule(self, module_name):
        module = models.Module(name = module_name)
        self.session.commit()
        for appname in self.modules[module_name].keys():
            if appname in  self.apps:
                self.mkApp(self.apps[appname], module)

    def mkInstance(self):
        instance_list = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'instance.txt')
        instances = []
        with open(instance_list, 'r') as instance_file:
            for line in instance_file:
                (Name, CPU, MEM, DiskType, DiskSize, Price) = line.strip().split('\t')
                instances.append(models.Instance(
                name=Name, cpu=CPU, mem=MEM, price=Price,
                disk_type=DiskType, disk_size=DiskSize) )

        self.session.add_all(instances)
        self.session.commit()

    def chooseInstance(self, app):
        instance_id = getAppConfig(app, ['requirements', 'instance', 'id'])
        (cpu, mem, disk_size, disk_type) = map(functools.partial(getResourceConfig, app=app), ['cpu', 'mem', 'disk', 'disk_type'])

        if instance_id is None:
            instance = self.session.query(models.Instance). \
            filter( models.Instance.cpu >= cpu ). \
            filter( models.Instance.mem >= unifyUnit(mem) ). \
            order_by( models.Instance.price ).first()
        else:
            instance = self.session.query(models.Instance).filter_by(name = instance_id).one()

        if instance is None:
            raise LookupError("No proper instance found!")

        return instance

    def mkApp(self, app, module):
        def mkTask(script):
            def mkMapping(mapping):
                m = models.Mapping(
                    name = mapping['name'],
                    source = mapping['source'],
                    destination = mapping['destination'],
                    is_write = mapping['is_write'],
                    is_immediate = mapping['is_immediate'],
                    is_required = mapping['is_required'])
                try:
                    self.session.add(m)
                    self.session.commit()
                except IntegrityError:
                    self.session.rollback()
                    m = self.session.query(models.Mapping).filter_by(
                        name = mapping['name'],
                        source = mapping['source'],
                        destination = mapping['destination'],
                        is_write = mapping['is_write'],
                        is_immediate = mapping['is_immediate']).one()
                return m

            script['task'] = models.Task(
                    shell = os.path.abspath(script['filename']),
                    cpu = cpu,
                    mem = unifyUnit(mem),
                    docker_image = app.docker_image,
                    disk_size = unifyUnit(disk_size),
                    disk_type = disk_type,
                    project = self.proj,
                    module = module,
                    app = app,
                    mapping = map(mkMapping, script['mappings']),
                    instance = instance)
            try:
                self.session.add(script['task'])
                self.session.commit()
            except IntegrityError:
                self.session.rollback()
                print dyeWARNING("'{sh}' not unique".format(sh=script['filename']))

        mem = getAppConfig(app, ['requirements', 'resources', 'mem'])
        (cpu, mem, disk_size, disk_type) = map(functools.partial(getResourceConfig, app=app), ['cpu', 'mem', 'disk', 'disk_type'])
        instance = self.chooseInstance(app)
        scripts = [s for s in app.scripts if s['module'] == module.name]

        app = models.App(
            name = app.appname,
            alias = getAppConfig(app, ['name']),
            docker_image = getAppConfig(app, ['requirements', 'container', 'image']),
            instance_image = getAppConfig(app, ['requirements', 'instance', 'image']),
            yaml = app.config_file,
            cpu = cpu,
            mem = unifyUnit(mem),
            disk_size = unifyUnit(disk_size),
            disk_type = disk_type,
            module = module,
            instance = instance)
        self.session.add(app)
        self.session.commit()

        map(mkTask, scripts)

    def mkDepends(self):
        def mkCombTaskDepends(tasks, dep_tasks):
            for task in tasks:
                for dep_task in dep_tasks:
                    task.depend_on.append(dep_task)
            self.session.commit()

        def mkSampleTaskDepends(app, module, dep_app, dep_module):
            def mkEachSampleTaskDepends(sample_name):
                tasks = getSampleTask(app, module, sample_name)
                dep_tasks = getSampleTask(dep_app, dep_module, sample_name)
                mkCombTaskDepends(tasks, dep_tasks)

            map(mkEachSampleTaskDepends, [sample['sample_name'] for sample in self.parameters['Samples']])

        def mkAppDepends(app, module_name, depends):
            for dep_appname in depends[app.name]['depends']:
                if dep_appname in depends:
                    dep_module_name = module_name
                    dep_module = self.session.query(models.Module).filter_by(name = dep_module_name).one()
                    dep_app = self.session.query(models.App).filter_by(name = dep_appname).filter_by(module_id = dep_module.id).one()
                else:
                    dep_module_name = getDepModule(dep_appname)
                    dep_module = self.session.query(models.Module).filter_by(name = dep_module_name).one()
                    dep_app = self.session.query(models.App).filter_by(name = dep_appname).filter_by(module_id = dep_module.id).one()

                if hasSampleName(module_name, app.name) and hasSampleName(dep_module_name, dep_app.name):
                    mkSampleTaskDepends(app, module_name, dep_app, dep_module_name)
                else:
                    tasks = getModuleAppTask(app, module_name)
                    dep_tasks = getModuleAppTask(dep_app, dep_module_name)
                    mkCombTaskDepends(tasks, dep_tasks)

        def getDepModule(dep_appname):
            dep_modules = [k for k, v in self.dependencies.iteritems() if dep_appname in v]
            if len(dep_modules) == 0:
                msg = '{dep_appname} not in any module'.format(dep_appname=dep_appname)
                print dyeFAIL(msg)
                raise KeyError(msg)
            elif len(dep_modules) > 1:
                msg = '{dep_appname} has more than one module: {modules}'.format(dep_appname=dep_appname, modules=dep_modules)
                print dyeFAIL(msg)
                raise KeyError(msg)
            elif len(dep_modules) == 1:
                dep_module = dep_modules[0]
            return dep_module

        def hasSampleName(module, appname):
            return self.dependencies[module][appname]['sh_file'].count('sample_name}}') > 0

        def getModuleAppTask(app, module):
            return [t for t in app.task if t.module.name == module]

        def getSampleTask(app, module, sample_name):
            return [s['task'] for s in self.apps[app.name].scripts if s['task'].module.name == module and s['extra']['sample_name'] == sample_name]

        def mkModuleDepend(name, depends):
            module = self.session.query(models.Module).filter_by(name = name).one()
            for app in module.app:
                mkAppDepends(app, module.name, depends)

        for name, depends in self.dependencies.iteritems():
            mkModuleDepend(name, depends)

    def mkOSSuploadSH(self):
        def addSource(source, destination):
            if source in file_size:
                return
            if not is_object_exists(destination):
                file_size[source] = os.path.getsize(source)
                cmd.append("ossutil cp -f %s %s" % (source, destination))
            elif is_size_differ_and_newer(source, destination):
                cmd.append("ossutil cp -f %s %s" % (source, destination))

        def tryAddSourceWithPrefix(source, destination):
            for each_source in glob.glob(source+'*'):
                each_destination = os.path.join(os.path.dirname(destination), os.path.basename(each_source))
                addSource(each_source, each_destination)

        def mkDataUpload():
            for m in self.session.query(models.Mapping). \
                    filter_by(is_write = 0, is_immediate = 0). \
                    filter(models.Mapping.name != 'sh').all():
               if os.path.exists(m.source):
                   addSource(m.source, m.destination)
               else:
                   msg = "{name}:{source} not exist.".format(name = m.name, source = m.source)
                   print dyeFAIL(msg)
                   tryAddSourceWithPrefix(m.source, m.destination)

            content = "\n".join(['set -ex'] + list(set(cmd)))
            print "uploadData2OSS.sh: %d files(%d GB) to upload" % (len(file_size), sum(file_size.values())/2**30)
            script_file = os.path.join(self.proj_path, 'uploadData2OSS.sh')
            write(script_file, content)

        def mkScriptUpload():
            for m in self.session.query(models.Mapping).filter_by(name = 'sh').all():
                addSource(m.source, m.destination)

            content = "\n".join(['set -ex'] + list(set(cmd)))
            print "uploadScripts2OSS.sh: %d files to upload" % len(cmd)
            script_file = os.path.join(self.proj_path, 'uploadScript2OSS.sh')
            write(script_file, content)

        cmd = []
        file_size = {}
        mkDataUpload()
        cmd = []
        mkScriptUpload()

def unifyUnit(size):
    if isinstance(size, int) or isinstance(size, float):
        return float(size)
    elif size.upper().endswith("M"):
        return float(size.upper().strip('M')) / 1024
    elif size.upper().endswith("G"):
        return float(size.upper().strip('G'))
    else:
        raise ValueError("Unkown Unit: {size}".format(size=size))

def getConfig(appconfig, keys):
    for key in keys:
        appconfig = appconfig.get(key)
        if appconfig is None:
            break
    return appconfig

def getAppConfig(app, keys):
    appconfig = app.config['app']
    return getConfig(appconfig, keys)

def getResourceConfig(key, app):
    keys = ['requirements', 'resources']
    keys.append(key)
    return getAppConfig(app, keys)
<|endoftext|>


<commit_before>import os
import pdb
import functools
import glob
from core import models
from core.ali.oss import BUCKET, oss2key, is_object_exists, is_size_differ_and_newer, is_source_newer
from core.misc import *
from colorMessage import dyeWARNING, dyeFAIL
from sqlalchemy import create_engine, UniqueConstraint
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError

class DB(object):
    def __init__(self, db_path, pipe_path, apps, parameters, dependencies, overwrite=False):
        super(DB, self).__init__()
        self.db_path = db_path
        self.proj_path = os.path.dirname(db_path)
        self.pipe_path = pipe_path
        self.apps = apps
        self.parameters = parameters
        self.modules = self.trim_parameters(parameters)
        self.dependencies = dependencies
        self.engine = create_engine('sqlite:///{db_path}'.format(db_path=self.db_path))
        self.proj = None
        if overwrite and os.path.exists(self.db_path):
            os.remove(self.db_path)
        if not os.path.exists(self.db_path):
            models.Base.metadata.create_all(self.engine)
        Session = sessionmaker(bind=self.engine)
        self.session = Session()

    def trim_parameters(self, parameters):
        skips = ['Inputs', 'Property', 'Parameters', 'CommonData', 'Samples', 'Outputs', 'Conditions', 'CommonParameters']
        return {k:v for k, v in parameters.iteritems() if k not in skips}

    def format(self):
        self.mkProj()
        self.mkInstance()
        map(self.mkModule, self.modules.keys())
        if self.db_path != ':memory:':
            self.mkDepends()
        self.session.commit()

    def add(self):
        snap_db_list = os.path.expanduser("~/.snap/db.yaml")
        db_list = {}
        if os.path.exists(snap_db_list):
            db_list = loadYaml(snap_db_list)
        if db_list is None:
            db_list = {}
        contract_id = self.parameters['CommonParameters']['ContractID']
        db_list[contract_id] = self.db_path
        dumpYaml(snap_db_list, db_list)

    def mkProj(self):
        commom_parameters = self.parameters['CommonParameters']
        self.proj = models.Project(
            name = commom_parameters['ContractID'],
            description = commom_parameters['project_description'],
            type = commom_parameters.get('BACKEND', models.BCS),
            pipe = self.pipe_path,
            path = commom_parameters.get('WORKSPACE', './'),
            max_job = commom_parameters.get('MAX_JOB', 50),
            mns = commom_parameters.get('MNS') )
        self.session.add(self.proj)
        self.session.commit()

    def mkModule(self, module_name):
        module = models.Module(name = module_name)
        self.session.commit()
        for appname in self.modules[module_name].keys():
            if appname in  self.apps:
                self.mkApp(self.apps[appname], module)

    def mkInstance(self):
        instance_list = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'instance.txt')
        instances = []
        with open(instance_list, 'r') as instance_file:
            for line in instance_file:
                (Name, CPU, MEM, DiskType, DiskSize, Price) = line.strip().split('\t')
                instances.append(models.Instance(
                name=Name, cpu=CPU, mem=MEM, price=Price,
                disk_type=DiskType, disk_size=DiskSize) )

        self.session.add_all(instances)
        self.session.commit()

    def chooseInstance(self, app):
        instance_id = getAppConfig(app, ['requirements', 'instance', 'id'])
        (cpu, mem, disk_size, disk_type) = map(functools.partial(getResourceConfig, app=app), ['cpu', 'mem', 'disk', 'disk_type'])

        if instance_id is None:
            instance = self.session.query(models.Instance). \
            filter( models.Instance.cpu >= cpu ). \
            filter( models.Instance.mem >= unifyUnit(mem) ). \
            order_by( models.Instance.price ).first()
        else:
            instance = self.session.query(models.Instance).filter_by(name = instance_id).one()

        if instance is None:
            raise LookupError("No proper instance found!")

        return instance

    def mkApp(self, app, module):
        def mkTask(script):
            def mkMapping(mapping):
                m = models.Mapping(
                    name = mapping['name'],
                    source = mapping['source'],
                    destination = mapping['destination'],
                    is_write = mapping['is_write'],
                    is_immediate = mapping['is_immediate'],
                    is_required = mapping['is_required'])
                try:
                    self.session.add(m)
                    self.session.commit()
                except IntegrityError:
                    self.session.rollback()
                    m = self.session.query(models.Mapping).filter_by(
                        name = mapping['name'],
                        source = mapping['source'],
                        destination = mapping['destination'],
                        is_write = mapping['is_write'],
                        is_immediate = mapping['is_immediate']).one()
                return m

            script['task'] = models.Task(
                    shell = os.path.abspath(script['filename']),
                    cpu = cpu,
                    mem = unifyUnit(mem),
                    docker_image = app.docker_image,
                    disk_size = unifyUnit(disk_size),
                    disk_type = disk_type,
                    project = self.proj,
                    module = module,
                    app = app,
                    mapping = map(mkMapping, script['mappings']),
                    instance = instance)
            try:
                self.session.add(script['task'])
                self.session.commit()
            except IntegrityError:
                self.session.rollback()
                print dyeWARNING("'{sh}' not unique".format(sh=script['filename']))

        mem = getAppConfig(app, ['requirements', 'resources', 'mem'])
        (cpu, mem, disk_size, disk_type) = map(functools.partial(getResourceConfig, app=app), ['cpu', 'mem', 'disk', 'disk_type'])
        instance = self.chooseInstance(app)
        scripts = [s for s in app.scripts if s['module'] == module.name]

        app = models.App(
            name = app.appname,
            alias = getAppConfig(app, ['name']),
            docker_image = getAppConfig(app, ['requirements', 'container', 'image']),
            instance_image = getAppConfig(app, ['requirements', 'instance', 'image']),
            yaml = app.config_file,
            cpu = cpu,
            mem = unifyUnit(mem),
            disk_size = unifyUnit(disk_size),
            disk_type = disk_type,
            module = module,
            instance = instance)
        self.session.add(app)
        self.session.commit()

        map(mkTask, scripts)

    def mkDepends(self):
        def mkCombTaskDepends(tasks, dep_tasks):
            for task in tasks:
                for dep_task in dep_tasks:
                    task.depend_on.append(dep_task)
            self.session.commit()

        def mkSampleTaskDepends(app, module, dep_app, dep_module):
            def mkEachSampleTaskDepends(sample_name):
                tasks = getSampleTask(app, module, sample_name)
                dep_tasks = getSampleTask(dep_app, dep_module, sample_name)
                mkCombTaskDepends(tasks, dep_tasks)

            map(mkEachSampleTaskDepends, [sample['sample_name'] for sample in self.parameters['Samples']])

        def mkAppDepends(app, module_name, depends):
            for dep_appname in depends[app.name]['depends']:
                if dep_appname not in apps_in_param:
                    print dyeWARNING("{appname}: skipping dependence app {dep_appname} since it's not in parameters.conf".format(appname=app.name, dep_appname=dep_appname))
                    continue

                if dep_appname in depends:
                    dep_module_name = module_name
                    dep_module = self.session.query(models.Module).filter_by(name = dep_module_name).one()
                    dep_app = self.session.query(models.App).filter_by(name = dep_appname).filter_by(module_id = dep_module.id).one()
                else:
                    dep_module_name = getDepModule(dep_appname)
                    dep_module = self.session.query(models.Module).filter_by(name = dep_module_name).one()
                    dep_app = self.session.query(models.App).filter_by(name = dep_appname).filter_by(module_id = dep_module.id).one()

                if hasSampleName(module_name, app.name) and hasSampleName(dep_module_name, dep_app.name):
                    mkSampleTaskDepends(app, module_name, dep_app, dep_module_name)
                else:
                    tasks = getModuleAppTask(app, module_name)
                    dep_tasks = getModuleAppTask(dep_app, dep_module_name)
                    mkCombTaskDepends(tasks, dep_tasks)

        def getDepModule(dep_appname):
            dep_modules = [k for k, v in self.dependencies.iteritems() if dep_appname in v]
            if len(dep_modules) == 0:
                msg = '{dep_appname} not in any module'.format(dep_appname=dep_appname)
                print dyeFAIL(msg)
                raise KeyError(msg)
            elif len(dep_modules) > 1:
                msg = '{dep_appname} has more than one module: {modules}'.format(dep_appname=dep_appname, modules=dep_modules)
                print dyeFAIL(msg)
                raise KeyError(msg)
            elif len(dep_modules) == 1:
                dep_module = dep_modules[0]
            return dep_module

        def hasSampleName(module, appname):
            return self.dependencies[module][appname]['sh_file'].count('sample_name}}') > 0

        def getModuleAppTask(app, module):
            return [t for t in app.task if t.module.name == module]

        def getSampleTask(app, module, sample_name):
            return [s['task'] for s in self.apps[app.name].scripts if s['task'].module.name == module and s['extra']['sample_name'] == sample_name]

        def mkModuleDepend(name, depends):
            module = self.session.query(models.Module).filter_by(name = name).one()
            for app in module.app:
                mkAppDepends(app, module.name, depends)

        apps_in_param = reduce(concat, [apps.keys() for apps in self.modules.values()])

        for name in self.modules.keys():
            mkModuleDepend(name, self.dependencies[name])

    def mkOSSuploadSH(self):
        def addSource(source, destination):
            if source in file_size:
                return
            if not is_object_exists(destination):
                file_size[source] = os.path.getsize(source)
                cmd.append("ossutil cp -f %s %s" % (source, destination))
            elif is_size_differ_and_newer(source, destination):
                cmd.append("ossutil cp -f %s %s" % (source, destination))
            elif is_source_newer(source, destination) and is_md5_differ(source, destination):
                cmd.append("ossutil cp -f %s %s" % (source, destination))

        def tryAddSourceWithPrefix(source, destination):
            for each_source in glob.glob(source+'*'):
                each_destination = os.path.join(os.path.dirname(destination), os.path.basename(each_source))
                addSource(each_source, each_destination)

        def mkDataUpload():
            for m in self.session.query(models.Mapping). \
                    filter_by(is_write = 0, is_immediate = 0). \
                    filter(models.Mapping.name != 'sh').all():
               if os.path.exists(m.source):
                   addSource(m.source, m.destination)
               else:
                   if not m.exists():
                       msg = "{name}:{source} not exist.".format(name = m.name, source = m.source)
                       print dyeFAIL(msg)
                       tryAddSourceWithPrefix(m.source, m.destination)

            content = "\n".join(['set -ex'] + list(set(cmd)))
            print "uploadData2OSS.sh: %d files(%d GB) to upload" % (len(file_size), sum(file_size.values())/2**30)
            script_file = os.path.join(self.proj_path, 'uploadData2OSS.sh')
            write(script_file, content)

        def mkScriptUpload():
            for m in self.session.query(models.Mapping).filter_by(name = 'sh').all():
                addSource(m.source, m.destination)

            content = "\n".join(['set -ex'] + list(set(cmd)))
            print "uploadScripts2OSS.sh: %d files to upload" % len(cmd)
            script_file = os.path.join(self.proj_path, 'uploadScript2OSS.sh')
            write(script_file, content)

        cmd = []
        file_size = {}
        mkDataUpload()
        cmd = []
        mkScriptUpload()

def unifyUnit(size):
    if isinstance(size, int) or isinstance(size, float):
        return float(size)
    elif size.upper().endswith("M"):
        return float(size.upper().strip('M')) / 1024
    elif size.upper().endswith("G"):
        return float(size.upper().strip('G'))
    else:
        raise ValueError("Unkown Unit: {size}".format(size=size))

def getConfig(appconfig, keys):
    for key in keys:
        appconfig = appconfig.get(key)
        if appconfig is None:
            break
    return appconfig

def getAppConfig(app, keys):
    appconfig = app.config['app']
    return getConfig(appconfig, keys)

def getResourceConfig(key, app):
    keys = ['requirements', 'resources']
    keys.append(key)
    return getAppConfig(app, keys)
<commit_msg>trim Groups from parameters; add mkOssSyncSH;<commit_after>import os
import pdb
import functools
import glob
from core import models
from core.ali.oss import BUCKET, oss2key, is_object_exists, is_size_differ_and_newer, is_source_newer
from core.misc import *
from colorMessage import dyeWARNING, dyeFAIL
from sqlalchemy import create_engine, UniqueConstraint
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError

class DB(object):
    def __init__(self, db_path, pipe_path, apps, parameters, dependencies, overwrite=False):
        super(DB, self).__init__()
        self.db_path = db_path
        self.proj_path = os.path.dirname(db_path)
        self.pipe_path = pipe_path
        self.apps = apps
        self.parameters = parameters
        self.modules = self.trim_parameters(parameters)
        self.dependencies = dependencies
        self.engine = create_engine('sqlite:///{db_path}'.format(db_path=self.db_path))
        self.proj = None
        if overwrite and os.path.exists(self.db_path):
            os.remove(self.db_path)
        if not os.path.exists(self.db_path):
            models.Base.metadata.create_all(self.engine)
        Session = sessionmaker(bind=self.engine)
        self.session = Session()

    def trim_parameters(self, parameters):
        skips = ['Inputs', 'Property', 'Parameters', 'CommonData', 'Samples', 'Groups', 'Outputs', 'Conditions', 'CommonParameters']
        return {k:v for k, v in parameters.iteritems() if k not in skips}

    def format(self):
        self.mkProj()
        self.mkInstance()
        map(self.mkModule, self.modules.keys())
        if self.db_path != ':memory:':
            self.mkDepends()
        self.session.commit()

    def add(self):
        snap_db_list = os.path.expanduser("~/.snap/db.yaml")
        db_list = {}
        if os.path.exists(snap_db_list):
            db_list = loadYaml(snap_db_list)
        if db_list is None:
            db_list = {}
        contract_id = self.parameters['CommonParameters']['ContractID']
        db_list[contract_id] = self.db_path
        dumpYaml(snap_db_list, db_list)

    def mkProj(self):
        commom_parameters = self.parameters['CommonParameters']
        self.proj = models.Project(
            name = commom_parameters['ContractID'],
            description = commom_parameters['project_description'],
            type = commom_parameters.get('BACKEND', models.BCS),
            pipe = self.pipe_path,
            path = commom_parameters.get('WORKSPACE', './'),
            max_job = commom_parameters.get('MAX_JOB', 50),
            mns = commom_parameters.get('MNS') )
        self.session.add(self.proj)
        self.session.commit()

    def mkModule(self, module_name):
        module = models.Module(name = module_name)
        self.session.commit()
        for appname in self.modules[module_name].keys():
            if appname in  self.apps:
                self.mkApp(self.apps[appname], module)

    def mkInstance(self):
        instance_list = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'instance.txt')
        instances = []
        with open(instance_list, 'r') as instance_file:
            for line in instance_file:
                (Name, CPU, MEM, DiskType, DiskSize, Price) = line.strip().split('\t')
                instances.append(models.Instance(
                name=Name, cpu=CPU, mem=MEM, price=Price,
                disk_type=DiskType, disk_size=DiskSize) )

        self.session.add_all(instances)
        self.session.commit()

    def chooseInstance(self, app):
        instance_id = getAppConfig(app, ['requirements', 'instance', 'id'])
        (cpu, mem, disk_size, disk_type) = map(functools.partial(getResourceConfig, app=app), ['cpu', 'mem', 'disk', 'disk_type'])

        if instance_id is None:
            instance = self.session.query(models.Instance). \
            filter( models.Instance.cpu >= cpu ). \
            filter( models.Instance.mem >= unifyUnit(mem) ). \
            order_by( models.Instance.price ).first()
        else:
            instance = self.session.query(models.Instance).filter_by(name = instance_id).one()

        if instance is None:
            raise LookupError("No proper instance found!")

        return instance

    def mkApp(self, app, module):
        def mkTask(script):
            def mkMapping(mapping):
                m = models.Mapping(
                    name = mapping['name'],
                    source = mapping['source'],
                    destination = mapping['destination'],
                    is_write = mapping['is_write'],
                    is_immediate = mapping['is_immediate'],
                    is_required = mapping['is_required'])
                try:
                    self.session.add(m)
                    self.session.commit()
                except IntegrityError:
                    self.session.rollback()
                    m = self.session.query(models.Mapping).filter_by(
                        name = mapping['name'],
                        source = mapping['source'],
                        destination = mapping['destination'],
                        is_write = mapping['is_write'],
                        is_immediate = mapping['is_immediate']).one()
                return m

            script['task'] = models.Task(
                    shell = os.path.abspath(script['filename']),
                    cpu = cpu,
                    mem = unifyUnit(mem),
                    docker_image = app.docker_image,
                    disk_size = unifyUnit(disk_size),
                    disk_type = disk_type,
                    project = self.proj,
                    module = module,
                    app = app,
                    mapping = map(mkMapping, script['mappings']),
                    instance = instance)
            try:
                self.session.add(script['task'])
                self.session.commit()
            except IntegrityError:
                self.session.rollback()
                print dyeWARNING("'{sh}' not unique".format(sh=script['filename']))

        mem = getAppConfig(app, ['requirements', 'resources', 'mem'])
        (cpu, mem, disk_size, disk_type) = map(functools.partial(getResourceConfig, app=app), ['cpu', 'mem', 'disk', 'disk_type'])
        instance = self.chooseInstance(app)
        scripts = [s for s in app.scripts if s['module'] == module.name]

        app = models.App(
            name = app.appname,
            alias = getAppConfig(app, ['name']),
            docker_image = getAppConfig(app, ['requirements', 'container', 'image']),
            instance_image = getAppConfig(app, ['requirements', 'instance', 'image']),
            yaml = app.config_file,
            cpu = cpu,
            mem = unifyUnit(mem),
            disk_size = unifyUnit(disk_size),
            disk_type = disk_type,
            module = module,
            instance = instance)
        self.session.add(app)
        self.session.commit()

        map(mkTask, scripts)

    def mkDepends(self):
        def mkCombTaskDepends(tasks, dep_tasks):
            for task in tasks:
                for dep_task in dep_tasks:
                    task.depend_on.append(dep_task)
            self.session.commit()

        def mkSampleTaskDepends(app, module, dep_app, dep_module):
            def mkEachSampleTaskDepends(sample_name):
                tasks = getSampleTask(app, module, sample_name)
                dep_tasks = getSampleTask(dep_app, dep_module, sample_name)
                mkCombTaskDepends(tasks, dep_tasks)

            map(mkEachSampleTaskDepends, [sample['sample_name'] for sample in self.parameters['Samples']])

        def mkAppDepends(app, module_name, depends):
            for dep_appname in depends[app.name]['depends']:
                if dep_appname not in apps_in_param:
                    print dyeWARNING("{appname}: skipping dependence app {dep_appname} since it's not in parameters.conf".format(appname=app.name, dep_appname=dep_appname))
                    continue

                if dep_appname in depends:
                    dep_module_name = module_name
                    dep_module = self.session.query(models.Module).filter_by(name = dep_module_name).one()
                    dep_app = self.session.query(models.App).filter_by(name = dep_appname).filter_by(module_id = dep_module.id).one()
                else:
                    dep_module_name = getDepModule(dep_appname)
                    dep_module = self.session.query(models.Module).filter_by(name = dep_module_name).one()
                    dep_app = self.session.query(models.App).filter_by(name = dep_appname).filter_by(module_id = dep_module.id).one()

                if hasSampleName(module_name, app.name) and hasSampleName(dep_module_name, dep_app.name):
                    mkSampleTaskDepends(app, module_name, dep_app, dep_module_name)
                else:
                    tasks = getModuleAppTask(app, module_name)
                    dep_tasks = getModuleAppTask(dep_app, dep_module_name)
                    mkCombTaskDepends(tasks, dep_tasks)

        def getDepModule(dep_appname):
            dep_modules = [k for k, v in self.dependencies.iteritems() if dep_appname in v]
            if len(dep_modules) == 0:
                msg = '{dep_appname} not in any module'.format(dep_appname=dep_appname)
                print dyeFAIL(msg)
                raise KeyError(msg)
            elif len(dep_modules) > 1:
                msg = '{dep_appname} has more than one module: {modules}'.format(dep_appname=dep_appname, modules=dep_modules)
                print dyeFAIL(msg)
                raise KeyError(msg)
            elif len(dep_modules) == 1:
                dep_module = dep_modules[0]
            return dep_module

        def hasSampleName(module, appname):
            return self.dependencies[module][appname]['sh_file'].count('sample_name}}') > 0

        def getModuleAppTask(app, module):
            return [t for t in app.task if t.module.name == module]

        def getSampleTask(app, module, sample_name):
            return [s['task'] for s in self.apps[app.name].scripts if s['task'].module.name == module and s['extra']['sample_name'] == sample_name]

        def mkModuleDepend(name, depends):
            module = self.session.query(models.Module).filter_by(name = name).one()
            for app in module.app:
                mkAppDepends(app, module.name, depends)

        apps_in_param = reduce(concat, [apps.keys() for apps in self.modules.values()])

        for name in self.modules.keys():
            mkModuleDepend(name, self.dependencies[name])

    def mkOSSuploadSH(self):
        def addSource(source, destination):
            if source in file_size:
                return
            if not is_object_exists(destination):
                file_size[source] = os.path.getsize(source)
                cmd.append("ossutil cp -f %s %s" % (source, destination))
            elif is_size_differ_and_newer(source, destination):
                cmd.append("ossutil cp -f %s %s" % (source, destination))
            elif is_source_newer(source, destination) and is_md5_differ(source, destination):
                cmd.append("ossutil cp -f %s %s" % (source, destination))

        def tryAddSourceWithPrefix(source, destination):
            for each_source in glob.glob(source+'*'):
                each_destination = os.path.join(os.path.dirname(destination), os.path.basename(each_source))
                addSource(each_source, each_destination)

        def mkDataUpload():
            for m in self.session.query(models.Mapping). \
                    filter_by(is_write = 0, is_immediate = 0). \
                    filter(models.Mapping.name != 'sh').all():
               if os.path.exists(m.source):
                   addSource(m.source, m.destination)
               else:
                   if not m.exists():
                       msg = "{name}:{source} not exist.".format(name = m.name, source = m.source)
                       print dyeFAIL(msg)
                       tryAddSourceWithPrefix(m.source, m.destination)

            content = "\n".join(['set -ex'] + list(set(cmd)))
            print "uploadData2OSS.sh: %d files(%d GB) to upload" % (len(file_size), sum(file_size.values())/2**30)
            script_file = os.path.join(self.proj_path, 'uploadData2OSS.sh')
            write(script_file, content)

        def mkScriptUpload():
            for m in self.session.query(models.Mapping).filter_by(name = 'sh').all():
                addSource(m.source, m.destination)

            content = "\n".join(['set -ex'] + list(set(cmd)))
            print "uploadScripts2OSS.sh: %d files to upload" % len(cmd)
            script_file = os.path.join(self.proj_path, 'uploadScript2OSS.sh')
            write(script_file, content)

        cmd = []
        file_size = {}
        mkDataUpload()
        cmd = []
        mkScriptUpload()

    def mkOssSyncSH(self):
        def mkDataSync():
            content = "\n".join([
                '{snap_path}/snap mapping list -p {project} -is_not_write -is_not_immediate -skip_existed',
                'read -p "Mappings above will be sync. Press enter to continue"',
                '{snap_path}/snap mapping sync -p {project} -is_not_write -is_not_immediate -estimate_size' ]).format(snap_path=snap_path, project=self.proj.name)
            script_file = os.path.join(self.proj_path, 'uploadData2OSS.sh')
            write(script_file, content)

        def mkScriptSync():
            content = "\n".join([
                '{snap_path}/snap mapping list -p {project} -name sh -skip_existed',
                'read -p "Mappings above will be sync. Press enter to continue"',
                '{snap_path}/snap mapping sync -p {project} -name sh -estimate_size' ]).format(snap_path=snap_path, project=self.proj.name)
            script_file = os.path.join(self.proj_path, 'uploadScript2OSS.sh')
            write(script_file, content)

        snap_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
        mkDataSync()
        mkScriptSync()

def unifyUnit(size):
    if isinstance(size, int) or isinstance(size, float):
        return float(size)
    elif size.upper().endswith("M"):
        return float(size.upper().strip('M')) / 1024
    elif size.upper().endswith("G"):
        return float(size.upper().strip('G'))
    else:
        raise ValueError("Unkown Unit: {size}".format(size=size))

def getConfig(appconfig, keys):
    for key in keys:
        appconfig = appconfig.get(key)
        if appconfig is None:
            break
    return appconfig

def getAppConfig(app, keys):
    appconfig = app.config['app']
    return getConfig(appconfig, keys)

def getResourceConfig(key, app):
    keys = ['requirements', 'resources']
    keys.append(key)
    return getAppConfig(app, keys)
<|endoftext|>


<commit_before>import xbmc
import xbmcaddon
import xbmcvfs

__addon__ = xbmcaddon.Addon()
__addonid__ = __addon__.getAddonInfo('id')

try:
    # import visuallogger
    # logger = visuallogger.Logger()
    xbmcaddon.Addon('script.design.helper')
    loggerInstalled = True
except:
    # logger = None
    loggerInstalled = False

def log(message, logToGui=True, level=xbmc.LOGDEBUG):
    if loggerInstalled:
        # Yeah, this is ugly, so def want it to be a module
        builtin = "RunScript(script.design.helper, log, %s, \"%s\"" % (__addonid__, message)
        if logToGui:
            builtin += ', logToGui'
        builtin += ')'
        xbmc.executebuiltin(builtin)
    else:
        xbmc.log('[%s] %s' % (__addonid__, message), level)

def main():
    fullUrl = xbmc.getInfoLabel("ListItem.FolderPath").decode("utf-8")
    # randomPlayer.playRandomFromFolderPath(folderPath)
    builtin = 'RunScript(script.playrandom, "%s")' % fullUrl
    xbmc.executebuiltin(builtin)

if __name__ == '__main__':
    main()
<commit_msg>Shuffle some stuff around<commit_after>import xbmc
import xbmcaddon
import xbmcvfs

__addon__ = xbmcaddon.Addon()
__addonid__ = __addon__.getAddonInfo('id')

try:
    # import visuallogger
    # logger = visuallogger.Logger()
    xbmcaddon.Addon('script.design.helper')
    loggerInstalled = True
except:
    # logger = None
    loggerInstalled = False

def log(message, logToGui=True, level=xbmc.LOGDEBUG):
    if loggerInstalled:
        # Yeah, this is ugly, so def want it to be a module
        builtin = 'RunScript(script.design.helper, log, %s, "%s"' % (__addonid__, message)
        if logToGui:
            builtin += ', logToGui'
        builtin += ')'
        xbmc.executebuiltin(builtin.encode('utf-8'))
    else:
        xbmc.log('[%s] %s' % (__addonid__, message.encode('utf-8')), level)

def main():
    fullUrl = xbmc.getInfoLabel("ListItem.FolderPath").decode("utf-8")
    # randomPlayer.playRandomFromFolderPath(folderPath)
    builtin = 'RunScript(script.playrandom, "%s")' % fullUrl
    xbmc.executebuiltin(builtin)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># coding=utf-8
import logging
import os
from ledman import DEBUG, config
import json

__author__ = 'Victor Häggqvist'

logger = logging.getLogger(__name__)
fh = logging.FileHandler('ledman.log')
fh.setLevel(logging.INFO)
fh.setFormatter(logging.Formatter(fmt='%(asctime)s:%(levelname)s:%(name)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
logger.addHandler(fh)
statefile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ledstate.json')

def store_state(pin, brightness):
    store = open(statefile, "r")
    state = store.read()
    store.close()

    try:
        statejson = json.loads(state)
    except:
        statejson = {}

    statejson[get_color_by_pin(pin)] = brightness
    ujson = json.dumps(statejson)

    store = open(statefile,'w')
    store.write(ujson)
    store.close()

def set_gpio(pin, brightness):
    if DEBUG:
        logger.info('not actually doing echo '+pin+'='+brightness+' > /dev/pi-blaster')
        store_state(pin, brightness)
    else:
        os.system('echo "'+pin+'='+brightness+'" > /dev/pi-blaster')
        store_state(pin, brightness)
        logger.info('set pin '+pin+' at level '+brightness)

def turn_on():
    set_gpio(config.GPIO_RED, config.RED_DEFAULT)
    set_gpio(config.GPIO_GREEN, config.GREEN_DEFAULT)
    set_gpio(config.GPIO_BLUE, config.BLUE_DEFAULT)

def turn_off():
    set_gpio(config.GPIO_RED, '0')
    set_gpio(config.GPIO_GREEN, '0')
    set_gpio(config.GPIO_BLUE, '0')

def get_pin_by_color(color):
    return {
        "r": config.GPIO_RED,
        "g": config.GPIO_GREEN,
        "b": config.GPIO_BLUE
    }[color]

def get_color_by_pin(pin):
    return {
        config.GPIO_RED: 'r',
        config.GPIO_GREEN: 'g',
        config.GPIO_BLUE: 'b'
    }[pin]

def set_color(color, level):
    """
    Set color pin to level
    :param color: str
    :param level: str
    :return:
    """
    ok = True
    if color not in ['r','g','b']:
        ok = False

    if len(level) > 3:
        ok = False

    if level in ['0','1']:
        ok = True

    if len(level) == 3 and is_float(level) and float(level) < 1:
        ok = True

    if ok:
        logger.info('set color '+color+' at level '+level)
        pin = get_pin_by_color(color)
        set_gpio(pin, level)
        return False
    else:
        logger.info('set color, unrecognized combination: color '+color+' at level '+level)
        return False

def get_status():
    f = open(statefile, 'r')
    state = f.read()
    f.close()
    return state

def is_float(num_string):
    try:
        float(num_string)
        return True
    except:
        return False
<commit_msg>fix: don't fail if no state<commit_after># coding=utf-8
import logging
import os
from ledman import DEBUG, config
import json

__author__ = 'Victor Häggqvist'

logger = logging.getLogger(__name__)
fh = logging.FileHandler('ledman.log')
fh.setLevel(logging.INFO)
fh.setFormatter(logging.Formatter(fmt='%(asctime)s:%(levelname)s:%(name)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
logger.addHandler(fh)
statefile = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ledstate.json')

def store_state(pin, brightness):
    try:
        store = open(statefile, "r")
        state = store.read()
        store.close()
        statejson = json.loads(state)
    except:
        statejson = {}

    statejson[get_color_by_pin(pin)] = brightness
    ujson = json.dumps(statejson)

    store = open(statefile,'w')
    store.write(ujson)
    store.close()

def set_gpio(pin, brightness):
    if DEBUG:
        logger.info('not actually doing echo '+pin+'='+brightness+' > /dev/pi-blaster')
        store_state(pin, brightness)
    else:
        os.system('echo "'+pin+'='+brightness+'" > /dev/pi-blaster')
        store_state(pin, brightness)
        logger.info('set pin '+pin+' at level '+brightness)

def turn_on():
    set_gpio(config.GPIO_RED, config.RED_DEFAULT)
    set_gpio(config.GPIO_GREEN, config.GREEN_DEFAULT)
    set_gpio(config.GPIO_BLUE, config.BLUE_DEFAULT)

def turn_off():
    set_gpio(config.GPIO_RED, '0')
    set_gpio(config.GPIO_GREEN, '0')
    set_gpio(config.GPIO_BLUE, '0')

def get_pin_by_color(color):
    return {
        "r": config.GPIO_RED,
        "g": config.GPIO_GREEN,
        "b": config.GPIO_BLUE
    }[color]

def get_color_by_pin(pin):
    return {
        config.GPIO_RED: 'r',
        config.GPIO_GREEN: 'g',
        config.GPIO_BLUE: 'b'
    }[pin]

def set_color(color, level):
    """
    Set color pin to level
    :param color: str
    :param level: str
    :return:
    """
    ok = True
    if color not in ['r','g','b']:
        ok = False

    if len(level) > 3:
        ok = False

    if level in ['0','1']:
        ok = True

    if len(level) == 3 and is_float(level) and float(level) < 1:
        ok = True

    if ok:
        logger.info('set color '+color+' at level '+level)
        pin = get_pin_by_color(color)
        set_gpio(pin, level)
        return False
    else:
        logger.info('set color, unrecognized combination: color '+color+' at level '+level)
        return False

def get_status():
    f = open(statefile, 'r')
    state = f.read()
    f.close()
    return state

def is_float(num_string):
    try:
        float(num_string)
        return True
    except:
        return False
<|endoftext|>


<commit_before>from keras.models import Sequential
from keras.utils import np_utils
from keras.layers.core import Dense, Activation, Dropout
from keras.layers import Flatten
from keras.layers.convolutional import Convolution2D
from keras.layers.convolutional import MaxPooling2D

import pandas as pd
import numpy as np

NUM_CLASSES = 10

def complex_model():
    model = Sequential()
    model.add(Convolution2D(30, 5, 5, border_mode='valid', input_shape=(1, 28, 28), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Convolution2D(15, 3, 3, activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Dense(50, activation='relu'))
    model.add(Dense(20, activation='relu'))
    model.add(Dense(NUM_CLASSES, activation='softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

if __name__ == '__main__':
    # fix random seed for reproducibility
    seed = 171717
    np.random.seed(seed)

    # read in the training and test data
    train = pd.read_csv('./data/train.csv')
    labels = train.ix[:, 0].values.astype('int32')
    X_train = (train.ix[:, 1:].values).astype('float32')
    X_test = (pd.read_csv('./data/test.csv').values).astype('float32')

    scale = np.max(X_train)
    X_train /= scale
    X_test /= scale
    mean = np.std(X_train)
    X_train -= mean
    X_test -= mean
    input_dim = X_train.shape[1]

    # one hot encode outputs
    y_train = np_utils.to_categorical(labels)

    model = complex_model()

    X_train = X_train.reshape(X_train.shape[0], 1, 28, 28).astype('float32')
    X_test = X_test.reshape(X_test.shape[0], 1, 28, 28).astype('float32')

    # Fit the model
    #model.fit(X_train, y_train, batch_size=200, nb_epoch=20, verbose=2)
    model.fit(X_train, y_train, batch_size=1000, nb_epoch=30, verbose=2)

    predictions = model.predict_classes(X_test, verbose=1)
    result = pd.DataFrame({"ImageId": list(range(1, len(predictions) + 1)), "Label": predictions})

    result.to_csv("./predictions.csv", columns=('ImageId', 'Label'), index=None)

<commit_msg>tweaked deep conv network<commit_after>from keras.models import Sequential
from keras.utils import np_utils
from keras.layers.core import Dense, Activation, Dropout
from keras.layers import Flatten
from keras.layers.convolutional import Convolution2D
from keras.layers.convolutional import MaxPooling2D

import pandas as pd
import numpy as np

NUM_CLASSES = 10

def complex_model():
    model = Sequential()
    model.add(Convolution2D(30, 5, 5, border_mode='valid', input_shape=(1, 28, 28), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Convolution2D(15, 3, 3, activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Dense(50, activation='relu'))
    model.add(Dense(20, activation='relu'))
    model.add(Dense(NUM_CLASSES, activation='softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

if __name__ == '__main__':
    # fix random seed for reproducibility
    seed = 171717
    np.random.seed(seed)

    # read in the training and test data
    train = pd.read_csv('./data/train.csv')
    labels = train.ix[:, 0].values.astype('int32')
    X_train = (train.ix[:, 1:].values).astype('float32')
    X_test = (pd.read_csv('./data/test.csv').values).astype('float32')

    scale = np.max(X_train)
    X_train /= scale
    X_test /= scale
    mean = np.std(X_train)
    X_train -= mean
    X_test -= mean
    input_dim = X_train.shape[1]

    # one hot encode outputs
    y_train = np_utils.to_categorical(labels)

    model = complex_model()

    X_train = X_train.reshape(X_train.shape[0], 1, 28, 28).astype('float32')
    X_test = X_test.reshape(X_test.shape[0], 1, 28, 28).astype('float32')

    # Fit the model
    #model.fit(X_train, y_train, batch_size=200, nb_epoch=20, verbose=2)
    model.fit(X_train, y_train, batch_size=1000, nb_epoch=50, verbose=2)

    predictions = model.predict_classes(X_test, verbose=1)
    result = pd.DataFrame({"ImageId": list(range(1, len(predictions) + 1)), "Label": predictions})

    result.to_csv("./predictions.csv", columns=('ImageId', 'Label'), index=None)

<|endoftext|>


<commit_before>import gensim

#word2vec embeddings start with a line with the number of lines (tokens?) and the number of dimensions of the file. This allows
#gensim to allocate memory accordingly for querying the model. Larger dimensions mean larger memory is held captive. Accordingly, this line
#has to be inserted into the GloVe embeddings file.

#GloVe Model File
#More models can be downloaded from http://nlp.stanford.edu/projects/glove/
fname="glove.6B.50d.txt"

#convert Glove vectors to word2vec format
word2vec_convert_file="C:/Users/Manas/Desktop/ML/Topics_Multiclass/Zomato_Reviews/Data/IMDB/word2vec_line.txt"

#to be a first line insert
num_lines = sum(1 for line in open(fname))
dims=50

print '%d lines with %d dimensions' %(num_lines,dims)

with open(word2vec_convert_file,'w') as f:
    f.write(str(num_lines)+ " " +str(dims) + '\n')
f.close()

model_file='glove_model.txt'

filenames = [word2vec_convert_file,fname]

with open(model_file, 'w') as outfile:
    for fname in filenames:
        with open(fname) as infile:
            for line in infile:
                outfile.write(line)
outfile.close()

#load converted model file
model=gensim.models.Word2Vec.load_word2vec_format(model_file,binary=False) #GloVe Model

print model.most_similar(positive=['australia'], topn=10)
print model.similarity('woman', 'man')
<commit_msg>Delete convert.py<commit_after><|endoftext|>


<commit_before># Module dedicated to this Nirvana cover of a David Bowie song
# https://open.spotify.com/track/15VRO9CQwMpbqUYA7e6Hwg

from pint import UnitRegistry

#Update schema
__url__ = "https://raw.githubusercontent.com/KittyHawkIrc/modules/production/" + __name__ + ".py"
__version__ = 1.0

#fuck bym
unit = UnitRegistry() #can't code for shit
convert = unit.Quantity #like how do you fuck up conversions that bad

def declare():  #SERIOUSLY. no one used the 30000 hooks in here.
    return {"convert":"privmsg", "fuckbym":"privmsg", "c":"privmsg"}

def tempconv(unit):     #the library is a little iffy on names
    c = ['c', 'celc', 'celsius']
    f = ['f', 'far', 'fahrenheit']
    k = ['k', 'kel', 'kelvin']
    r = ['r', 'rank', 'rankine']

    unit = unit.lower()

    if unit in c:
        return 'degC'

    elif unit in f:
        return 'degF'

    elif unit in k:
        return 'degK'

    elif unit in r:
        return 'degR'

    else:
        return unit

def prettytemp(unit, num):

    if unit == 'degC':
        return 'C'

    elif unit == 'degF':
        return 'F'

    elif unit == 'degK':
        return 'K'

    elif unit == 'degR':
        return 'R'

    else:
        if num == 1:
            return unit
        else:
            return unit + 's'

def stringparse(text): #it took me 15 mintues to write a better string parser than bym took a week
    text_list = text.split()

    if len(text_list) < 2:  #2c f is the smallest I can think of
        return False, False, False

    unit2 = text_list[len(text_list) - 1] #it's implied the last string is always going to be the conversion
                                      #thus, '2 cm wooooo 2k16 wooooo in' would still work

    if text_list[0].isdigit(): #2 cm
        num = text_list[0]
        unit1 = text_list[1]

    else:   #2c/2cm/etc
        num = ''
        unit1 = ''

        for i in text_list[0]:
            if i.isdigit():
                num += i
            elif i.isalpha():
                unit1 += i

    unit1 = tempconv(unit1) #fixing any naming issues
    unit2 = tempconv(unit2)

    return unit1, float(num), unit2

def callback(self): #remove_bym
    try:
        unit1, num, unit2 = stringparse(self.message.split(' ', 1)[1])
    except:
        return self.msg(self.msg, 'You have to actually add things after the command')

    if unit1 == False:
        return self.msg(self.channel, "Error, it doesn't look like you're trying to convert properly")

    try:
        num, unit2 = '{0}'.format(convert(num, unit(unit1)).to(unit2)).split() #dirty, but it works perfectly

        num = round(float(num), 3)
        unit2 = prettytemp(unit2, num)                                               #things bym can't say ^

    except:
        return self.msg(self.channel, 'Conversion error!')

    return self.msg(self.channel, "%s %s" % (num, unit2))

class api: #You're now known as remove_bym
    def msg(self, channel, text):
        return "[%s] %s" % (channel, text)

if __name__ == "__main__":
    api = api()
    setattr(api, 'isop', True)
    setattr(api, 'type', 'privmsg')
    setattr(api, 'command', 'convert')
    setattr(api, 'channel', "#test")

    setattr(api, 'user', 'joe!username@hostmask')
    setattr(api, 'message', '^convert 18cm in')

    if not 'inchs' in callback(api):
        exit(1)

    setattr(api, 'message', '^convert 18 cm to in')
    if not 'inchs' in callback(api):
        exit(2)

    setattr(api, 'message', '^convert 18 cm in')
    if not 'inchs' in callback(api):
        exit(3)

    setattr(api, 'message', '^convert 18cm in in')
    if not 'inchs' in callback(api):
        exit(4)
<commit_msg>[convert] 4 changes, see description<commit_after># Module dedicated to this Nirvana cover of a David Bowie song
# https://open.spotify.com/track/15VRO9CQwMpbqUYA7e6Hwg

from pint import UnitRegistry

#Update schema
__url__ = "https://raw.githubusercontent.com/KittyHawkIrc/modules/production/" + __name__ + ".py"
__version__ = 1.0

#fuck bym
unit = UnitRegistry() #can't code for shit
convert = unit.Quantity #like how do you fuck up conversions that bad

def declare():  #SERIOUSLY. no one used the 30000 hooks in here.
    return {"convert":"privmsg", "fuckbym":"privmsg", "c":"privmsg"}

def tempconv(unit):     #the library is a little iffy on names
    c = ['c', 'celc', 'celsius']
    f = ['f', 'far', 'fahrenheit']
    k = ['k', 'kel', 'kelvin']
    r = ['r', 'rank', 'rankine']

    unit = unit.lower()

    if unit in c:
        return 'degC'

    elif unit in f:
        return 'degF'

    elif unit in k:
        return 'degK'

    elif unit in r:
        return 'degR'

    else:
        return unit

def prettytemp(unit, num):

    if unit == 'degC':
        return 'C'

    elif unit == 'degF':
        return 'F'

    elif unit == 'degK':
        return 'K'

    elif unit == 'degR':
        return 'R'

    elif unit == 'foot':
        return 'feet'

    else:
        if num == 1:
            return unit
        else:
            return unit + 's'

def stringparse(text): #it took me 15 mintues to write a better string parser than bym took a week
    text_list = text.split()

    if len(text_list) < 1:  #18F is the smallest I can think of
        return False, False, False

    unit2 = text_list[len(text_list) - 1] #it's implied the last string is always going to be the conversion
                                      #thus, '2 cm wooooo 2k16 wooooo in' would still work

    if text_list[0].replace('.', '').isdigit(): #2 cm (decimal workaround)
        num = text_list[0]
        unit1 = text_list[1]

    elif text_list[0].isalpha(): # ^c kg lbs
        num = 1
        unit1 = text_list[0]

    else:   #2c/2cm/etc
        num = ''
        unit1 = ''

        for i in text_list[0]:
            if i.isdigit():
                num += i
            elif i.isalpha():
                unit1 += i
            elif i == '.': #decimal workaround
                num += i

    unit1 = tempconv(unit1) #fixing any naming issues
    unit2 = tempconv(unit2)

    if len(text_list) <= 2 and unit1.startswith('deg'): #support single item conversions for temp
        if unit1 == 'degF': #We can safely assume this is either 18f or 18 f
            unit2 = 'degC'

        elif unit1 == 'degC':
            unit2 = 'degF'

    return unit1, float(num), unit2

def callback(self): #remove_bym
    try:
        unit1, num, unit2 = stringparse(self.message.split(' ', 1)[1])
    except:
        return self.msg(self.msg, 'You have to actually add things after the command')

    if unit1 == False:
        return self.msg(self.channel, "Error, it doesn't look like you're trying to convert properly")

    try:
        num, unit2 = '{0}'.format(convert(num, unit(unit1)).to(unit2)).split() #dirty, but it works perfectly

        num = str(round(float(num), 3)).replace('.0', '') #fix 123.0 bug
        unit2 = prettytemp(unit2, float(num))                                               #things bym can't say ^

    except Exception as e:
        return self.msg(self.channel, 'Conversion error! (%s)' % (e))

    return self.msg(self.channel, "%s %s" % (num, unit2))

class api: #You're now known as remove_bym
    def msg(self, channel, text):
        return "[%s] %s" % (channel, text)

if __name__ == "__main__":
    api = api()
    setattr(api, 'isop', True)
    setattr(api, 'type', 'privmsg')
    setattr(api, 'command', 'convert')
    setattr(api, 'channel', "#test")
    setattr(api, 'user', 'joe!username@hostmask')

    setattr(api, 'message', '^convert 18.5 cm in')
    if not 'inchs' in callback(api):
        exit(1)

    setattr(api, 'message', '^convert 5 cm to mm')
    if '.0' in callback(api):
        exit(2)

    setattr(api, 'message', '^convert kg lbs')
    if not 'pounds' in callback(api):
        exit(3)

    setattr(api, 'message', '^c 18f')
    if not 'C' in callback(api):
        exit(4)
<|endoftext|>


<commit_before>import boto3
import ffmpy
import json

s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
dynamo = boto3.resource('dynamodb')
table = dynamo.Table('FT_SegmentState')
# We don't interact with SQS in this job because it runs in parallel with so many others.

def lambda_handler(event, context):
    # Get the object from the event and show its content type

#    print(json.dumps(event, context))
#    for record in event['Records']:
#        print record
    ConversionID = event['ConversionID']
    SegmentID = event['SegmentID']
#    response = table.get_item(
#    Key={
#        'conversionID': conversion,
#        'segmentID': segment
#    }

    global bucket
    bucket = event['Records'][0]['s3']['bucket']['name']
    global key
    key = event['Records'][0]['s3']['object']['key']

    print "key is {}".format(key)
    print "bucket is {}".format(bucket)

    if not key.endswith('/'):
        try:
            # Finagle S3 bucket naming conventions so that boto retrieves the correct file.
            global split_key
            split_key = key.split('/')
            global file_name
            file_name = split_key[-1]
            print "Downloading source file..."
            s3_client.download_file(bucket, key, '/tmp/'+file_name)

            transcode()

            global destination
            destination = 'Converted/'+transportstream
            print "Uploading to s3..."
            return = s3_client.upload_file('/tmp/'+transportstream, bucket, destination)
            if return == 1:
                table.update_item(
                key={
                    'SegmentID': SegmentID,
                    },
                UpdateExpression="set Completed = 1",
                )

                segmentstatus = table.get_item

                # Update SegmentState with segment completion
                # Check if all segments are complete: if they are, trigger concat step
                allsegments = table.query(KeyConditionExpression=Key('ConversionID').eq(ConversionID))
                allstatus = allsegments['Completed']
                if checksegments(allstatus):
                    nexttable = dynamo.Table('FT_ConversionState')
                    nexttable.update_item(
                       Key={
                            'ConversionID': conversionID,
                        },
                        UpdateExpression="set ConcatReady = 1",
                    )
        except Exception as e:
            print(e)
            print('ERROR! Key: {} Bucket: {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
            raise e

# Converts video segment
def transcode():
    if key is not None:
        print "Converting File..."
        global convertedfile
        convertedfile = file_name+'CONVERTED.mp4'
        ff = ffmpy.FFmpeg(
        executable='./ffmpeg/ffmpeg',
        inputs={'/tmp/'+file_name : None},
        outputs={'/tmp/'+convertedfile : '-y'}
        )
        ff.run()
        print "Transcoding to lossless transport stream..."
        global transportstream
        transportstream = file_name+'.ts'
        fff = ffmpy.FFmpeg(
        executable='./ffmpeg/ffmpeg',
        inputs={'/tmp/'+convertedfile : None},
        outputs={'/tmp/'+transportstream : '-y -c copy -bsf:v h264_mp4toannexb -f mpegts'}
        )
        fff.run()

# Checks if all segments are complete
def checksegments(iterator):
    iterator = iter(iterator)
    try:
        first = next(iterator)
    except StopIteration:
        return True
    return all(first == rest for rest in iterator)
<commit_msg>update SegmentState with segment completion when conversion is done for each segment<commit_after>import boto3
import ffmpy
import json

s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
dynamo = boto3.resource('dynamodb')
table = dynamo.Table('FT_SegmentState')
# We don't interact with SQS in this job because it runs in parallel with so many others.

def lambda_handler(event, context):
    # Get the object from the event and show its content type

#    print(json.dumps(event, context))
#    for record in event['Records']:
#        print record
    ConversionID = event['ConversionID']
    SegmentID = event['SegmentID']
#    response = table.get_item(
#    Key={
#        'conversionID': conversion,
#        'segmentID': segment
#    }

    global bucket
    bucket = event['Records'][0]['s3']['bucket']['name']
    global key
    key = event['Records'][0]['s3']['object']['key']

    print "key is {}".format(key)
    print "bucket is {}".format(bucket)

    if not key.endswith('/'):
        try:
            # Finagle S3 bucket naming conventions so that boto retrieves the correct file.
            global split_key
            split_key = key.split('/')
            global file_name
            file_name = split_key[-1]
            print "Downloading source file..."
            s3_client.download_file(bucket, key, '/tmp/'+file_name)

            transcode()

            global destination
            destination = 'Converted/'+transportstream
            print "Uploading to s3..."
            return = s3_client.upload_file('/tmp/'+transportstream, bucket, destination)
            if return == 1:
                table.update_item(
                key={
                    'SegmentID': SegmentID,
                    },
                UpdateExpression="set Completed = 1",
                )

                segmentstatus = table.get_item

                # Update SegmentState with segment completion
                table.update_item(
                   Key={
                        'SegmentID': SegmentID,
                    },
                    UpdateExpression="set completed = 1",
                )
                # Check if all segments are complete: if they are, trigger concat step
                allsegments = table.query(KeyConditionExpression=Key('ConversionID').eq(ConversionID))
                allstatus = allsegments['Completed']
                if checksegments(allstatus):
                    nexttable = dynamo.Table('FT_ConversionState')
                    nexttable.update_item(
                       Key={
                            'ConversionID': conversionID,
                        },
                        UpdateExpression="set ConcatReady = 1",
                    )
        except Exception as e:
            print(e)
            print('ERROR! Key: {} Bucket: {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
            raise e

# Converts video segment
def transcode():
    if key is not None:
        print "Converting File..."
        global convertedfile
        convertedfile = file_name+'CONVERTED.mp4'
        ff = ffmpy.FFmpeg(
        executable='./ffmpeg/ffmpeg',
        inputs={'/tmp/'+file_name : None},
        outputs={'/tmp/'+convertedfile : '-y'}
        )
        ff.run()
        print "Transcoding to lossless transport stream..."
        global transportstream
        transportstream = file_name+'.ts'
        fff = ffmpy.FFmpeg(
        executable='./ffmpeg/ffmpeg',
        inputs={'/tmp/'+convertedfile : None},
        outputs={'/tmp/'+transportstream : '-y -c copy -bsf:v h264_mp4toannexb -f mpegts'}
        )
        fff.run()

# Checks if all segments are complete
def checksegments(iterator):
    iterator = iter(iterator)
    try:
        first = next(iterator)
    except StopIteration:
        return True
    return all(first == rest for rest in iterator)
<|endoftext|>


<commit_before>from math import sqrt, sin, cos, pi

def deg2cartesian(lat, lon, h):
    """convert lot & lag to earth centered xyz position"""
    f = 1 / 298.257223563
    a = 6378137
    phi = lat * pi / 180
    lamda = lon * pi / 180
    e = sqrt(2 * f - f * f)
    N = a / sqrt(1 - (e * sin(phi)) ** 2)

    x = (N + h) * cos(phi) * cos(lamda)
    y = (N + h) * cos(phi) * sin(lamda)
    z = (N * (1 - e ** 2) + h) * sin(phi)

    return (x,y,z)

<commit_msg>new function:sat2cartesian<commit_after>from math import sqrt, sin, cos, atan, pi
from sgp4.propagation import _gstime
from sgp4.ext import jday
import numpy as np

def deg2cartesian(lat, lon, h):
    """convert lot & lag to earth centered xyz position"""
    f = 1 / 298.257223563
    a = 6378137
    phi = lat * pi / 180
    lamda = lon * pi / 180
    e = sqrt(2 * f - f * f)
    N = a / sqrt(1 - (e * sin(phi)) ** 2)

    x = (N + h) * cos(phi) * cos(lamda)
    y = (N + h) * cos(phi) * sin(lamda)
    z = (N * (1 - e ** 2) + h) * sin(phi)

    return (x, y, z)

def sat2cartesian(xs, ys, zs, year, month, day, hour, minute, sec):
    """
    Convert sgp4 output(x faces vernal equinox) to earth centerd
    cartesian position(x faces prime meridian)
    """

    Tg = _gstime(jday(year, month, day, hour, minute, sec))
<|endoftext|>