<commit_before>import logging
import os
import shutil

from fusesoc.config import Config
from fusesoc import utils
from fusesoc.utils import pr_info
from fusesoc.core import OptionSectionMissing
from .simulator import Simulator

logger = logging.getLogger(__name__)

class Source(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class Verilator(Simulator):

    TOOL_NAME = 'VERILATOR'
    def __init__(self, system):
        super(Verilator, self).__init__(system)

        self.verilator_options = []
        self.src_files = []
        self.include_files = []
        self.include_dirs = []
        self.tb_toplevel = ""
        self.src_type = ''
        self.define_files = []
        self.libs = []
        self.top_module = ""

        if system.verilator is not None:
             self.verilator_options  = system.verilator.verilator_options
             self.src_files          = system.verilator.src_files
             self.include_files      = system.verilator.include_files
             self.tb_toplevel        = system.verilator.tb_toplevel
             self.src_type           = system.verilator.source_type
             self.define_files       = system.verilator.define_files
             self.top_module         = system.verilator.top_module
             self.fusesoc_cli_parser = system.verilator.cli_parser and (system.verilator.cli_parser == 'fusesoc')

        if self.top_module == '':
            raise OptionSectionMissing('top_module')
        if self.tb_toplevel == '':
            raise OptionSectionMissing('tb_toplevel')

    def export(self):
        src_dir = self.system.files_root
        dst_dir = os.path.join(self.src_root, self.system.sanitized_name)
        src_files = [f.name for f in self.src_files]
        src_files += [f.name for f in self.include_files]
        src_files += [self.tb_toplevel]
        dirs = list(set(map(os.path.dirname, src_files)))
        for d in dirs:
            if not os.path.exists(os.path.join(dst_dir, d)):
                os.makedirs(os.path.join(dst_dir, d))

        for f in src_files:
            if(os.path.exists(os.path.join(src_dir, f))):
                shutil.copyfile(os.path.join(src_dir, f),
                                os.path.join(dst_dir, f))

    def configure(self, args):
        skip = not self.fusesoc_cli_parser
        super(Verilator, self).configure(args, skip_params = skip)
        self.export()
        self._write_config_files()
        #self.object_files = [os.path.splitext(os.path.basename(s))[0]+'.o' for s in self.src_files]

    def _write_config_files(self):
        self.verilator_file = 'input.vc'
        f = open(os.path.join(self.sim_root,self.verilator_file),'w')

        incdirs = set()
        src_files = []

        (src_files, incdirs) = self._get_fileset_files(['synth', 'verilator'])
        for include_dir in incdirs:
            f.write("+incdir+" + include_dir + '\n')
        for src_file in src_files:
            f.write(src_file.name + '\n')
        f.close()
        #convert verilog defines into C file
        for files in self.define_files:
            read_file = os.path.join(self.src_root,files)
            write_file = os.path.join(os.path.dirname(os.path.join(self.sim_root,self.tb_toplevel)),os.path.splitext(os.path.basename(files))[0]+'.h')
            utils.convert_V2H(read_file, write_file)

    def _verilate(self):
        if self.src_type == 'systemC':
            args = ['--sc']
        else:
            args = ['--cc']
        args += ['-f', self.verilator_file]
        args += ['--top-module', self.top_module]
        args += ['--exe']
        args += ['-LDFLAGS "']
        args += ['-Wl,--start-group']
        args += [os.path.join(self.sim_root, s) for s in self.archives]
        args += ['-Wl,--end-group']
        args += [l for l in self.libs]
        args += ['"']
        args += ['-CFLAGS ' + '-I' + i for i in self.include_dirs]
        args += [os.path.join(self.src_root, self.system.sanitized_name, self.tb_toplevel)]
        for key, value in self.vlogparam.items():
            args += ['-G{}={}'.format(key, value)]
        args += self.verilator_options

        cmd = utils.find_verilator()
        if cmd is None:
             raise RuntimeError("VERILATOR_ROOT not set and there is no verilator program in your PATH")
        cmd += ' ' + ' '.join(args)

        if (self.src_type == 'systemC') and not utils.check_systemc_env():
            raise RuntimeError("Need $SYSTEMC_LIBDIR and $SYSTEMC_INCLUDE in environment or when Verilator configured")

        l = utils.Launcher(cmd,
                           shell=True,
                           cwd = self.sim_root,
                           stderr = open(os.path.join(self.sim_root,'verilator.log'),'w'),
                           stdout = open(os.path.join(self.sim_root, 'verilator.out.log'),'w')
        )
        print('')
        pr_info("Starting Verilator:")
        if Config().verbose:
             pr_info("  Verilator working dir: " + self.sim_root)
             pr_info("  Verilator command: " + cmd)
        print('')
        l.run()

    def build(self):
        super(Verilator, self).build()
        self.archives = []
        for core_name in self.cores:
            core = self.cm.get_core(core_name)
            if core.verilator:
                 if core.verilator.archive:
                      self.archives += [core.sanitized_name+'.a']
                 self.libs += core.verilator.libs
                 self.include_dirs += [os.path.join(self.src_root, core.sanitized_name, d) for d in core.verilator.include_dirs]
        self.include_dirs += [os.path.dirname(os.path.join(self.sim_root, self.tb_toplevel))]

        self.include_dirs += [self.src_root]

        pr_info("Verilating source")
        self._verilate()
        for core_name in self.cores:
            core = self.cm.get_core(core_name)
            if core.verilator:
                 self._build(core, self.sim_root, self.src_root)

        pr_info("Building verilator executable:")
        args = ['-f', 'V' + self.top_module + '.mk', 'V' + self.top_module]
        l = utils.Launcher('make', args,
                       cwd=os.path.join(self.sim_root, 'obj_dir'),
                       stdout = open(os.path.join(self.sim_root,
                                                  'verilator.make.log'),'w'))
        if Config().verbose:
             pr_info("  Verilator executable working dir: "
                     + os.path.join(self.sim_root, 'obj_dir'))
             pr_info("  Verilator executable command: make " + ' '.join(args))
        l.run()

    def _build(self, core, sim_root, src_root):
        source_type = core.verilator.source_type
        if source_type == 'C' or source_type == '':
             self.build_C(core, sim_root, src_root)
        elif source_type == 'CPP':
            self.build_CPP(core, sim_root, src_root)
        elif source_type == 'systemC':
            self.build_SysC(core, sim_root, src_root)
        else:
            raise Source(core.verilator.source_type)

        if core.verilator._object_files:
            args = []
            args += ['rvs']
            args += [core.sanitized_name+'.a']
            args += core.verilator._object_files
            l = utils.Launcher('ar', args,
                     cwd=sim_root)
            if Config().verbose:
                pr_info("  linker working dir: " + sim_root)
                pr_info("  linker command: ar " + ' '.join(args))
            l.run()
            print()

    def build_C(self, core, sim_root, src_root):
        args = ['-c']
        args += ['-std=c99']
        args += ['-I'+src_root]
        args += ['-I'+os.path.join(src_root, core.sanitized_name, s) for s in core.verilator.include_dirs]
        for src_file in core.verilator.src_files:
            pr_info("Compiling " + src_file.name)
            l = utils.Launcher('gcc',
                     args + [os.path.join(src_root, core.sanitized_name, src_file.name)],
                         cwd=sim_root,
                         stderr = open(os.path.join(sim_root, 'gcc.err.log'),'a'),
                         stdout = open(os.path.join(sim_root, 'gcc.out.log'),'a'))
            if Config().verbose:
                pr_info("  C compilation working dir: " + sim_root)
                pr_info("  C compilation command: gcc " + ' '.join(args) + ' ' + os.path.join(src_root, core.sanitized_name, src_file.name))
            l.run()

    def build_CPP(self, core, sim_root, src_root):
        verilator_root = utils.get_verilator_root()
        if verilator_root is None:
            verilator_root = utils.get_verilator_root()
        args = ['-c']
        args += ['-I'+src_root]
        args += ['-I'+os.path.join(src_root, core.sanitized_name, s) for s in self.include_dirs]
        args += ['-Iobj_dir']
        args += ['-I'+os.path.join(verilator_root,'include')]
        args += ['-I'+os.path.join(verilator_root,'include', 'vltstd')]
        for src_file in core.verilator.src_files:
            pr_info("Compiling " + src_file.name)
            l = utils.Launcher('g++', args + [os.path.join(src_root, core.sanitized_name, src_file.name)],
                         cwd=sim_root,
                         stderr = open(os.path.join(sim_root, 'g++.err.log'),'a'))
            if Config().verbose:
                pr_info("  C++ compilation working dir: " + sim_root)
                pr_info("  C++ compilation command: g++ " + ' '.join(args) + ' ' + os.path.join(src_root, core.sanitized_name, src_file.name))
            l.run()

    def build_SysC(self, core, sim_root, src_root):
        verilator_root = utils.get_verilator_root()
        args = ['-I.']
        args += ['-MMD']
        args += ['-I'+src_root]
        args += ['-I'+s for s in self.include_dirs]
        args += ['-Iobj_dir']
        args += ['-I'+os.path.join(verilator_root,'include')]
        args += ['-I'+os.path.join(verilator_root,'include', 'vltstd')]
        args += ['-DVL_PRINTF=printf']
        args += ['-DVM_TRACE=1']
        args += ['-DVM_COVERAGE=0']
        if os.getenv('SYSTEMC_INCLUDE'):
            args += ['-I'+os.getenv('SYSTEMC_INCLUDE')]
        if os.getenv('SYSTEMC'):
            args += ['-I'+os.path.join(os.getenv('SYSTEMC'),'include')]
        args += ['-Wno-deprecated']
        if os.getenv('SYSTEMC_CXX_FLAGS'):
             args += [os.getenv('SYSTEMC_CXX_FLAGS')]
        args += ['-c']
        args += ['-g']

        for src_file in core.verilator.src_files:
            pr_info("Compiling " + src_file.name)
            l = utils.Launcher('g++', args + [os.path.join(src_root, core.sanitized_name, src_file.name)],
                         cwd=sim_root,
                         stderr = open(os.path.join(sim_root, 'g++.err.log'),'a'))
            if Config().verbose:
                pr_info("  SystemC compilation working dir: " + sim_root)
                pr_info("  SystemC compilation command: g++ " + ' '.join(args) + ' ' + os.path.join(src_root, core.sanitized_name, src_file.name))
            l.run()

    def run(self, args):
        if not self.fusesoc_cli_parser:
            self.plusarg = []
        super(Verilator, self).run(args)
        self.env = os.environ.copy()
        self.env['CORE_ROOT'] = os.path.abspath(self.system.core_root)
        self.env['BUILD_ROOT'] = os.path.abspath(self.build_root)
        self.env['SIM_ROOT'] = os.path.abspath(self.sim_root)
        if self.fusesoc_cli_parser:
            _args = []
            for key, value in self.plusarg.items():
                _args += ['+{}={}'.format(key, value)]
        else:
            _args = args
        pr_info("Running simulation")
        utils.Launcher('./V' + self.top_module,
                       _args,
                       cwd=os.path.join(self.sim_root, 'obj_dir'),
                       env = self.env).run()
<commit_msg>Run make with multiple parallel jobs<commit_after>import logging
import multiprocessing
import os
import shutil

from fusesoc.config import Config
from fusesoc import utils
from fusesoc.utils import pr_info
from fusesoc.core import OptionSectionMissing
from .simulator import Simulator

logger = logging.getLogger(__name__)

class Source(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class Verilator(Simulator):

    TOOL_NAME = 'VERILATOR'
    def __init__(self, system):
        super(Verilator, self).__init__(system)

        self.verilator_options = []
        self.src_files = []
        self.include_files = []
        self.include_dirs = []
        self.tb_toplevel = ""
        self.src_type = ''
        self.define_files = []
        self.libs = []
        self.top_module = ""

        if system.verilator is not None:
             self.verilator_options  = system.verilator.verilator_options
             self.src_files          = system.verilator.src_files
             self.include_files      = system.verilator.include_files
             self.tb_toplevel        = system.verilator.tb_toplevel
             self.src_type           = system.verilator.source_type
             self.define_files       = system.verilator.define_files
             self.top_module         = system.verilator.top_module
             self.fusesoc_cli_parser = system.verilator.cli_parser and (system.verilator.cli_parser == 'fusesoc')

        if self.top_module == '':
            raise OptionSectionMissing('top_module')
        if self.tb_toplevel == '':
            raise OptionSectionMissing('tb_toplevel')

    def export(self):
        src_dir = self.system.files_root
        dst_dir = os.path.join(self.src_root, self.system.sanitized_name)
        src_files = [f.name for f in self.src_files]
        src_files += [f.name for f in self.include_files]
        src_files += [self.tb_toplevel]
        dirs = list(set(map(os.path.dirname, src_files)))
        for d in dirs:
            if not os.path.exists(os.path.join(dst_dir, d)):
                os.makedirs(os.path.join(dst_dir, d))

        for f in src_files:
            if(os.path.exists(os.path.join(src_dir, f))):
                shutil.copyfile(os.path.join(src_dir, f),
                                os.path.join(dst_dir, f))

    def configure(self, args):
        skip = not self.fusesoc_cli_parser
        super(Verilator, self).configure(args, skip_params = skip)
        self.export()
        self._write_config_files()
        #self.object_files = [os.path.splitext(os.path.basename(s))[0]+'.o' for s in self.src_files]

    def _write_config_files(self):
        self.verilator_file = 'input.vc'
        f = open(os.path.join(self.sim_root,self.verilator_file),'w')

        incdirs = set()
        src_files = []

        (src_files, incdirs) = self._get_fileset_files(['synth', 'verilator'])
        for include_dir in incdirs:
            f.write("+incdir+" + include_dir + '\n')
        for src_file in src_files:
            f.write(src_file.name + '\n')
        f.close()
        #convert verilog defines into C file
        for files in self.define_files:
            read_file = os.path.join(self.src_root,files)
            write_file = os.path.join(os.path.dirname(os.path.join(self.sim_root,self.tb_toplevel)),os.path.splitext(os.path.basename(files))[0]+'.h')
            utils.convert_V2H(read_file, write_file)

    def _verilate(self):
        if self.src_type == 'systemC':
            args = ['--sc']
        else:
            args = ['--cc']
        args += ['-f', self.verilator_file]
        args += ['--top-module', self.top_module]
        args += ['--exe']
        args += ['-LDFLAGS "']
        args += ['-Wl,--start-group']
        args += [os.path.join(self.sim_root, s) for s in self.archives]
        args += ['-Wl,--end-group']
        args += [l for l in self.libs]
        args += ['"']
        args += ['-CFLAGS ' + '-I' + i for i in self.include_dirs]
        args += [os.path.join(self.src_root, self.system.sanitized_name, self.tb_toplevel)]
        for key, value in self.vlogparam.items():
            args += ['-G{}={}'.format(key, value)]
        args += self.verilator_options

        cmd = utils.find_verilator()
        if cmd is None:
             raise RuntimeError("VERILATOR_ROOT not set and there is no verilator program in your PATH")
        cmd += ' ' + ' '.join(args)

        if (self.src_type == 'systemC') and not utils.check_systemc_env():
            raise RuntimeError("Need $SYSTEMC_LIBDIR and $SYSTEMC_INCLUDE in environment or when Verilator configured")

        l = utils.Launcher(cmd,
                           shell=True,
                           cwd = self.sim_root,
                           stderr = open(os.path.join(self.sim_root,'verilator.log'),'w'),
                           stdout = open(os.path.join(self.sim_root, 'verilator.out.log'),'w')
        )
        print('')
        pr_info("Starting Verilator:")
        if Config().verbose:
             pr_info("  Verilator working dir: " + self.sim_root)
             pr_info("  Verilator command: " + cmd)
        print('')
        l.run()

    def build(self):
        super(Verilator, self).build()
        self.archives = []
        for core_name in self.cores:
            core = self.cm.get_core(core_name)
            if core.verilator:
                 if core.verilator.archive:
                      self.archives += [core.sanitized_name+'.a']
                 self.libs += core.verilator.libs
                 self.include_dirs += [os.path.join(self.src_root, core.sanitized_name, d) for d in core.verilator.include_dirs]
        self.include_dirs += [os.path.dirname(os.path.join(self.sim_root, self.tb_toplevel))]

        self.include_dirs += [self.src_root]

        pr_info("Verilating source")
        self._verilate()
        for core_name in self.cores:
            core = self.cm.get_core(core_name)
            if core.verilator:
                 self._build(core, self.sim_root, self.src_root)

        # Do parallel builds with <number of cpus> * 2 jobs.
        make_job_count = multiprocessing.cpu_count() * 2

        pr_info("Building verilator executable:")
        args = ['-f', 'V' + self.top_module + '.mk', '-j', str(make_job_count), 'V' + self.top_module]
        l = utils.Launcher('make', args,
                       cwd=os.path.join(self.sim_root, 'obj_dir'),
                       stdout = open(os.path.join(self.sim_root,
                                                  'verilator.make.log'),'w'))
        if Config().verbose:
             pr_info("  Verilator executable working dir: "
                     + os.path.join(self.sim_root, 'obj_dir'))
             pr_info("  Verilator executable command: make " + ' '.join(args))
        l.run()

    def _build(self, core, sim_root, src_root):
        source_type = core.verilator.source_type
        if source_type == 'C' or source_type == '':
             self.build_C(core, sim_root, src_root)
        elif source_type == 'CPP':
            self.build_CPP(core, sim_root, src_root)
        elif source_type == 'systemC':
            self.build_SysC(core, sim_root, src_root)
        else:
            raise Source(core.verilator.source_type)

        if core.verilator._object_files:
            args = []
            args += ['rvs']
            args += [core.sanitized_name+'.a']
            args += core.verilator._object_files
            l = utils.Launcher('ar', args,
                     cwd=sim_root)
            if Config().verbose:
                pr_info("  linker working dir: " + sim_root)
                pr_info("  linker command: ar " + ' '.join(args))
            l.run()
            print()

    def build_C(self, core, sim_root, src_root):
        args = ['-c']
        args += ['-std=c99']
        args += ['-I'+src_root]
        args += ['-I'+os.path.join(src_root, core.sanitized_name, s) for s in core.verilator.include_dirs]
        for src_file in core.verilator.src_files:
            pr_info("Compiling " + src_file.name)
            l = utils.Launcher('gcc',
                     args + [os.path.join(src_root, core.sanitized_name, src_file.name)],
                         cwd=sim_root,
                         stderr = open(os.path.join(sim_root, 'gcc.err.log'),'a'),
                         stdout = open(os.path.join(sim_root, 'gcc.out.log'),'a'))
            if Config().verbose:
                pr_info("  C compilation working dir: " + sim_root)
                pr_info("  C compilation command: gcc " + ' '.join(args) + ' ' + os.path.join(src_root, core.sanitized_name, src_file.name))
            l.run()

    def build_CPP(self, core, sim_root, src_root):
        verilator_root = utils.get_verilator_root()
        if verilator_root is None:
            verilator_root = utils.get_verilator_root()
        args = ['-c']
        args += ['-I'+src_root]
        args += ['-I'+os.path.join(src_root, core.sanitized_name, s) for s in self.include_dirs]
        args += ['-Iobj_dir']
        args += ['-I'+os.path.join(verilator_root,'include')]
        args += ['-I'+os.path.join(verilator_root,'include', 'vltstd')]
        for src_file in core.verilator.src_files:
            pr_info("Compiling " + src_file.name)
            l = utils.Launcher('g++', args + [os.path.join(src_root, core.sanitized_name, src_file.name)],
                         cwd=sim_root,
                         stderr = open(os.path.join(sim_root, 'g++.err.log'),'a'))
            if Config().verbose:
                pr_info("  C++ compilation working dir: " + sim_root)
                pr_info("  C++ compilation command: g++ " + ' '.join(args) + ' ' + os.path.join(src_root, core.sanitized_name, src_file.name))
            l.run()

    def build_SysC(self, core, sim_root, src_root):
        verilator_root = utils.get_verilator_root()
        args = ['-I.']
        args += ['-MMD']
        args += ['-I'+src_root]
        args += ['-I'+s for s in self.include_dirs]
        args += ['-Iobj_dir']
        args += ['-I'+os.path.join(verilator_root,'include')]
        args += ['-I'+os.path.join(verilator_root,'include', 'vltstd')]
        args += ['-DVL_PRINTF=printf']
        args += ['-DVM_TRACE=1']
        args += ['-DVM_COVERAGE=0']
        if os.getenv('SYSTEMC_INCLUDE'):
            args += ['-I'+os.getenv('SYSTEMC_INCLUDE')]
        if os.getenv('SYSTEMC'):
            args += ['-I'+os.path.join(os.getenv('SYSTEMC'),'include')]
        args += ['-Wno-deprecated']
        if os.getenv('SYSTEMC_CXX_FLAGS'):
             args += [os.getenv('SYSTEMC_CXX_FLAGS')]
        args += ['-c']
        args += ['-g']

        for src_file in core.verilator.src_files:
            pr_info("Compiling " + src_file.name)
            l = utils.Launcher('g++', args + [os.path.join(src_root, core.sanitized_name, src_file.name)],
                         cwd=sim_root,
                         stderr = open(os.path.join(sim_root, 'g++.err.log'),'a'))
            if Config().verbose:
                pr_info("  SystemC compilation working dir: " + sim_root)
                pr_info("  SystemC compilation command: g++ " + ' '.join(args) + ' ' + os.path.join(src_root, core.sanitized_name, src_file.name))
            l.run()

    def run(self, args):
        if not self.fusesoc_cli_parser:
            self.plusarg = []
        super(Verilator, self).run(args)
        self.env = os.environ.copy()
        self.env['CORE_ROOT'] = os.path.abspath(self.system.core_root)
        self.env['BUILD_ROOT'] = os.path.abspath(self.build_root)
        self.env['SIM_ROOT'] = os.path.abspath(self.sim_root)
        if self.fusesoc_cli_parser:
            _args = []
            for key, value in self.plusarg.items():
                _args += ['+{}={}'.format(key, value)]
        else:
            _args = args
        pr_info("Running simulation")
        utils.Launcher('./V' + self.top_module,
                       _args,
                       cwd=os.path.join(self.sim_root, 'obj_dir'),
                       env = self.env).run()
<|endoftext|>


<commit_before>import time
import traceback
import ConfigParser

from lib.pipeline import *
from lib.utils import *
from lib.cache import *
from lib.event import *

SYSTEM_CONF_FILE = "conf/system.conf"
PIPELINE_CONF_FILE = "conf/pipeline.conf"
BOTS_CONF_FILE = "conf/bots.conf"
LOGS_PATH = "logs/"

class Bot(object):

    def __init__(self, bot_id):
        self.bot_id = bot_id

        self.logger = self.load_logger()
        self.logger.info('Bot is starting')

        self.load_configurations()

        src_queue, dest_queues = self.load_queues()
        self.pipeline = self.load_pipeline(src_queue, dest_queues)

        self.init()

    def init(self):
        pass

    def start(self):
         self.logger.info('Bot start processing')

         count = 0
         while True:
             try:
                self.process()
                time.sleep(int(self.parameters.processing_interval))
                count = 0
             except:
                # self.close() # add this method
                self.connect()
                self.logger.error(traceback.format_exc())
                count += 1
                if count > 5:
                    self.stop()
                time.sleep(5)

    def stop(self):
        self.logger.error("Bot found an error. Exiting")
        exit(-1)

    def load_configurations(self):
        self.parameters = Parameters()
        config = ConfigParser.ConfigParser()
        config.read(BOTS_CONF_FILE)

        default_section = "default"
        self.logger.debug("Loading configuration in default section from '%s' file" % BOTS_CONF_FILE)

        if config.has_section(default_section):
            for option in config.options(default_section):
                setattr(self.parameters, option, config.get(default_section, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(default_section, option)))

        self.logger.debug("Loading configuration in %s section from '%s' file" % (self.bot_id, BOTS_CONF_FILE))

        if config.has_section(self.bot_id):
            for option in config.options(self.bot_id):
                setattr(self.parameters, option, config.get(self.bot_id, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(self.bot_id, option)))

    def load_logger(self):
        config = ConfigParser.ConfigParser()
        config.read(SYSTEM_CONF_FILE)
        loglevel = config.get('Logging','level')
        return log(LOGS_PATH, self.bot_id, loglevel)

    def load_pipeline(self, src_queue, dest_queues):
        self.logger.debug("Connecting to pipeline queues")
        return Pipeline(src_queue, dest_queues)

    def load_queues(self):
        config = ConfigParser.ConfigParser()
        config.read(PIPELINE_CONF_FILE)
        self.logger.debug("Loading pipeline queues from '%s' file" % PIPELINE_CONF_FILE)

        for option in config.options("Pipeline"):
            if option == self.bot_id:
                queues = config.get("Pipeline", self.bot_id)
                queues = queues.split('|')

                if len(queues) == 2:
                    src_queue = queues[0].strip()
                    dest_queues = queues[1].strip()

                    if src_queue == "None":
                        src_queue = None
                    if dest_queues == "None":
                        dest_queue = None

                    dest_queues_list = list()
                    for queue in dest_queues.split(','):
                        dest_queues_list.append(queue.strip())

                    self.logger.info("Source queue '%s'" % src_queue)
                    self.logger.info("Destination queue(s) '%s'" % dest_queues_list)

                    return [src_queue, dest_queues_list]

        self.logger.error("Failed to load queues")
        self.stop()

    def send_message(self, message):
        try:
            message = unicode(message)
        except:
            message = force_decode(message)
        self.pipeline.send(message)

    def receive_message(self):
        raw_message = self.pipeline.receive()
        try:
            message = Event.from_unicode(raw_message)
        except:
            message = raw_message
        return message

    def acknowledge_message(self):
        self.pipeline.acknowledge()

class Parameters(object):
    pass

<commit_msg>Update bot.py<commit_after>import time
import traceback
import ConfigParser

from lib.pipeline import *
from lib.utils import *
from lib.cache import *
from lib.event import *

SYSTEM_CONF_FILE = "conf/system.conf"
PIPELINE_CONF_FILE = "conf/pipeline.conf"
BOTS_CONF_FILE = "conf/bots.conf"
LOGS_PATH = "logs/"

class Bot(object):

    def __init__(self, bot_id):
        self.bot_id = bot_id

        self.logger = self.load_logger()
        self.logger.info('Bot is starting')

        self.load_configurations()

        src_queue, dest_queues = self.load_queues()
        self.pipeline = self.load_pipeline(src_queue, dest_queues)

        self.init()

    def init(self):
        pass

    def start(self):
         self.logger.info('Bot start processing')

         count = 0
         while True:
             try:
                self.process()
                time.sleep(int(self.parameters.processing_interval))
                count = 0
             except:
                # self.close() # add this method
                self.logger.info('Bot is restarting')
                self.connect()
                self.logger.error(traceback.format_exc())
                count += 1
                if count > 5:
                    self.stop()
                time.sleep(5)

    def stop(self):
        self.logger.error("Bot found an error. Exiting")
        exit(-1)

    def load_configurations(self):
        self.parameters = Parameters()
        config = ConfigParser.ConfigParser()
        config.read(BOTS_CONF_FILE)

        default_section = "default"
        self.logger.debug("Loading configuration in default section from '%s' file" % BOTS_CONF_FILE)

        if config.has_section(default_section):
            for option in config.options(default_section):
                setattr(self.parameters, option, config.get(default_section, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(default_section, option)))

        self.logger.debug("Loading configuration in %s section from '%s' file" % (self.bot_id, BOTS_CONF_FILE))

        if config.has_section(self.bot_id):
            for option in config.options(self.bot_id):
                setattr(self.parameters, option, config.get(self.bot_id, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(self.bot_id, option)))

    def load_logger(self):
        config = ConfigParser.ConfigParser()
        config.read(SYSTEM_CONF_FILE)
        loglevel = config.get('Logging','level')
        return log(LOGS_PATH, self.bot_id, loglevel)

    def load_pipeline(self, src_queue, dest_queues):
        self.logger.debug("Connecting to pipeline queues")
        return Pipeline(src_queue, dest_queues)

    def load_queues(self):
        config = ConfigParser.ConfigParser()
        config.read(PIPELINE_CONF_FILE)
        self.logger.debug("Loading pipeline queues from '%s' file" % PIPELINE_CONF_FILE)

        for option in config.options("Pipeline"):
            if option == self.bot_id:
                queues = config.get("Pipeline", self.bot_id)
                queues = queues.split('|')

                if len(queues) == 2:
                    src_queue = queues[0].strip()
                    dest_queues = queues[1].strip()

                    if src_queue == "None":
                        src_queue = None
                    if dest_queues == "None":
                        dest_queue = None

                    dest_queues_list = list()
                    for queue in dest_queues.split(','):
                        dest_queues_list.append(queue.strip())

                    self.logger.info("Source queue '%s'" % src_queue)
                    self.logger.info("Destination queue(s) '%s'" % dest_queues_list)

                    return [src_queue, dest_queues_list]

        self.logger.error("Failed to load queues")
        self.stop()

    def send_message(self, message):
        try:
            message = unicode(message)
        except:
            message = force_decode(message)
        self.pipeline.send(message)

    def receive_message(self):
        raw_message = self.pipeline.receive()
        try:
            message = Event.from_unicode(raw_message)
        except:
            message = raw_message
        return message

    def acknowledge_message(self):
        self.pipeline.acknowledge()

class Parameters(object):
    pass

<|endoftext|>


<commit_before>#----------------------------------------------------------------------
# Copyright (c) 2010 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------
"""
Reference GENI GCF Clearinghouse. Uses SFA Certificate and credential objects.
NOTE it requires a user cert at startup to get the full certificate chain.
Run from gch.py
Will produce signed user credentials from a GID, return a
(hard coded) list of aggregates, and create a new Slice Credential.

"""

import logging
import os
import uuid

from SecureXMLRPCServer import SecureXMLRPCServer
from credential import CredentialVerifier, create_credential, publicid_to_urn
import sfa.trust.gid as gid
import sfa.trust.certificate as cert

# Substitute eg "openflow//stanford ch"
# Be sure this matches init-ca.py:GCF_CERT_PREFIX
#   with // -> :
SLICEPUBID_PREFIX = "geni.net//gpo//gcf"
SLICE_GID_SUBJ = "gcf.slice"

# Credential lifetimes in seconds
# Extend slice lifetimes to actually use the resources
USER_CRED_LIFE = 86400
SLICE_CRED_LIFE = 3600

# The list of Aggregates that this Clearinghouse knows about
# should be defined in the geni_aggregates file
# ListResources will refer the client to these aggregates
# Clearinghouse.runserver currently does the register_aggregate_pair
# calls for each row in that file
# but this should be doable dynamically
# Some sample pairs:
# GPOMYPLC = ('urn:publicid:IDN+plc:gpo1+authority+sa',
#             'http://myplc1.gpolab.bbn.com:12348')
# TESTGCFAM = ('urn:publicid:IDN+geni.net:gpo+authority+gcf',
#              'https://127.0.0.1:8001')
# OTHERGPOMYPLC = ('urn:publicid:IDN+plc:gpo+authority+site2',
#                    'http://128.89.81.74:12348')
# ELABINELABAM = ('urn:publicid:IDN+elabinelab.geni.emulab.net',
#                 'https://myboss.elabinelab.geni.emulab.net:443/protogeni/xmlrpc/am')

class ClearinghouseServer(object):
    """The public API for the Clearinghouse.  This class provides the
    XMLRPC interface and invokes a delegate for all the operations.

    """

    def __init__(self, delegate):
        self._delegate = delegate

    def GetVersion(self):
        return self._delegate.GetVersion()

    def Resolve(self, urn):
        return self._delegate.Resolve(urn)

class SampleClearinghouseServer(ClearinghouseServer):
    """Add Additional functionality that is not part of the geni API
    specification.

    """

    def __init__(self, delegate):
        self._delegate = delegate

    def CreateSlice(self, urn=None):
        return self._delegate.CreateSlice(urn_req=urn)

    def DeleteSlice(self, urn):
        return self._delegate.DeleteSlice(urn)

    def ListAggregates(self):
        return self._delegate.ListAggregates()

    def CreateUserCredential(self, cert):
        return self._delegate.CreateUserCredential(cert)

class Clearinghouse(object):

    def __init__(self):
        self.logger = logging.getLogger('gch')
        self.slices = {}
        self.aggs = []

    def register_aggregate_pair(self, aggpair):
        '''Add an aggregate URN and URL pair to the known set. URL is unverified'''
        if aggpair is None:
            self.logger.debug('Null aggpair ignored')
            return
        if len(aggpair) != 2:
            self.logger.debug('Len %d aggpair ignored', len(aggpair))
            return
        if aggpair[0] is None or aggpair[0].strip() == "":
            self.logger.debug('Empty URN in Aggpair - ignore')
            return
        if aggpair[1] is None or aggpair[1].strip() == "":
            self.logger.debug('Empty URL in Aggpair for URN %s: ignore', aggpair[0])
            return

        # TODO: Avoid dupes, confirm the AM is reachable, ?
        # Real CH would require an AM contact, AM cert, ?
        self.logger.info("Registering AM %r", aggpair)
        self.aggs.append(aggpair)

        # ca_certs is a file of 1 ca cert possibly, preferably a dir of several
    def runserver(self, addr, keyfile=None, certfile=None,
                  ca_certs=None, aggfile=None):
        """Run the clearinghouse server."""
        self.keyfile = keyfile
        self.certfile = certfile

        # Error check the keyfile, certfile all exist
        if keyfile is None or not os.path.isfile(os.path.expanduser(keyfile)):
            raise Exception("Missing CH key file %s" % keyfile)
        if certfile is None or not os.path.isfile(os.path.expanduser(certfile)):
            raise Exception("Missing CH cert file %s" % certfile)

        if ca_certs is None:
            raise Exception("Missing CA cert(s) arg")

        if not os.path.exists(os.path.expanduser(ca_certs)):
            raise Exception("Missing CA cert(s): %s" % ca_certs)

        # Load up the aggregates
        if os.path.isfile(aggfile):
            for line in file(aggfile):
                spl = line.strip().split(',')
                if len(spl) == 2:
                    if spl[0].strip() == "":
                        self.logger.info("Empty URN for AM URL %s" % spl[1])
                        continue
                    elif spl[1].strip() == "":
                        self.logger.info("Empty URL for AM URN %s" % spl[0])
                        continue
                    # FIXME: Check to see if that URN is known? That URL?
                    self.register_aggregate_pair((spl[0].strip(),spl[1].strip()))
        self.logger.info("%d Aggregate Managers registered from aggregates file %r", len(self.aggs), aggfile)

        # This is the arg to _make_server
        ca_certs_onefname = CredentialVerifier.getCAsFileFromDir(ca_certs)

        # This is used below by CreateSlice
        self.ca_cert_fnames = []
        if os.path.isfile(os.path.expanduser(ca_certs)):
            self.ca_cert_fnames = [os.path.expanduser(ca_certs)]
        elif os.path.isdir(os.path.expanduser(ca_certs)):
            self.ca_cert_fnames = [os.path.join(os.path.expanduser(ca_certs), name) for name in os.listdir(os.path.expanduser(ca_certs)) if name != CredentialVerifier.CATEDCERTSFNAME]

        # Create the xmlrpc server, load the rootkeys and do the ssl thing.
        self._server = self._make_server(addr, keyfile, certfile,
                                         ca_certs_onefname)
        self._server.register_instance(SampleClearinghouseServer(self))
        self.logger.info('GENI CH Listening on port %d...' % (addr[1]))
        self._server.serve_forever()

    def _make_server(self, addr, keyfile=None, certfile=None,
                     ca_certs=None):
        """Creates the XML RPC server."""
        # ca_certs is a file of concatenated certs
        return SecureXMLRPCServer(addr, keyfile=keyfile, certfile=certfile,
                                  ca_certs=ca_certs)

    def GetVersion(self):
        self.logger.info("Called GetVersion")
        version = dict()
        version['geni_api'] = 1
        version['pg_api'] = 2
        version['geni_stitching'] = False
        return version

    def Resolve(self, urn):
        self.logger.info("Called Resolve URN %s" % urn)
        result = dict()
        result['urn'] = urn
        return result

    def CreateSlice(self, urn_req = None):
        self.logger.info("Called CreateSlice URN REQ %r" % urn_req)
        if urn_req and self.slices.has_key(urn_req):
            # FIXME: If the Slice has expired, treat this as
            # a request to renew
            return self.slices[urn_req].save_to_string()

        # FIXME: Validate urn_req has the right form
        # to be issued by this CH

        # Create a random uuid for the slice
        slice_uuid = uuid.uuid4()
        # Where was the slice created?
        (ipaddr, port) = self._server.socket._sock.getsockname()
        # FIXME: Get public_id start from a properties file
        public_id = 'IDN %s slice %s//%s:%d' % (SLICEPUBID_PREFIX, slice_uuid.__str__()[4:12],
                                                                   ipaddr,
                                                                   port)
        if urn_req:
            urn = urn_req
        else:
            # this func adds the urn:publicid:
            # and converts spaces to +'s, and // to :
            urn = publicid_to_urn(public_id)

        # Create a credential authorizing this user to use this slice.
        slice_gid = self.create_slice_gid(slice_uuid, urn)[0]

        # Get client x509 cert from the SSL connection
        # It doesnt have the chain but should be signed
        # by this CHs cert, which should also be a trusted
        # root at any federated AM. So everyone can verify it as is.
        # Note that if a user from a different CH (installed
        # as trusted by this CH for some reason) called this method,
        # that user would be used here - and should still work.
        # If there are problems, the fix would be
        # If cert.Certificate(string=server.pem_cert).is_signed_by_cert(cert.Certificate(self.certfile))
        # do nothing
        # else for cafname in self.ca_Cert_fnames
        # do same for those
        # user_gid = gid.GID (string=str(self.pem_Cert+serverstr))
        # But I really dont think its all needed
        user_gid = gid.GID(string=self._server.pem_cert)

        # OK have a user_gid so can get a slice credential
        try:
            slice_cred = self.create_slice_credential(user_gid,
                                                      slice_gid)
        except Exception:
            import traceback
            self.logger.error('CreateSlice failed to get slice credential for user %r, slice %r: %s', user_gid, slice_gid, traceback.format_exc())
            raise
        self.logger.info('Created slice %r' % (urn))

        self.slices[urn] = slice_cred

        return slice_cred.save_to_string()

    def DeleteSlice(self, urn_req):
        self.logger.info("Called DeleteSlice %r" % urn_req)
        if self.slices.has_key(urn_req):
            self.slices.pop(urn_req)
            return True
        self.logger.info('Slice was not found')
        # Slice not found!
        return False

    def ListAggregates(self):
        self.logger.info("Called ListAggregates")
        # TODO: Allow dynamic registration of aggregates
        return self.aggs

    def create_slice_gid(self, subject, slice_urn):
        # FIXME: Validate the slice_urn has the right prefix
        # FIXME: subject is ignored?
        newgid = gid.GID(create=True, subject=SLICE_GID_SUBJ, uuid=gid.create_uuid(), urn=slice_urn)
        keys = cert.Keypair(create=True)
        newgid.set_pubkey(keys)
        issuer_key = cert.Keypair(filename=self.keyfile)
        issuer_cert = gid.GID(filename=self.certfile)
        newgid.set_issuer(issuer_key, cert=issuer_cert)
        newgid.set_parent(issuer_cert)
        newgid.encode()
        newgid.sign()
        return newgid, keys

    def CreateUserCredential(self, user_gid):
        '''Return string representation of a user credential
        issued by this CH with caller/object this user_gid (string)
        with user privileges'''
        # FIXME: Validate arg - non empty, my user
        user_gid = gid.GID(string=user_gid)
        self.logger.info("Called CreateUserCredential for GID %s" % user_gid.get_hrn())
        ucred = create_credential(user_gid, user_gid, USER_CRED_LIFE, 'user', self.keyfile, self.certfile)
        return ucred.save_to_string()

    def create_slice_credential(self, user_gid, slice_gid):
        '''Create a Slice credential object for this user_gid (object) on given slice gid (object)'''
        # FIXME: Validate the user_gid and slice_gid
        # are my user and slice
        return create_credential(user_gid, slice_gid, SLICE_CRED_LIFE, 'slice', self.keyfile, self.certfile)

<commit_msg>If no trusted roots arg received just use the CHs own cert. Refactor slice creation to only construct a publicid if we will use it cause no URN was supplied. Comments.<commit_after>#----------------------------------------------------------------------
# Copyright (c) 2010 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------
"""
Reference GENI GCF Clearinghouse. Uses SFA Certificate and credential objects.
Run from gch.py
Will produce signed user credentials from a GID, return a
list of aggregates read from a config file, and create a new Slice Credential.

"""

import logging
import os
import uuid

from SecureXMLRPCServer import SecureXMLRPCServer
from credential import CredentialVerifier, create_credential, publicid_to_urn
import sfa.trust.gid as gid
import sfa.trust.certificate as cert

# Substitute eg "openflow//stanford ch"
# Be sure this matches init-ca.py:CERT_AUTHORITY
# This is in publicid format
SLICEPUBID_PREFIX = "geni.net//gpo//gcf"

# FIXME: Explain relationship of this name to above
# Subject of the Slice credential in URN notation
SLICE_GID_SUBJ = "gcf.slice"

# Credential lifetimes in seconds
# Extend slice lifetimes to actually use the resources
USER_CRED_LIFE = 86400
SLICE_CRED_LIFE = 3600

# The list of Aggregates that this Clearinghouse knows about
# should be defined in the geni_aggregates file
# ListResources will refer the client to these aggregates
# Clearinghouse.runserver currently does the register_aggregate_pair
# calls for each row in that file
# but this should be doable dynamically
# Some sample pairs:
# GPOMYPLC = ('urn:publicid:IDN+plc:gpo1+authority+sa',
#             'http://myplc1.gpolab.bbn.com:12348')
# TESTGCFAM = ('urn:publicid:IDN+geni.net:gpo+authority+gcf',
#              'https://127.0.0.1:8001')
# OTHERGPOMYPLC = ('urn:publicid:IDN+plc:gpo+authority+site2',
#                    'http://128.89.81.74:12348')
# ELABINELABAM = ('urn:publicid:IDN+elabinelab.geni.emulab.net',
#                 'https://myboss.elabinelab.geni.emulab.net:443/protogeni/xmlrpc/am')

class ClearinghouseServer(object):
    """The public API for the Clearinghouse.  This class provides the
    XMLRPC interface and invokes a delegate for all the operations.

    """

    def __init__(self, delegate):
        self._delegate = delegate

    def GetVersion(self):
        return self._delegate.GetVersion()

    def Resolve(self, urn):
        return self._delegate.Resolve(urn)

class SampleClearinghouseServer(ClearinghouseServer):
    """Add Additional functionality that is not part of the geni API
    specification.

    """

    def __init__(self, delegate):
        self._delegate = delegate

    def CreateSlice(self, urn=None):
        return self._delegate.CreateSlice(urn_req=urn)

    def DeleteSlice(self, urn):
        return self._delegate.DeleteSlice(urn)

    def ListAggregates(self):
        return self._delegate.ListAggregates()

    def CreateUserCredential(self, cert):
        return self._delegate.CreateUserCredential(cert)

class Clearinghouse(object):

    def __init__(self):
        self.logger = logging.getLogger('gch')
        self.slices = {}
        self.aggs = []

    def register_aggregate_pair(self, aggpair):
        '''Add an aggregate URN and URL pair to the known set. URL is unverified'''
        if aggpair is None:
            self.logger.debug('Null aggpair ignored')
            return
        if len(aggpair) != 2:
            self.logger.debug('Len %d aggpair ignored', len(aggpair))
            return
        if aggpair[0] is None or aggpair[0].strip() == "":
            self.logger.debug('Empty URN in Aggpair - ignore')
            return
        if aggpair[1] is None or aggpair[1].strip() == "":
            self.logger.debug('Empty URL in Aggpair for URN %s: ignore', aggpair[0])
            return

        # TODO: Avoid dupes, confirm the AM is reachable, ?
        # Real CH would require an AM contact, AM cert, ?
        self.logger.info("Registering AM %r", aggpair)
        self.aggs.append(aggpair)

        # ca_certs is a file of 1 ch cert possibly (itself), preferably a dir of several
    def runserver(self, addr, keyfile=None, certfile=None,
                  ca_certs=None, aggfile=None):
        """Run the clearinghouse server."""
        self.keyfile = keyfile
        self.certfile = certfile

        # Error check the keyfile, certfile all exist
        if keyfile is None or not os.path.isfile(os.path.expanduser(keyfile)):
            raise Exception("Missing CH key file %s" % keyfile)
        if certfile is None or not os.path.isfile(os.path.expanduser(certfile)):
            raise Exception("Missing CH cert file %s" % certfile)

        if ca_certs is None:
            ca_certs = certfile
            self.logger.info("Using only my CH cert as a trusted root cert")

        if not os.path.exists(os.path.expanduser(ca_certs)):
            raise Exception("Missing CA cert(s): %s" % ca_certs)

        # Load up the aggregates
        if os.path.isfile(aggfile):
            for line in file(aggfile):
                spl = line.strip().split(',')
                if len(spl) == 2:
                    if spl[0].strip() == "":
                        self.logger.info("Empty URN for AM URL %s" % spl[1])
                        continue
                    elif spl[1].strip() == "":
                        self.logger.info("Empty URL for AM URN %s" % spl[0])
                        continue
                    # FIXME: Check to see if that URN is known? That URL?
                    self.register_aggregate_pair((spl[0].strip(),spl[1].strip()))
        self.logger.info("%d Aggregate Managers registered from aggregates file %r", len(self.aggs), aggfile)

        # This is the arg to _make_server
        ca_certs_onefname = CredentialVerifier.getCAsFileFromDir(ca_certs)

        # This is used below by CreateSlice
        self.ca_cert_fnames = []
        if os.path.isfile(os.path.expanduser(ca_certs)):
            self.ca_cert_fnames = [os.path.expanduser(ca_certs)]
        elif os.path.isdir(os.path.expanduser(ca_certs)):
            self.ca_cert_fnames = [os.path.join(os.path.expanduser(ca_certs), name) for name in os.listdir(os.path.expanduser(ca_certs)) if name != CredentialVerifier.CATEDCERTSFNAME]

        # Create the xmlrpc server, load the rootkeys and do the ssl thing.
        self._server = self._make_server(addr, keyfile, certfile,
                                         ca_certs_onefname)
        self._server.register_instance(SampleClearinghouseServer(self))
        self.logger.info('GENI CH Listening on port %d...' % (addr[1]))
        self._server.serve_forever()

    def _make_server(self, addr, keyfile=None, certfile=None,
                     ca_certs=None):
        """Creates the XML RPC server."""
        # ca_certs is a file of concatenated certs
        return SecureXMLRPCServer(addr, keyfile=keyfile, certfile=certfile,
                                  ca_certs=ca_certs)

    def GetVersion(self):
        self.logger.info("Called GetVersion")
        version = dict()
        version['geni_api'] = 1
        version['pg_api'] = 2
        version['geni_stitching'] = False
        return version

    def Resolve(self, urn):
        self.logger.info("Called Resolve URN %s" % urn)
        result = dict()
        result['urn'] = urn
        return result

    def CreateSlice(self, urn_req = None):
        self.logger.info("Called CreateSlice URN REQ %r" % urn_req)
        if urn_req and self.slices.has_key(urn_req):
            # FIXME: If the Slice has expired, treat this as
            # a request to renew
            return self.slices[urn_req].save_to_string()

        # FIXME: Validate urn_req has the right form
        # to be issued by this CH

        # Create a random uuid for the slice
        slice_uuid = uuid.uuid4()

        if urn_req:
            urn = urn_req
        else:
            # Where was the slice created?
            (ipaddr, port) = self._server.socket._sock.getsockname()
            # FIXME: Get public_id start from a properties file
            public_id = 'IDN %s slice %s//%s:%d' % (SLICEPUBID_PREFIX, slice_uuid.__str__()[4:12],
                                                                   ipaddr,
                                                                   port)
            # this func adds the urn:publicid:
            # and converts spaces to +'s, and // to :
            urn = publicid_to_urn(public_id)

        # Create a credential authorizing this user to use this slice.
        slice_gid = self.create_slice_gid(slice_uuid, urn)[0]

        # Get client x509 cert from the SSL connection
        # It doesnt have the chain but should be signed
        # by this CHs cert, which should also be a trusted
        # root at any federated AM. So everyone can verify it as is.
        # Note that if a user from a different CH (installed
        # as trusted by this CH for some reason) called this method,
        # that user would be used here - and should still work.
        # If there are problems, the fix would be
        # If cert.Certificate(string=server.pem_cert).is_signed_by_cert(cert.Certificate(self.certfile))
        # do nothing
        # else for cafname in self.ca_Cert_fnames
        # do same for those
        # user_gid = gid.GID (string=str(self.pem_Cert+serverstr))
        # But I really dont think its all needed
        user_gid = gid.GID(string=self._server.pem_cert)

        # OK have a user_gid so can get a slice credential
        try:
            slice_cred = self.create_slice_credential(user_gid,
                                                      slice_gid)
        except Exception:
            import traceback
            self.logger.error('CreateSlice failed to get slice credential for user %r, slice %r: %s', user_gid, slice_gid, traceback.format_exc())
            raise
        self.logger.info('Created slice %r' % (urn))

        self.slices[urn] = slice_cred

        return slice_cred.save_to_string()

    def DeleteSlice(self, urn_req):
        self.logger.info("Called DeleteSlice %r" % urn_req)
        if self.slices.has_key(urn_req):
            self.slices.pop(urn_req)
            return True
        self.logger.info('Slice was not found')
        # Slice not found!
        return False

    def ListAggregates(self):
        self.logger.info("Called ListAggregates")
        # TODO: Allow dynamic registration of aggregates
        return self.aggs

    def create_slice_gid(self, subject, slice_urn):
        # FIXME: Validate the slice_urn has the right prefix
        # FIXME: subject is ignored?
        newgid = gid.GID(create=True, subject=SLICE_GID_SUBJ, uuid=gid.create_uuid(), urn=slice_urn)
        keys = cert.Keypair(create=True)
        newgid.set_pubkey(keys)
        issuer_key = cert.Keypair(filename=self.keyfile)
        issuer_cert = gid.GID(filename=self.certfile)
        newgid.set_issuer(issuer_key, cert=issuer_cert)
        newgid.set_parent(issuer_cert)
        newgid.encode()
        newgid.sign()
        return newgid, keys

    def CreateUserCredential(self, user_gid):
        '''Return string representation of a user credential
        issued by this CH with caller/object this user_gid (string)
        with user privileges'''
        # FIXME: Validate arg - non empty, my user
        user_gid = gid.GID(string=user_gid)
        self.logger.info("Called CreateUserCredential for GID %s" % user_gid.get_hrn())
        ucred = create_credential(user_gid, user_gid, USER_CRED_LIFE, 'user', self.keyfile, self.certfile)
        return ucred.save_to_string()

    def create_slice_credential(self, user_gid, slice_gid):
        '''Create a Slice credential object for this user_gid (object) on given slice gid (object)'''
        # FIXME: Validate the user_gid and slice_gid
        # are my user and slice
        return create_credential(user_gid, slice_gid, SLICE_CRED_LIFE, 'slice', self.keyfile, self.certfile)

<|endoftext|>


<commit_before>#! /usr/bin/env python

from __future__ import print_function

from sys import version_info, exit
import os
import hashlib
import re
from numbers import Number

import time
import datetime
import argparse
import json
import webbrowser

if version_info[0] == 3:
    from urllib.request import urlopen
    from urllib.parse import quote
    basestring = str
    raw_input = input
else:
    from urllib import urlopen, quote

pyinput = lambda x: eval(raw_input(x), {}, {})

GEO_ROOT = os.path.join(os.getenv("RED_SPIDER_ROOT"), "work", "geohash")
DEFAULTS_FILE = os.path.join(GEO_ROOT, "defaults")
CACHE_FILE = os.path.join(GEO_ROOT, "cache")
URL_DOW = r"http://geo.crox.net/djia/{year:d}/{month:02d}/{day:02d}"
MAPS = "https://maps.google.com/maps?q={:f},{:f}"
MAPS_LOOKUP = "https://maps.google.com/maps?q={}"

def geohash(latitude, longitude, datedow):
    '''Compute geohash() using the Munroe algorithm.

    >>> geohash(37.421542, -122.085589, b'2005-05-26-10458.68')
    37.857713 -122.544543

    '''
    # http://xkcd.com/426/
    # adapted from antigravity.py
    datedow = datedow.encode("utf-8")
    h = hashlib.md5(datedow).hexdigest()
    p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])]
    return [float("{}{}".format(int(x), y[1:])) for x, y in ((latitude, p), (longitude, q))]

def memoize_to_disk(filename, invalid=set()):
    def decorator(func):
        try:
            with open(filename, "r") as fp:
                cache = json.load(fp)
        except (IOError, ValueError):
            cache = {}

        def memoize(*args):
            chk = str(args)
            if chk not in cache:
                ret = func(*args)
                if not ret in invalid:
                    cache[chk] = ret
                    with open(filename, "w") as fp:
                        json.dump(cache, fp)
                return ret
            else:
                return cache[chk]
        return memoize
    return decorator

def parse_date(date):
    formats = ("%x", "%Y-{m}-%d","%d-{m}-%Y", "{m}-%d-%Y")
    # deal with date delimiters
    datefract = re.findall("\d+|\w+", date)
    assert len(datefract) == 3
    check_date = "-".join(datefract)
    for check_format in formats:
        for month in {"%m", "%b", "%B"}:
            try:
                return time.strptime(check_date, check_format.format(m=month))
            except:
                pass
    else:
        raise ValueError("Invalid date format.")

def store_defaults(args, filepath):
    if not os.path.exists(os.path.split(filepath)[0]):
        os.makedirs(os.path.split(filepath)[0])
    with open(filepath, "w") as fp:
        json.dump(args.__dict__, fp)

def set_defaults(args, filepath):
    if os.path.exists(filepath) and os.path.isfile(filepath):
        with open(filepath, "r") as fp:
            defaults = json.load(fp)
        for key, value in defaults.items():
            if key in args.__dict__:
                chk = getattr(args, key)
                if not chk and chk != value:
                    print("Default {} = {}".format( key, value ))
                    setattr(args, key, value)
        print()

def make_datedow(date, dow):
    date = time.strftime("%Y-%m-%d", date)
    if isinstance(dow, basestring):
        dow = float(dow)
    return "{}-{:.2f}".format(date, dow)

def get_date_of_dow(date, coords):
    date = datetime.date(*date[:3])
    if 0 > coords[1] > -30:
        date -= datetime.timedelta(days=1)
    wkday = date.weekday()
    if wkday > 4:
        date -= datetime.timedelta(wkday - 4)
    date = tuple(date.timetuple())
    return date

@memoize_to_disk(CACHE_FILE, invalid={None,})
def get_dow(date):
    try:
        request = URL_DOW.format(year=date[0], month=date[1], day=date[2])
        req = urlopen(request)
        dow = float(req.read().decode("utf-8"))
        req.close()
        return dow
    except:
        return

def get_location_coords(gen_location):
    webbrowser.open(MAPS_LOOKUP.format( quote(gen_location) ))
    print("`exit()` to abort.")
    prompt = "Please enter {}: "
    coords = []
    for req in ("LATITUDE", "LONGITUDE"):
        while True:
            try:
                x = pyinput( prompt.format(req) )
                if isinstance(x, Number):
                    break
            except Exception as exc:
                pass
        coords += [x]
    return coords

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Generate a geohash based on the Munroe Algorithm.")
    parser.add_argument("-ll", dest="location",
                        metavar=("LATITUDE", "LONGITUDE"),nargs=2,
                        default=None, type=float, help="Coordinates to complement the geohashing algorithm")
    parser.add_argument("-l", dest="gen_location", metavar=("LOCATION"),
                        default="", nargs="*", help="Look up a locations coordinates in google.maps")
    parser.add_argument("-t", dest="date", metavar=("DATE"),
                        default=None, help="Date for which the geohash is valid")
    parser.add_argument("-d", dest="dow", metavar=("DJIA"),
                        default=None, help="Dow Jones Industrial Average required for hashing")
    parser.add_argument("-s", "--store-defaults", action="store_true",
                        help="Save the supplied command line arguments as defaults")
    parser.add_argument("-n", "--no-defaults", action="store_true",
                        help="Ignore default command line arguments")
    parser.add_argument("-cc", "--clear-cache", action="store_true",
                        help="Empty the DJIA cache")
    parser.add_argument("-m", "--maps", action="store_true",
                        help="Show the geohash coordinates in google.maps")

    args = parser.parse_args()
    flag_location = True if type(args.gen_location) is list else False
    args.gen_location = " ".join(args.gen_location)

    if not os.path.exists(GEO_ROOT):
        os.makedirs(GEO_ROOT)

    if args.clear_cache:
        try:
            os.remove(CACHE_FILE)
        except OSError:
            pass
        exit()

    if args.no_defaults:
        del args.no_defaults
    else:
        set_defaults(args, DEFAULTS_FILE)

    if args.store_defaults:
        del args.store_defaults
        store_defaults(args, DEFAULTS_FILE)

    if not args.location and flag_location:
        args.location = get_location_coords(args.gen_location)

    if args.location:
        assert len(args.location) == 2
        date = parse_date(args.date) if args.date else time.localtime()
        date_of_dow = get_date_of_dow(date, args.location)
        datedow = None

        if not args.dow:
            print("Fetching DOW from the web..")
            args.dow = get_dow(date_of_dow)

        if args.dow:
            datedow = make_datedow(date_of_dow, args.dow)

            print()
            print("Input: {}".format(datedow))
            unpack = args.location + [datedow]
            geo_location = geohash(*unpack)
            print("Output: {}, {}".format(*geo_location))
            if args.maps:
                webbrowser.open(MAPS.format(*geo_location))
        else:
            print("drats!")
<commit_msg>Improve messages<commit_after>#! /usr/bin/env python

from __future__ import print_function

from sys import version_info, exit
import os
import hashlib
import re
from numbers import Number

import time
import datetime
import argparse
import json
import webbrowser

if version_info[0] == 3:
    from urllib.request import urlopen
    from urllib.parse import quote
    basestring = str
    raw_input = input
else:
    from urllib import urlopen, quote

pyinput = lambda x: eval(raw_input(x), {}, {})

GEO_ROOT = os.path.join(os.getenv("RED_SPIDER_ROOT"), "work", "geohash")
DEFAULTS_FILE = os.path.join(GEO_ROOT, "defaults")
CACHE_FILE = os.path.join(GEO_ROOT, "cache")
URL_DOW = r"http://geo.crox.net/djia/{year:d}/{month:02d}/{day:02d}"
MAPS = "https://maps.google.com/maps?q={:f},{:f}"
MAPS_LOOKUP = "https://maps.google.com/maps?q={}"

def geohash(latitude, longitude, datedow):
    '''Compute geohash() using the Munroe algorithm.

    >>> geohash(37.421542, -122.085589, b'2005-05-26-10458.68')
    37.857713 -122.544543

    '''
    # http://xkcd.com/426/
    # adapted from antigravity.py
    datedow = datedow.encode("utf-8")
    h = hashlib.md5(datedow).hexdigest()
    p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])]
    return [float("{}{}".format(int(x), y[1:])) for x, y in ((latitude, p), (longitude, q))]

def memoize_to_disk(filename, invalid=set()):
    def decorator(func):
        try:
            with open(filename, "r") as fp:
                cache = json.load(fp)
        except (IOError, ValueError):
            cache = {}

        def memoize(*args):
            chk = str(args)
            if chk not in cache:
                ret = func(*args)
                if not ret in invalid:
                    cache[chk] = ret
                    with open(filename, "w") as fp:
                        json.dump(cache, fp)
                return ret
            else:
                return cache[chk]
        return memoize
    return decorator

def parse_date(date):
    formats = ("%x", "%Y-{m}-%d","%d-{m}-%Y", "{m}-%d-%Y")
    # deal with date delimiters
    datefract = re.findall("\d+|\w+", date)
    assert len(datefract) == 3
    check_date = "-".join(datefract)
    for check_format in formats:
        for month in {"%m", "%b", "%B"}:
            try:
                return time.strptime(check_date, check_format.format(m=month))
            except:
                pass
    else:
        raise ValueError("Invalid date format.")

def store_defaults(args, filepath):
    if not os.path.exists(os.path.split(filepath)[0]):
        os.makedirs(os.path.split(filepath)[0])
    with open(filepath, "w") as fp:
        json.dump(args.__dict__, fp)

def set_defaults(args, filepath):
    if os.path.exists(filepath) and os.path.isfile(filepath):
        with open(filepath, "r") as fp:
            defaults = json.load(fp)
        for key, value in defaults.items():
            if key in args.__dict__:
                chk = getattr(args, key)
                if not chk and chk != value:
                    print("Default {} = {}".format( key, value ))
                    setattr(args, key, value)

def make_datedow(date, dow):
    date = time.strftime("%Y-%m-%d", date)
    if isinstance(dow, basestring):
        dow = float(dow)
    return "{}-{:.2f}".format(date, dow)

def get_date_of_dow(date, coords):
    date = datetime.date(*date[:3])
    if 0 > coords[1] > -30:
        date -= datetime.timedelta(days=1)
    wkday = date.weekday()
    if wkday > 4:
        date -= datetime.timedelta(wkday - 4)
    date = tuple(date.timetuple())
    return date

@memoize_to_disk(CACHE_FILE, invalid={None,})
def get_dow(date):
    request = URL_DOW.format(year=date[0], month=date[1], day=date[2])
    req = urlopen(request)
    page = req.read().decode("utf-8")
    req.close()
    if page == u"error\ndata not available yet":
        print("DJIA for this date is not available (yet?)")
        return
    dow = float(page)
    return dow

def get_location_coords(gen_location):
    webbrowser.open(MAPS_LOOKUP.format( quote(gen_location) ))
    print("`exit()` to abort.")
    prompt = "{: <9} >>> "
    coords = []
    for req in ("LATITUDE", "LONGITUDE"):
        while True:
            try:
                x = pyinput( prompt.format(req) )
                if isinstance(x, Number):
                    break
            except Exception as exc:
                pass
        coords += [x]
    return coords

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Generate a geohash based on the Munroe Algorithm.")
    parser.add_argument("-ll", dest="location",
                        metavar=("LATITUDE", "LONGITUDE"),nargs=2,
                        default=None, type=float, help="Coordinates to complement the geohashing algorithm")
    parser.add_argument("-l", dest="gen_location", metavar=("LOCATION"),
                        default="", nargs="*", help="Look up a locations coordinates in google.maps")
    parser.add_argument("-t", dest="date", metavar=("DATE"),
                        default=None, help="Date for which the geohash is valid")
    parser.add_argument("-d", dest="dow", metavar=("DJIA"),
                        default=None, help="Dow Jones Industrial Average required for hashing")
    parser.add_argument("-s", "--store-defaults", action="store_true",
                        help="Save the supplied command line arguments as defaults")
    parser.add_argument("-n", "--no-defaults", action="store_true",
                        help="Ignore default command line arguments")
    parser.add_argument("-cc", "--clear-cache", action="store_true",
                        help="Empty the DJIA cache")
    parser.add_argument("-m", "--maps", action="store_true",
                        help="Show the geohash coordinates in google.maps")

    args = parser.parse_args()
    flag_location = True if type(args.gen_location) is list else False
    args.gen_location = " ".join(args.gen_location)

    if not os.path.exists(GEO_ROOT):
        os.makedirs(GEO_ROOT)

    if args.clear_cache:
        try:
            os.remove(CACHE_FILE)
        except OSError:
            pass
        exit()

    if args.no_defaults:
        del args.no_defaults
    else:
        set_defaults(args, DEFAULTS_FILE)

    if args.store_defaults:
        del args.store_defaults
        store_defaults(args, DEFAULTS_FILE)

    if not args.location and flag_location:
        args.location = get_location_coords(args.gen_location)

    if args.location:
        assert len(args.location) == 2
        date = parse_date(args.date) if args.date else time.localtime()
        date_of_dow = get_date_of_dow(date, args.location)
        datedow = None

        if not args.dow:
            print("\nRetrieving DJIA..")
            args.dow = get_dow(date_of_dow)

        if args.dow:
            datedow = make_datedow(date_of_dow, args.dow)

            print()
            print("Input: {}".format(datedow))
            unpack = args.location + [datedow]
            geo_location = geohash(*unpack)
            print("Output: {}, {}".format(*geo_location))
            if args.maps:
                webbrowser.open(MAPS.format(*geo_location))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2020 shmilee

'''
Gdpy3's logger module.

* [G]dp: gdpy3
* [L]oad: loaders
* [S]ave: savers
* [C]onvert: converters
* [D]ig: diggers
* [E]xport: exporters
* [P]rocess: processors
* [V]isualize, plot: visplters
'''

import os
import tempfile
import getpass
import time
import logging
import logging.config

logfile = os.path.join(
    tempfile.gettempdir(),
    'gdpy3-%s-%s.log' % (getpass.getuser(), time.strftime('%Y')))

logger_common_config = {
    'level': 'DEBUG',
    'handlers': ['console', 'file'],
    'propagate': False,
}

gloggerConfig = {
    'version': 1,
    'formatters': {
        'detailed': {
            'class': 'logging.Formatter',
            'format': '%(asctime)s - %(name)s:%(module)s:%(lineno)d:%(levelname)s - %(message)s',
            'datefmt': '%m-%d %H:%M:%S',
        },
        'simple': {
            'class': 'logging.Formatter',
            'format': '[%(name)s]%(levelname)-7s - %(message)s',
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'level': 'INFO',
            'formatter': 'simple',
            'stream': 'ext://sys.stdout',
        },
        'file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'level': 'DEBUG',
            'formatter': 'detailed',
            'filename': logfile,
            'maxBytes': 3 * 1024 * 1024,
            'backupCount': 9,
        },
    },
    'loggers': {
        'G': logger_common_config,  # gdpy3 # gdpy3.GUI
        'L': logger_common_config,  # gdpy3.loaders
        'S': logger_common_config,  # gdpy3.savers
        'C': logger_common_config,  # gdpy3.cores.converter
        'D': logger_common_config,  # gdpy3.cores.digger
        'E': logger_common_config,  # gdpy3.cores.exporter
        'P': logger_common_config,  # gdpy3.processors
        'V': logger_common_config,  # gdpy3.visplters
    },
}

class GLogger(logging.Logger):
    '''
    Modify the logging.Logger class for gdpy3.
    '''

    def parm(self, msg, *args, **kwargs):
        """
        Log 'msg % args' with severity 'INFO'.
        Add 'Parameter,' before the *msg*.
        """
        msg = '%s, %s' % ('Parameter', msg)
        if self.isEnabledFor(logging.INFO):
            self._log(logging.INFO, msg, args, **kwargs)

def getGLogger(name):
    if name not in gloggerConfig['loggers']:
        raise KeyError("'%s' not found in supported loggers!" % name)
    return logging.getLogger(name)

logging.setLoggerClass(GLogger)
logging.config.dictConfig(gloggerConfig)
<commit_msg>glogger multiprocessing support<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2020 shmilee

'''
Gdpy3's logger module.

* [G]dp: gdpy3
* [L]oad: loaders
* [S]ave: savers
* [C]onvert: converters
* [D]ig: diggers
* [E]xport: exporters
* [P]rocess: processors
* [V]isualize, plot: visplters
'''

import os
import tempfile
import getpass
import time
import logging
import logging.config
import contextlib
import multiprocessing

def get_glogger_config(c, logfile=None, queue=None):
    '''
    Parameters
    ----------
    c: str
        main, for main process
        listen, for listener main process, add processName in format
        work, for worker child process
    logfile: str
        needed when c is 'main' or 'listen'
    queue: Queue
        needed when c is 'work'
    '''
    if c in ['main', 'listen']:
        pn = '{%(processName)s}' if c == 'listen' else ''
        msg = '%s %%(message)s' % pn
        detailed_fmt = '%(asctime)s - %(name)s:%(module)s:%(lineno)d:%(levelname)s - ' + msg
        simple_fmt = '[%(name)s]%(levelname)-7s - ' + msg
        formatters = {
            'detailed': {
                'class': 'logging.Formatter',
                'format': detailed_fmt,
                'datefmt': '%m-%d %H:%M:%S',
            },
            'simple': {
                'class': 'logging.Formatter',
                'format': simple_fmt,
            },
        }
        handlers = {
            'console': {
                'class': 'logging.StreamHandler',
                'level': 'INFO',
                'formatter': 'simple',
                'stream': 'ext://sys.stderr',
            },
            'file': {
                'class': 'logging.handlers.RotatingFileHandler',
                'level': 'DEBUG',
                'formatter': 'detailed',
                'filename': logfile,
                'maxBytes': 3 * 1024 * 1024,
                'backupCount': 9,
            },
        }
    else:
        formatters = {}
        handlers = {
            'queue': {
                'class': 'logging.handlers.QueueHandler',
                'queue': queue,
            },
        }
    _common_logger_config = {
        'level': 'DEBUG',
        'handlers': list(handlers.keys()),
        'propagate': False,
    }
    return {
        'version': 1,
        'disable_existing_loggers': True,
        'formatters': formatters,
        'handlers': handlers,
        'loggers': {
            'G': _common_logger_config,  # gdpy3 # gdpy3.GUI
            'L': _common_logger_config,  # gdpy3.loaders
            'S': _common_logger_config,  # gdpy3.savers
            'C': _common_logger_config,  # gdpy3.cores.converter
            'D': _common_logger_config,  # gdpy3.cores.digger
            'E': _common_logger_config,  # gdpy3.cores.exporter
            'P': _common_logger_config,  # gdpy3.processors
            'V': _common_logger_config,  # gdpy3.visplters
        },
    }

class GLogger(logging.Logger):
    '''
    Modify the logging.Logger class for gdpy3.
    '''

    def parm(self, msg, *args, **kwargs):
        """
        Log 'msg % args' with severity 'INFO'.
        Add 'Parameter,' before the *msg*.
        """
        msg = '%s, %s' % ('Parameter', msg)
        if self.isEnabledFor(logging.INFO):
            self._log(logging.INFO, msg, args, **kwargs)

logfile = os.path.join(
    tempfile.gettempdir(),
    'gdpy3-%s-%s.log' % (getpass.getuser(), time.strftime('%Y')))
glogger_config_main = get_glogger_config('main', logfile=logfile)
glogger_config_listen = get_glogger_config('listen', logfile=logfile)

def getGLogger(name):
    '''
    Support name: G, L etc. and G.x.y, L.m.n etc.
    '''
    if name.split('.')[0] not in glogger_config_main['loggers']:
        raise KeyError("Logger '%s' not supported!" % name)
    return logging.getLogger(name)

logging.setLoggerClass(GLogger)
logging.config.dictConfig(glogger_config_main)

@contextlib.contextmanager
def get_glogger_work_initializer():
    # listener in MainProcess
    logqueue = multiprocessing.Manager().Queue(-1)
    logging.config.dictConfig(glogger_config_listen)
    queue_listener = logging.handlers.QueueListener(
        logqueue,
        *logging.getLogger('G').handlers,
        respect_handler_level=True)
    queue_listener.start()
    try:
        def initializer():
            glogger_config_work = get_glogger_config('work', queue=logqueue)
            logging.config.dictConfig(glogger_config_work)
        yield initializer
    finally:
        queue_listener.stop()
        logging.config.dictConfig(glogger_config_main)

# multiprocessing logging ref:
# https://docs.python.org/3/howto/logging-cookbook.html#a-more-elaborate-multiprocessing-example
# https://gist.github.com/schlamar/7003737
# https://github.com/jruere/multiprocessing-logging
<|endoftext|>


<commit_before>import telegram
import time
import pytesseract
import config
import os

try:
	import Image
except ImportError:
	from PIL import Image

help_text = """
Send me an image and I will run Google's Tesseract OCR tool on it and send you back the results.

You can change the language I use on Tesseract by sending me `/lang <code>`.

The <code> must be one of the following codes:

Code -> Language
eng  -> English
por  -> Portuguese
spa  -> Spanish
fra  -> French
ita  -> Italian
jpn  -> Japanese
"""

# /setcommands
# lang - query or set your current language
# help - detailed information about the bot

DEFAULT_LANG = 'por'

lang_dict = {}

available_langs = {'eng' : 'English',
					'por' : 'Portuguese',
					'spa' : 'Spanish',
					'fra' : 'French',
					'ita' : 'Italian',
					'jpn' : 'Japanese',
					}

def start(bot, update):
	bot.sendMessage(chat_id=update.message.chat_id, text="Hello! I'm Tesseract Bot!\n\n"+help_text, parse_mode=telegram.ParseMode.MARKDOWN)

def help(bot, update):
	bot.sendMessage(chat_id=update.message.chat_id, text=help_text, parse_mode=telegram.ParseMode.MARKDOWN)

def unknown(bot, update):
	bot.sendMessage(chat_id=update.message.chat_id, text="Sorry, I didn't understand that command.")

def message(bot, update):

	print(update.message.chat_id)

	if not update.message.photo:
		return

	try:
		photoFile = bot.getFile(update.message.photo[-1].file_id)

		filename = config.CACHE_DIR+'/photo_'+''.join(str(time.time()).split('.'))+'.jpg'

		photoFile.download(filename)

		language = lang_dict.get(update.message.chat_id, DEFAULT_LANG)

		image_text = pytesseract.image_to_string(Image.open(filename), language)

		if config.CACHE_TEMP:
			os.remove(filename)

		sanitized_string = _sanitize_string(image_text)

		if sanitized_string:
			response_msg = 'Parsed in {}:\n\n```\n{}\n```'.format(
				available_langs[language], sanitized_string)
		else:
			response_msg = 'Nothing found! :(\nParsed in {}'.format(available_langs[language])

		bot.sendMessage(chat_id=update.message.chat_id,
						text=response_msg,
						parse_mode=telegram.ParseMode.MARKDOWN)
	except Exception as e:
		_something_wrong(bot, update, e)

def lang(bot, update):
	try:
		_, language = update.message.text.split(' ')
	except ValueError:

		try:
			current_language = available_langs[lang_dict[update.message.chat_id]]
			bot.sendMessage(chat_id=update.message.chat_id, text='Your current language is {}.'.format(current_language))
		except KeyError:
			bot.sendMessage(chat_id=update.message.chat_id, text='Your current language is Default ({}).'.format(available_langs[DEFAULT_LANG]))

	else:
		if language in available_langs:
			lang_dict[update.message.chat_id] = language
			bot.sendMessage(chat_id=update.message.chat_id, text='Your language is now {}.'.format(available_langs[language]))
		else:
			bot.sendMessage(chat_id=update.message.chat_id, text="This language isn't available.")

def tesseract(bot, update):
	pass

def _something_wrong(bot, update, e):
	bot.sendMessage(chat_id=update.message.chat_id, text='Something went wrong...\nError type: {}\nError message: {}'.format(type(e), e))

def _sanitize_string(string):
	illegal_chars = '_*`[]()\\'

	sanitized = ''

	for char in string:
		sanitized += '\\' + char if char in illegal_chars else char

	return sanitized
<commit_msg>Improved group behavior<commit_after>import telegram
import time
import pytesseract
import config
import os

try:
	import Image
except ImportError:
	from PIL import Image

help_text = """
Send me an image and I will run Google's Tesseract OCR tool on it \
and send you back the results.

You can change the language I use on Tesseract by sending me `/lang <code>`.

The <code> must be one of the following codes:

Code -> Language
eng  -> English
por  -> Portuguese
spa  -> Spanish
fra  -> French
ita  -> Italian
jpn  -> Japanese

In groups, I will not automatically parse the images, \
I will wait for someoneto use the /tesseract command, \
then I'll parse the last image sent.

Made by caiopoliveira@gmail.com
Source on https://github.com/caiopo/tesseract-bot
"""

# /setcommands
# lang - query or set your current language
# help - detailed information about the bot

DEFAULT_LANG = 'por'

lang_dict = {}

group_photos = {}

available_langs = {'eng' : 'English',
					'por' : 'Portuguese',
					'spa' : 'Spanish',
					'fra' : 'French',
					'ita' : 'Italian',
					'jpn' : 'Japanese',
					}

def start(bot, update):
	bot.sendMessage(chat_id=update.message.chat_id,
		text="Hello! I'm Tesseract Bot!\n\n"+help_text,
		parse_mode=telegram.ParseMode.MARKDOWN,
		disable_web_page_preview=True)

def help(bot, update):
	bot.sendMessage(chat_id=update.message.chat_id,
		text=help_text,
		parse_mode=telegram.ParseMode.MARKDOWN,
		disable_web_page_preview=True)

def unknown(bot, update):
	if update.message.chat_id > 0: # user
		bot.sendMessage(chat_id=update.message.chat_id,
			text="Sorry, I didn't understand that command.")

def message(bot, update):
	if not update.message.photo:
		return

	photosize = bot.getFile(update.message.photo[-1].file_id)

	if update.message.chat_id > 0: # user

		_photosize_to_parsed(bot, update, photosize)

	else: # group

		group_photos[update.message.chat_id] = photosize

def lang(bot, update):
	try:
		_, language = update.message.text.split(' ')
	except ValueError:

		try:
			current_language = available_langs[lang_dict[update.message.chat_id]]
			bot.sendMessage(chat_id=update.message.chat_id, text='Your current language is {}.'.format(current_language))
		except KeyError:
			bot.sendMessage(chat_id=update.message.chat_id, text='Your current language is Default ({}).'.format(available_langs[DEFAULT_LANG]))

	else:
		if language in available_langs:
			lang_dict[update.message.chat_id] = language
			bot.sendMessage(chat_id=update.message.chat_id, text='Your language is now {}.'.format(available_langs[language]))
		else:
			bot.sendMessage(chat_id=update.message.chat_id, text="This language isn't available.")

def tesseract(bot, update):
	if update.message.chat_id > 0:
		bot.sendMessage(chat_id=update.message.chat_id, text='/tesseract command is only available to groups.')
	else:
		_photosize_to_parsed(bot, update, group_photos[update.message.chat_id])

def _photosize_to_parsed(bot, update, photosize):
	try:
		filename = config.CACHE_DIR+'/photo_'+''.join(str(time.time()).split('.'))+'.jpg'

		photosize.download(filename)

		language = lang_dict.get(update.message.chat_id, DEFAULT_LANG)

		image_text = pytesseract.image_to_string(Image.open(filename), language)

		if config.CACHE_TEMP:
			os.remove(filename)

		sanitized_string = _sanitize_string(image_text)

		if sanitized_string:
			response_msg = 'Parsed in {}:\n\n```\n{}\n```'.format(
				available_langs[language], sanitized_string)
		else:
			response_msg = 'Nothing found! :(\nParsed in {}'.format(available_langs[language])

		bot.sendMessage(chat_id=update.message.chat_id,
						text=response_msg,
					parse_mode=telegram.ParseMode.MARKDOWN)
	except Exception as e:
		_something_wrong(bot, update, e)

def _something_wrong(bot, update, e):
	bot.sendMessage(chat_id=update.message.chat_id, text='Something went wrong...\nError type: {}\nError message: {}'.format(type(e), e))

def _sanitize_string(string):
	illegal_chars = '_*`[]()\\'

	sanitized = ''

	for char in string:
		sanitized += '\\' + char if char in illegal_chars else char

	return sanitized
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import os
import sys
from os.path import join
import subprocess
import multiprocessing

import conda.config as cc

from conda_build.config import config

from conda_build import source
from conda_build.scripts import prepend_bin_path

def get_perl_ver():
    return str(config.CONDA_PERL)

def get_py_ver():
    return '.'.join(str(config.CONDA_PY))

def get_npy_ver():
    if config.CONDA_NPY:
        # Convert int -> string, e.g.
        #   17 -> '1.7'
        #   110 -> '1.10'
        conda_npy = str(config.CONDA_NPY)
        return conda_npy[0] + '.' + conda_npy[1:]
    return ''

def get_stdlib_dir():
    return join(config.build_prefix, 'Lib' if sys.platform == 'win32' else
                                'lib/python%s' % get_py_ver())

def get_sp_dir():
    return join(get_stdlib_dir(), 'site-packages')

def get_git_build_info(src_dir):
    env = os.environ.copy()
    d = {}
    git_dir = join(src_dir, '.git')
    if os.path.exists(git_dir):
        env['GIT_DIR'] = git_dir
    else:
        return d

    # grab information from describe
    key_name = lambda a: "GIT_DESCRIBE_{}".format(a)
    keys = [key_name("TAG"), key_name("NUMBER"), key_name("HASH")]
    env = {str(key): str(value) for key, value in env.items()}
    process = subprocess.Popen(["git", "describe", "--tags", "--long", "HEAD"],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               env=env)
    output = process.communicate()[0].strip()
    output = output.decode('utf-8')
    parts = output.rsplit('-', 2)
    parts_length = len(parts)
    if parts_length == 3:
        d.update(dict(zip(keys, parts)))
    # get the _full_ hash of the current HEAD
    process = subprocess.Popen(["git", "rev-parse", "HEAD"],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               env=env)
    output = process.communicate()[0].strip()
    output = output.decode('utf-8')
    d['GIT_FULL_HASH'] = output
    # set up the build string
    if key_name('NUMBER') in d and key_name('HASH') in d:
        d['GIT_BUILD_STR'] = '{}_{}'.format(d[key_name('NUMBER')],
                                            d[key_name('HASH')])

    return d

def get_dict(m=None, prefix=None):
    if not prefix:
        prefix = config.build_prefix

    python = config.build_python
    d = {'CONDA_BUILD': '1', 'PYTHONNOUSERSITE': '1'}
    d['CONDA_DEFAULT_ENV'] = config.build_prefix
    d['ARCH'] = str(cc.bits)
    d['PREFIX'] = prefix
    d['PYTHON'] = python
    d['PY3K'] = str(config.PY3K)
    d['STDLIB_DIR'] = get_stdlib_dir()
    d['SP_DIR'] = get_sp_dir()
    d['SYS_PREFIX'] = sys.prefix
    d['SYS_PYTHON'] = sys.executable
    d['PERL_VER'] = get_perl_ver()
    d['PY_VER'] = get_py_ver()
    if get_npy_ver():
        d['NPY_VER'] = get_npy_ver()
    d['SRC_DIR'] = source.get_dir()
    if "LANG" in os.environ:
        d['LANG'] = os.environ['LANG']
    if "HTTPS_PROXY" in os.environ:
        d['HTTPS_PROXY'] = os.environ['HTTPS_PROXY']
    if "HTTP_PROXY" in os.environ:
        d['HTTP_PROXY'] = os.environ['HTTP_PROXY']

    if m:
        for var_name in m.get_value('build/script_env', []):
            value = os.getenv(var_name)
            if value is None:
                value = '<UNDEFINED>'
            d[var_name] = value

    try:
        d['CPU_COUNT'] = str(multiprocessing.cpu_count())
    except NotImplementedError:
        d['CPU_COUNT'] = "1"

    d.update(**get_git_build_info(d['SRC_DIR']))
    d['PATH'] = dict(os.environ)['PATH']
    d = prepend_bin_path(d, prefix)

    if sys.platform == 'win32':         # -------- Windows
        d['SCRIPTS'] = join(prefix, 'Scripts')
        d['LIBRARY_PREFIX'] = join(prefix, 'Library')
        d['LIBRARY_BIN'] = join(d['LIBRARY_PREFIX'], 'bin')
        d['LIBRARY_INC'] = join(d['LIBRARY_PREFIX'], 'include')
        d['LIBRARY_LIB'] = join(d['LIBRARY_PREFIX'], 'lib')
        # This probably should be done more generally
        d['CYGWIN_PREFIX'] = prefix.replace('\\', '/').replace('C:', '/cygdrive/c')

        d['R'] = join(prefix, 'Scripts', 'R.exe')
    else:                               # -------- Unix
        d['HOME'] = os.getenv('HOME', 'UNKNOWN')
        d['PKG_CONFIG_PATH'] = join(prefix, 'lib', 'pkgconfig')
        d['R'] = join(prefix, 'bin', 'R')

    if sys.platform == 'darwin':         # -------- OSX
        d['OSX_ARCH'] = 'i386' if cc.bits == 32 else 'x86_64'
        d['CFLAGS'] = '-arch %(OSX_ARCH)s' % d
        d['CXXFLAGS'] = d['CFLAGS']
        d['LDFLAGS'] = d['CFLAGS']
        d['MACOSX_DEPLOYMENT_TARGET'] = '10.6'

    elif sys.platform.startswith('linux'):      # -------- Linux
        d['LD_RUN_PATH'] = prefix + '/lib'

    if m:
        d['PKG_NAME'] = m.name()
        d['PKG_VERSION'] = m.version()
        d['PKG_BUILDNUM'] = str(m.build_number())
        d['PKG_BUILD_STRING'] = str(m.build_id())
        d['RECIPE_DIR'] = m.path

    return d

if __name__ == '__main__':
    e = get_dict()
    for k in sorted(e):
        assert isinstance(e[k], str), k
        print('%s=%s' % (k, e[k]))
<commit_msg>Fix #645<commit_after>from __future__ import absolute_import, division, print_function

import os
import sys
from os.path import join
import subprocess
import multiprocessing

import conda.config as cc

from conda_build.config import config

from conda_build import source
from conda_build.scripts import prepend_bin_path

def get_perl_ver():
    return str(config.CONDA_PERL)

def get_py_ver():
    return '.'.join(str(config.CONDA_PY))

def get_npy_ver():
    if config.CONDA_NPY:
        # Convert int -> string, e.g.
        #   17 -> '1.7'
        #   110 -> '1.10'
        conda_npy = str(config.CONDA_NPY)
        return conda_npy[0] + '.' + conda_npy[1:]
    return ''

def get_stdlib_dir():
    return join(config.build_prefix, 'Lib' if sys.platform == 'win32' else
                                'lib/python%s' % get_py_ver())

def get_sp_dir():
    return join(get_stdlib_dir(), 'site-packages')

def get_git_build_info(src_dir):
    env = os.environ.copy()
    d = {}
    git_dir = join(src_dir, '.git')
    if os.path.exists(git_dir):
        env['GIT_DIR'] = git_dir
    else:
        return d

    # grab information from describe
    key_name = lambda a: "GIT_DESCRIBE_{}".format(a)
    keys = [key_name("TAG"), key_name("NUMBER"), key_name("HASH")]
    env = {str(key): str(value) for key, value in env.items()}
    process = subprocess.Popen(["git", "describe", "--tags", "--long", "HEAD"],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               env=env)
    output = process.communicate()[0].strip()
    output = output.decode('utf-8')
    parts = output.rsplit('-', 2)
    parts_length = len(parts)
    if parts_length == 3:
        d.update(dict(zip(keys, parts)))
    # get the _full_ hash of the current HEAD
    process = subprocess.Popen(["git", "rev-parse", "HEAD"],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               env=env)
    output = process.communicate()[0].strip()
    output = output.decode('utf-8')
    d['GIT_FULL_HASH'] = output
    # set up the build string
    if key_name('NUMBER') in d and key_name('HASH') in d:
        d['GIT_BUILD_STR'] = '{}_{}'.format(d[key_name('NUMBER')],
                                            d[key_name('HASH')])

    return d

def get_dict(m=None, prefix=None):
    if not prefix:
        prefix = config.build_prefix

    python = config.build_python
    d = {'CONDA_BUILD': '1', 'PYTHONNOUSERSITE': '1'}
    d['CONDA_DEFAULT_ENV'] = config.build_prefix
    d['ARCH'] = str(cc.bits)
    d['PREFIX'] = prefix
    d['PYTHON'] = python
    d['PY3K'] = str(config.PY3K)
    d['STDLIB_DIR'] = get_stdlib_dir()
    d['SP_DIR'] = get_sp_dir()
    d['SYS_PREFIX'] = sys.prefix
    d['SYS_PYTHON'] = sys.executable
    d['PERL_VER'] = get_perl_ver()
    d['PY_VER'] = get_py_ver()
    if get_npy_ver():
        d['NPY_VER'] = get_npy_ver()
    d['SRC_DIR'] = source.get_dir()
    if "LANG" in os.environ:
        d['LANG'] = os.environ['LANG']
    if "HTTPS_PROXY" in os.environ:
        d['HTTPS_PROXY'] = os.environ['HTTPS_PROXY']
    if "HTTP_PROXY" in os.environ:
        d['HTTP_PROXY'] = os.environ['HTTP_PROXY']

    if m:
        for var_name in m.get_value('build/script_env', []):
            value = os.getenv(var_name)
            if value is None:
                value = '<UNDEFINED>'
            d[var_name] = value

    if sys.platform == "darwin":
        # multiprocessing.cpu_count() is not reliable on OSX
        # See issue #645 on github.com/conda/conda-build
        out, err = subprocess.Popen('sysctl -n hw.logicalcpu', shell=True, stdout=subprocess.PIPE).communicate()
        d['CPU_COUNT'] = out.decode('utf-8').strip()
    else:
        try:
            d['CPU_COUNT'] = str(multiprocessing.cpu_count())
        except NotImplementedError:
            d['CPU_COUNT'] = "1"

    d.update(**get_git_build_info(d['SRC_DIR']))
    d['PATH'] = dict(os.environ)['PATH']
    d = prepend_bin_path(d, prefix)

    if sys.platform == 'win32':         # -------- Windows
        d['SCRIPTS'] = join(prefix, 'Scripts')
        d['LIBRARY_PREFIX'] = join(prefix, 'Library')
        d['LIBRARY_BIN'] = join(d['LIBRARY_PREFIX'], 'bin')
        d['LIBRARY_INC'] = join(d['LIBRARY_PREFIX'], 'include')
        d['LIBRARY_LIB'] = join(d['LIBRARY_PREFIX'], 'lib')
        # This probably should be done more generally
        d['CYGWIN_PREFIX'] = prefix.replace('\\', '/').replace('C:', '/cygdrive/c')

        d['R'] = join(prefix, 'Scripts', 'R.exe')
    else:                               # -------- Unix
        d['HOME'] = os.getenv('HOME', 'UNKNOWN')
        d['PKG_CONFIG_PATH'] = join(prefix, 'lib', 'pkgconfig')
        d['R'] = join(prefix, 'bin', 'R')

    if sys.platform == 'darwin':         # -------- OSX
        d['OSX_ARCH'] = 'i386' if cc.bits == 32 else 'x86_64'
        d['CFLAGS'] = '-arch %(OSX_ARCH)s' % d
        d['CXXFLAGS'] = d['CFLAGS']
        d['LDFLAGS'] = d['CFLAGS']
        d['MACOSX_DEPLOYMENT_TARGET'] = '10.6'

    elif sys.platform.startswith('linux'):      # -------- Linux
        d['LD_RUN_PATH'] = prefix + '/lib'

    if m:
        d['PKG_NAME'] = m.name()
        d['PKG_VERSION'] = m.version()
        d['PKG_BUILDNUM'] = str(m.build_number())
        d['PKG_BUILD_STRING'] = str(m.build_id())
        d['RECIPE_DIR'] = m.path

    return d

if __name__ == '__main__':
    e = get_dict()
    for k in sorted(e):
        assert isinstance(e[k], str), k
        print('%s=%s' % (k, e[k]))
<|endoftext|>