<commit_before>import itertools
from typing import Tuple, Union, Sequence

from thinglang.compiler.errors import InvalidReference, SelfInStaticMethod, UnfilledGenericParameters
from thinglang.compiler.references import ElementReference, LocalReference, Reference
from thinglang.foundation import serializer
from thinglang.lexer.values.identifier import Identifier, GenericIdentifier
from thinglang.parser.definitions.thing_definition import ThingDefinition
from thinglang.parser.values.indexed_access import IndexedAccess
from thinglang.parser.values.named_access import NamedAccess
from thinglang.symbols.symbol import Symbol
from thinglang.symbols.symbol_map import SymbolMap
from thinglang.utils import collection_utils

class SymbolMapper(object):

    """
    The symbol mapper is a container for all the symbol maps needed to perform a compilation pass.
    Its primary function is to resolve compile-time references into the appropriate symbol.
    """

    FOUNDATION = {symbol_map.name: symbol_map for symbol_map in serializer.read_foundation_symbols()}
    INDEX_GETTER_NAME = Identifier('get')

    def __init__(self, ast=None, include_foundation=True):
        """
        Creates a new symbol map (generally from a source AST)
        :param ast: the AST being compiled
        :param include_foundation: should foundation classes (base types) be merged into the symbol mapper?
        """
        self.maps = {}
        self.indexer = itertools.count()
        self.internal_indexing = {}
        self.user_indexing = None

        if include_foundation:
            self.maps.update(SymbolMapper.FOUNDATION)

        if ast:
            self.set_ast(ast)

    def set_ast(self, ast):
        self.user_indexing = {}

        for index, thing in enumerate(x for x in ast.children if isinstance(x, ThingDefinition)):
            self.maps[thing.name] = SymbolMap.from_thing(thing, self.maps.get(thing.extends))  # TODO: check for name collisions
            self.user_indexing[thing.name] = index

        return self

    def resolve(self, target: Union[Identifier, NamedAccess], method_locals: dict=(), current_generics: list=()) -> Reference:
        """
        Resolve a reference into a Reference object
        :param target: the reference being resolved (can be either an identifier or access object)
        :param method_locals: the locals of the method being compiled
        :return: new Reference
        """
        assert not target.STATIC

        if isinstance(target, Identifier):
            if target in current_generics:
                return Reference(Identifier.object())
            elif target.untyped in self:  # TODO: verify name collisions
                return Reference(target)
            else:
                local = method_locals[target]
                if not local.allowed:
                    raise SelfInStaticMethod(target)
                if local.type not in current_generics and self[local.type].generics and local.source != 'self':  # Check if any generic type parameters have been left unfilled
                    raise UnfilledGenericParameters(target, self[local.type], None)
                return LocalReference(method_locals[target], target)
        elif isinstance(target, NamedAccess):
            return self.resolve_named(target, method_locals, current_generics)

        raise Exception("Unknown reference type {}".format(target))

    def resolve_named(self, target: Sequence, method_locals=(), current_generics=(), generic_validation=True) -> ElementReference:
        """
        Resolves an identifier pair (a.b) into an element reference
        :param target: the Access object to resolve
        :param method_locals: the current method's locals
        :return: new ElementReference
        """
        assert len(target) == 2

        first, second, local = target[0], target[1], None

        if first.STATIC:
            container = self[first.type]
        elif isinstance(first, IndexedAccess):
            container = self.resolve_indexed(first, method_locals)
        elif first.untyped in current_generics:  # TODO: what about name collisions?
            container = self[Identifier.object()]  # TODO: implement `with type T implement Interface`
        elif first.untyped in self.maps:
            container = self[first]
        elif first in method_locals:
            local = method_locals[first]
            if not local.allowed:
                raise SelfInStaticMethod(target)
            container = self[local.type]
        else:
            raise Exception('Cannot resolve first level access {} (on {}) from {}'.format(first, first.source_ref, method_locals))

        container, element = self.pull(container, second, target)

        remaining_generics = set(container.generics) - set(current_generics)  # TODO: are we sure generic name conflicts don't prevent this validation?
        if generic_validation and not element.is_complete({x: Identifier.invalid() for x in remaining_generics}):  # Check if any generic type parameters have been left unfilled
            raise UnfilledGenericParameters(target, container, element)

        return ElementReference(container, self.index(container), element, local)

    def resolve_indexed(self, access: IndexedAccess, method_locals=()) -> SymbolMap:
        """
        Resolves an IndexedAccess object into a symbol map describing the type of the indexed property.
        Assumes the getter method is called "get"
        """
        target, index = access.target, access.index
        resolved = self.resolve(target, method_locals)
        getter = self[resolved.type][self.INDEX_GETTER_NAME]
        return self[getter.type]

    def pull(self, start: SymbolMap, item: Identifier, target) -> Tuple[SymbolMap, Symbol]:
        """
        Pull a symbol from a symbol map, iteratively traversing its parents until a match is found
        :param start: the original symbol map
        :param item: the item to find
        :return: a pair of SymbolMap, Symbol
        """
        container = start

        while item not in container:
            if container.extends:
                container = self[container.extends]
            else:
                raise InvalidReference(item, start, target)

        return container, container[item]

    def inheritance(self, start):
        """
        Emits an inheritance chain (from descendant to ancestor)
        :param start: the ThingDefinition to start from
        """
        current = self[start]
        yield current

        while current.extends:
            current = self[current.extends]
            yield current

    def entry(self) -> Reference:
        """
        Get the index of the program's entry point
        """
        return self.resolve(NamedAccess([Identifier('Program'), Identifier.constructor()]), {})

    @collection_utils.drain()
    def descendants(self, root_map: SymbolMap, include_root: bool=True, collected: set=None):
        if not collected:
            collected = {root_map}
        else:
            collected.add(root_map)

        if include_root:
            yield root_map

        for symbol_map in self.maps.values():
            if symbol_map not in collected and symbol_map.extends == root_map.name:
                yield from self.descendants(symbol_map, collected=collected)

    @property
    def indexed(self):
        """
        Returns a sorted list of the indexed symbol maps in this mapper
        """
        return sorted(((name, index) for name, index in self.internal_indexing.items() if self.maps[name].convention is Symbol.INTERNAL),
                      key=lambda x: x[1])

    def index(self, symbol_map):
        if symbol_map.convention == Symbol.INTERNAL:
            return self.internal_indexing[symbol_map.name.untyped]
        else:
            return self.user_indexing[symbol_map.name.untyped]

    def __getitem__(self, item) -> SymbolMap:
        """
        If given a GenericIdentifier, generates a new symbol map with the relevant generics filled in.
        Otherwise, returns the symbol map identified by `item`
        """
        if isinstance(item, GenericIdentifier):
            symbol_map = self[item.value]
            parameters = {parameter: replacement for parameter, replacement in zip(symbol_map.generics, item.generics)}
            assert len(symbol_map.generics) == len(parameters)
            return symbol_map.parameterize(parameters)

        symbol_map = self.maps[item]

        if item not in self.user_indexing and item not in self.internal_indexing:
            self.internal_indexing[item] = next(self.indexer)

        return symbol_map

    def __contains__(self, item) -> bool:
        """
        Checks if this mapper contains a specific map
        """
        return item in self.maps
<commit_msg>Add explicit error for invalid indexing access<commit_after>import itertools
from typing import Tuple, Union, Sequence

from thinglang.compiler.errors import InvalidReference, SelfInStaticMethod, UnfilledGenericParameters
from thinglang.compiler.references import ElementReference, LocalReference, Reference
from thinglang.foundation import serializer
from thinglang.lexer.values.identifier import Identifier, GenericIdentifier
from thinglang.parser.definitions.thing_definition import ThingDefinition
from thinglang.parser.values.indexed_access import IndexedAccess
from thinglang.parser.values.named_access import NamedAccess
from thinglang.symbols.symbol import Symbol
from thinglang.symbols.symbol_map import SymbolMap
from thinglang.utils import collection_utils

class SymbolMapper(object):

    """
    The symbol mapper is a container for all the symbol maps needed to perform a compilation pass.
    Its primary function is to resolve compile-time references into the appropriate symbol.
    """

    FOUNDATION = {symbol_map.name: symbol_map for symbol_map in serializer.read_foundation_symbols()}
    INDEX_GETTER_NAME = Identifier('get')

    def __init__(self, ast=None, include_foundation=True):
        """
        Creates a new symbol map (generally from a source AST)
        :param ast: the AST being compiled
        :param include_foundation: should foundation classes (base types) be merged into the symbol mapper?
        """
        self.maps = {}
        self.indexer = itertools.count()
        self.internal_indexing = {}
        self.user_indexing = None

        if include_foundation:
            self.maps.update(SymbolMapper.FOUNDATION)

        if ast:
            self.set_ast(ast)

    def set_ast(self, ast):
        self.user_indexing = {}

        for index, thing in enumerate(x for x in ast.children if isinstance(x, ThingDefinition)):
            self.maps[thing.name] = SymbolMap.from_thing(thing, self.maps.get(thing.extends))  # TODO: check for name collisions
            self.user_indexing[thing.name] = index

        return self

    def resolve(self, target: Union[Identifier, NamedAccess], method_locals: dict=(), current_generics: list=()) -> Reference:
        """
        Resolve a reference into a Reference object
        :param target: the reference being resolved (can be either an identifier or access object)
        :param method_locals: the locals of the method being compiled
        :return: new Reference
        """
        assert not target.STATIC

        if isinstance(target, Identifier):
            if target in current_generics:
                return Reference(Identifier.object())
            elif target.untyped in self:  # TODO: verify name collisions
                return Reference(target)
            else:
                local = method_locals[target]
                if not local.allowed:
                    raise SelfInStaticMethod(target)
                if local.type not in current_generics and self[local.type].generics and local.source != 'self':  # Check if any generic type parameters have been left unfilled
                    raise UnfilledGenericParameters(target, self[local.type], None)
                return LocalReference(method_locals[target], target)
        elif isinstance(target, NamedAccess):
            return self.resolve_named(target, method_locals, current_generics)

        raise Exception("Unknown reference type {}".format(target))

    def resolve_named(self, target: Sequence, method_locals=(), current_generics=(), generic_validation=True) -> ElementReference:
        """
        Resolves an identifier pair (a.b) into an element reference
        :param target: the Access object to resolve
        :param method_locals: the current method's locals
        :return: new ElementReference
        """
        assert len(target) == 2

        first, second, local = target[0], target[1], None

        if first.STATIC:
            container = self[first.type]
        elif isinstance(first, IndexedAccess):
            container = self.resolve_indexed(first, method_locals)
        elif first.untyped in current_generics:  # TODO: what about name collisions?
            container = self[Identifier.object()]  # TODO: implement `with type T implement Interface`
        elif first.untyped in self.maps:
            container = self[first]
        elif first in method_locals:
            local = method_locals[first]
            if not local.allowed:
                raise SelfInStaticMethod(target)
            container = self[local.type]
        else:
            raise Exception('Cannot resolve first level access {} (on {}) from {}'.format(first, first.source_ref, method_locals))

        container, element = self.pull(container, second, target)

        remaining_generics = set(container.generics) - set(current_generics)  # TODO: are we sure generic name conflicts don't prevent this validation?
        if generic_validation and not element.is_complete({x: Identifier.invalid() for x in remaining_generics}):  # Check if any generic type parameters have been left unfilled
            raise UnfilledGenericParameters(target, container, element)

        return ElementReference(container, self.index(container), element, local)

    def resolve_indexed(self, access: IndexedAccess, method_locals=()) -> SymbolMap:
        """
        Resolves an IndexedAccess object into a symbol map describing the type of the indexed property.
        Assumes the getter method is called "get"
        """
        target, index = access.target, access.index
        resolved = self.resolve(target, method_locals)
        if self.INDEX_GETTER_NAME not in self[resolved.type]:
            raise Exception(f'{resolved.type} does not support indexing (at {access.source_ref})')

        getter = self[resolved.type][self.INDEX_GETTER_NAME]
        return self[getter.type]

    def pull(self, start: SymbolMap, item: Identifier, target) -> Tuple[SymbolMap, Symbol]:
        """
        Pull a symbol from a symbol map, iteratively traversing its parents until a match is found
        :param start: the original symbol map
        :param item: the item to find
        :return: a pair of SymbolMap, Symbol
        """
        container = start

        while item not in container:
            if container.extends:
                container = self[container.extends]
            else:
                raise InvalidReference(item, start, target)

        return container, container[item]

    def inheritance(self, start):
        """
        Emits an inheritance chain (from descendant to ancestor)
        :param start: the ThingDefinition to start from
        """
        current = self[start]
        yield current

        while current.extends:
            current = self[current.extends]
            yield current

    def entry(self) -> Reference:
        """
        Get the index of the program's entry point
        """
        return self.resolve(NamedAccess([Identifier('Program'), Identifier.constructor()]), {})

    @collection_utils.drain()
    def descendants(self, root_map: SymbolMap, include_root: bool=True, collected: set=None):
        if not collected:
            collected = {root_map}
        else:
            collected.add(root_map)

        if include_root:
            yield root_map

        for symbol_map in self.maps.values():
            if symbol_map not in collected and symbol_map.extends == root_map.name:
                yield from self.descendants(symbol_map, collected=collected)

    @property
    def indexed(self):
        """
        Returns a sorted list of the indexed symbol maps in this mapper
        """
        return sorted(((name, index) for name, index in self.internal_indexing.items() if self.maps[name].convention is Symbol.INTERNAL),
                      key=lambda x: x[1])

    def index(self, symbol_map):
        if symbol_map.convention == Symbol.INTERNAL:
            return self.internal_indexing[symbol_map.name.untyped]
        else:
            return self.user_indexing[symbol_map.name.untyped]

    def __getitem__(self, item) -> SymbolMap:
        """
        If given a GenericIdentifier, generates a new symbol map with the relevant generics filled in.
        Otherwise, returns the symbol map identified by `item`
        """
        if isinstance(item, GenericIdentifier):
            symbol_map = self[item.value]
            parameters = {parameter: replacement for parameter, replacement in zip(symbol_map.generics, item.generics)}
            assert len(symbol_map.generics) == len(parameters)
            return symbol_map.parameterize(parameters)

        symbol_map = self.maps[item]

        if item not in self.user_indexing and item not in self.internal_indexing:
            self.internal_indexing[item] = next(self.indexer)

        return symbol_map

    def __contains__(self, item) -> bool:
        """
        Checks if this mapper contains a specific map
        """
        return item in self.maps
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    l10n_cr_base.py
#    l10n_cr_base
#    First author: Carlos Vásquez <carlos.vasquez@clearcorp.co.cr> (ClearCorp S.A.)
#    Copyright (c) 2010-TODAY ClearCorp S.A. (http://clearcorp.co.cr). All rights reserved.
#
#    Redistribution and use in source and binary forms, with or without modification, are
#    permitted provided that the following conditions are met:
#
#       1. Redistributions of source code must retain the above copyright notice, this list of
#          conditions and the following disclaimer.
#
#       2. Redistributions in binary form must reproduce the above copyright notice, this list
#          of conditions and the following disclaimer in the documentation and/or other materials
#          provided with the distribution.
#
#    THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND ANY EXPRESS OR IMPLIED
#    WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
#    FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> OR
#    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
#    CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
#    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
#    ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
#    ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#    The views and conclusions contained in the software and documentation are those of the
#    authors and should not be interpreted as representing official policies, either expressed
#    or implied, of ClearCorp S.A..
#
##############################################################################

from osv import osv,fields
import pooler
import time
from base.res.partner.partner import _lang_get

class res_partner_function(osv.osv):
    '''
    Inherits res.partner.function to add translation to code and name fields
    '''
    _inherit = 'res.partner.function'

    _columns = {
        'name': fields.char('Function Name', size=64, required=True, translate=True),
        'code': fields.char('Code', size=8, required=True, translate=True),
    }
res_partner_function()

class res_partner_title(osv.osv):
    '''
    Inherits res.partner.title to add translation to shortcut field
    '''
    _inherit = 'res.partner.title'

    _columns = {
        'shortcut': fields.char('Shortcut', required=True, size=16, translate=True),
    }
res_partner_title()

class res_partner(osv.osv):
    '''
    Inherits res.partner to add id_number field, and lang default value
    '''
    _inherit = 'res.partner'
    _columns = {
        'id_number': fields.char('ID Number', size=30,required=False, select=1),
        'lang': fields.selection(_lang_get, 'Language', size=5, required=True, help="If the selected language is loaded in the system, all documents related to this partner will be printed in this language. If not, it will be english."),
    }
    _defaults = {
        'lang': lambda *a: 'es_ES',
        'title': lambda *a: 'S.A.',
        'date': lambda *args: time.strftime('%Y-%m-%d'),
    }

    def lang_es_install(self, cr, uid):
        lang = 'es_ES'
        modobj = pooler.get_pool(cr.dbname).get('ir.module.module')
        mids = modobj.search(cr, uid, [('state', '=', 'installed')])
        modobj.update_translations(cr, uid, mids, lang)
        return {}

res_partner()

class res_partner_address(osv.osv):
    '''
    Inherits res.partner.address to add country and state default values
    '''
    _inherit = 'res.partner.address'
    _defaults = {
        'country_id': lambda self,cr,uid,ctx={}: self.pool.get('res.country').search(cr, uid, [('name','=','Costa Rica')])[0],
        'state_id': lambda self,cr,uid,ctx={}: self.pool.get('res.country.state').search(cr, uid, [('country_id','=','Costa Rica'),('name','=','San José')])[0],
    }
res_partner_address()

class res_users(osv.osv):
    '''
    Inherits res.users to add lang and tz default values
    '''
    _inherit = 'res.users'
    _defaults = {
        'context_lang': lambda *a: 'es_ES',
        'context_tz': lambda *a: 'America/Costa_Rica',
    }
res_users()
<commit_msg>[FIX] default value for res_partner title<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    l10n_cr_base.py
#    l10n_cr_base
#    First author: Carlos Vásquez <carlos.vasquez@clearcorp.co.cr> (ClearCorp S.A.)
#    Copyright (c) 2010-TODAY ClearCorp S.A. (http://clearcorp.co.cr). All rights reserved.
#
#    Redistribution and use in source and binary forms, with or without modification, are
#    permitted provided that the following conditions are met:
#
#       1. Redistributions of source code must retain the above copyright notice, this list of
#          conditions and the following disclaimer.
#
#       2. Redistributions in binary form must reproduce the above copyright notice, this list
#          of conditions and the following disclaimer in the documentation and/or other materials
#          provided with the distribution.
#
#    THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND ANY EXPRESS OR IMPLIED
#    WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
#    FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> OR
#    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
#    CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
#    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
#    ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
#    ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#    The views and conclusions contained in the software and documentation are those of the
#    authors and should not be interpreted as representing official policies, either expressed
#    or implied, of ClearCorp S.A..
#
##############################################################################

from osv import osv,fields
import pooler
import time
from base.res.partner.partner import _lang_get

class res_partner_function(osv.osv):
    '''
    Inherits res.partner.function to add translation to code and name fields
    '''
    _inherit = 'res.partner.function'

    _columns = {
        'name': fields.char('Function Name', size=64, required=True, translate=True),
        'code': fields.char('Code', size=8, required=True, translate=True),
    }
res_partner_function()

class res_partner_title(osv.osv):
    '''
    Inherits res.partner.title to add translation to shortcut field
    '''
    _inherit = 'res.partner.title'

    _columns = {
        'shortcut': fields.char('Shortcut', required=True, size=16, translate=True),
    }
res_partner_title()

class res_partner(osv.osv):
    '''
    Inherits res.partner to add id_number field, and lang default value
    '''
    _inherit = 'res.partner'
    _columns = {
        'id_number': fields.char('ID Number', size=30,required=False, select=1),
        'lang': fields.selection(_lang_get, 'Language', size=5, required=True, help="If the selected language is loaded in the system, all documents related to this partner will be printed in this language. If not, it will be english."),
    }
    _defaults = {
        'lang': lambda *a: 'es_ES',
        'title': lambda *a: 'Corporation',
        'date': lambda *args: time.strftime('%Y-%m-%d'),
    }

    def lang_es_install(self, cr, uid):
        lang = 'es_ES'
        modobj = pooler.get_pool(cr.dbname).get('ir.module.module')
        mids = modobj.search(cr, uid, [('state', '=', 'installed')])
        modobj.update_translations(cr, uid, mids, lang)
        return {}

res_partner()

class res_partner_address(osv.osv):
    '''
    Inherits res.partner.address to add country and state default values
    '''
    _inherit = 'res.partner.address'
    _defaults = {
        'country_id': lambda self,cr,uid,ctx={}: self.pool.get('res.country').search(cr, uid, [('name','=','Costa Rica')])[0],
        'state_id': lambda self,cr,uid,ctx={}: self.pool.get('res.country.state').search(cr, uid, [('country_id','=','Costa Rica'),('name','=','San José')])[0],
    }
res_partner_address()

class res_users(osv.osv):
    '''
    Inherits res.users to add lang and tz default values
    '''
    _inherit = 'res.users'
    _defaults = {
        'context_lang': lambda *a: 'es_ES',
        'context_tz': lambda *a: 'America/Costa_Rica',
    }
res_users()
<|endoftext|>


<commit_before>import tempfile
import subprocess

from pandaharvester.harvestercore import CoreUtils
from pandaharvester.harvestercore.PluginBase import PluginBase

# logger
baseLogger = CoreUtils.setupLogger()

# submitter for SLURM batch system
class SlurmSubmitter (PluginBase):

    # constructor
    def __init__(self,**kwarg):
        PluginBase.__init__(self,**kwarg)
        # template for batch script
        self.template = """
#!/bin/bash -l
#SBATCH -p {partition}
#SBATCH -N {nodes}
#SBATCH -t {time}
#SBATCH -L {license}
srun {application} {accessPoint}
"""
        # remove the first newline
        self.template = self.template[1:]

    # submit workers
    def submitWorkers(self,workSpecs):
        retList = []
        retStrList = []
        for workSpec in workSpecs:
            # make logger
            tmpLog = CoreUtils.makeLogger(baseLogger,'workerID={0}'.format(workSpec.workerID))
            # make batch script
            batchFile = self.makeBatchScript(workSpec)
            # command
            comStr = "sbatch {0}".format(batchFile)
            # submit
            tmpLog.debug('submit with {0}'.format(batchFile))
            p = subprocess.Popen(comStr.split(),
                                 shell=False,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # check return code
            stdOut,stdErr = p.communicate()
            retCode = p.returncode
            tmpLog.debug('retCode={0}'.format(retCode))
            if retCode == 0:
                # extract batchID
                workSpec.batchID = stdOut.split()[-1]
                tmpLog.debug('batchID={0}'.format(workSpec.batchID))
                tmpRetVal = (True,'')
            else:
                # failed
                errStr = stdOut+' '+stdErr
                tmpLog.error(errStr)
                tmpRetVal = (False,errStr)
            retList.append(tmpRetVal)
        return retList

    # make batch script
    def makeBatchScript(self,workSpec):
        tmpFile = tempfile.NamedTemporaryFile(delete=False)
        tmpFile.write(self.template.format(partition=self.partition,
                                           nodes=self.nodes,
                                           time=self.time,
                                           license=self.license,
                                           application=self.application,
                                           accessPoint=workSpec.accessPoint)
                      )
        tmpFile.close()
        return tmpFile.name

<commit_msg>added workdir<commit_after>import tempfile
import subprocess

from pandaharvester.harvestercore import CoreUtils
from pandaharvester.harvestercore.PluginBase import PluginBase

# logger
baseLogger = CoreUtils.setupLogger()

# submitter for SLURM batch system
class SlurmSubmitter (PluginBase):

    # constructor
    def __init__(self,**kwarg):
        PluginBase.__init__(self,**kwarg)
        # template for batch script
        self.template = """
#!/bin/bash -l
#SBATCH -p {partition}
#SBATCH -N {nodes}
#SBATCH -t {time}
#SBATCH -L {license}
#SBATCH -D {workDir}
srun {application} {accessPoint}
"""
        # remove the first newline
        self.template = self.template[1:]

    # submit workers
    def submitWorkers(self,workSpecs):
        retList = []
        retStrList = []
        for workSpec in workSpecs:
            # make logger
            tmpLog = CoreUtils.makeLogger(baseLogger,'workerID={0}'.format(workSpec.workerID))
            # make batch script
            batchFile = self.makeBatchScript(workSpec)
            # command
            comStr = "sbatch {0}".format(batchFile)
            # submit
            tmpLog.debug('submit with {0}'.format(batchFile))
            p = subprocess.Popen(comStr.split(),
                                 shell=False,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            # check return code
            stdOut,stdErr = p.communicate()
            retCode = p.returncode
            tmpLog.debug('retCode={0}'.format(retCode))
            if retCode == 0:
                # extract batchID
                workSpec.batchID = stdOut.split()[-1]
                tmpLog.debug('batchID={0}'.format(workSpec.batchID))
                tmpRetVal = (True,'')
            else:
                # failed
                errStr = stdOut+' '+stdErr
                tmpLog.error(errStr)
                tmpRetVal = (False,errStr)
            retList.append(tmpRetVal)
        return retList

    # make batch script
    def makeBatchScript(self,workSpec):
        tmpFile = tempfile.NamedTemporaryFile(delete=False)
        tmpFile.write(self.template.format(partition=self.partition,
                                           nodes=self.nodes,
                                           time=self.time,
                                           license=self.license,
                                           application=self.application,
                                           workDir=self.workDir,
                                           accessPoint=workSpec.accessPoint)
                      )
        tmpFile.close()
        return tmpFile.name

<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014-2015 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Steve Ferry
#
#    The licence is in the file __openerp__.py
#
##############################################################################
from openerp.osv import orm

class ResPartnerBank(orm.Model):
    """ This class upgrade the partners.bank to match Compassion needs.
    """

    _inherit = 'res.partner.bank'

    def create(self, cr, uid, data, context):
        """Override function to notify creation in a message
        """
        obj_partner = self.pool['res.partner']
        part = obj_partner.browse(cr, uid, data['partner_id'], context=context)
        part.pool.get('mail.thread').message_post(cr, uid, part.id,
                                                  "<b>Account number: </b>" +
                                                  data['acc_number'],
                                                  "New account created",
                                                  'comment',
                                                  context={'thread_model':
                                                           part._name})
        result = super(ResPartnerBank, self).create(cr, uid, data,
                                                    context=context)
        self.post_write(cr, uid, [result], context=context)
        return result

    def unlink(self, cr, uid, ids, context):
        """Override function to notify delte in a message
        """

        obj_bank = self.pool['res.partner.bank']

        for account in obj_bank.browse(cr, uid, ids, context=context):
            account.partner_id.pool.get('mail.thread'). \
                message_post(cr, uid, account.partner_id.id,
                             "<b>Account number: </b>" +
                             account.acc_number,
                             "Account deleted",
                             'comment',
                             context={'thread_model':
                                      account.partner_id._name})
        result = super(ResPartnerBank, self).unlink(cr, uid, ids,
                                                    context=context)
        return result
<commit_msg>Correcting reviews<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014-2015 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Steve Ferry
#
#    The licence is in the file __openerp__.py
#
##############################################################################
from openerp.osv import orm

class ResPartnerBank(orm.Model):
    """ This class upgrade the partners.bank to match Compassion needs.
    """

    _inherit = 'res.partner.bank'

    def create(self, cr, uid, data, context):
        """Override function to notify creation in a message
        """
        obj_partner = self.pool['res.partner']
        part = obj_partner.browse(cr, uid, data['partner_id'], context=context)
        part.pool.get('mail.thread').message_post(cr, uid, part.id,
                                                  "<b>Account number: </b>" +
                                                  data['acc_number'],
                                                  "New account created",
                                                  'comment',
                                                  context={'thread_model':
                                                           part._name})
        result = super(ResPartnerBank, self).create(cr, uid, data,
                                                    context=context)
        self.post_write(cr, uid, [result], context=context)
        return result

    def unlink(self, cr, uid, ids, context):
        """Override function to notify delte in a message
        """

        obj_bank = self.pool['res.partner.bank']

        for account in obj_bank.browse(cr, uid, ids, context=context):
            account.partner_id.pool.get('mail.thread'). \
                message_post(
                    cr, uid, account.partner_id.id, "<b>Account number: </b>" +
                    account.acc_number, "Account deleted", 'comment',
                    context={'thread_model': account.partner_id._name})

        result = super(ResPartnerBank, self).unlink(cr, uid, ids,
                                                    context=context)
        return result
<|endoftext|>


<commit_before>"""Python script to do FIO test for GCSFuse Vs GCSFuse/Goofys.

To run the script:
>> python3 compare_fuse_types_using_fio.py -- fuse_type_1 fuse_type_1_version fuser_type_2 fuse_type_2_version jobfile_path

-> Supported fuse_types are gcsfuse and goofys.
-> Incase of gcsfuse you can specify version as any of its released version or master to run from the source.
-> Incase of goofys specify the version as latest, as we will be comparing gcsfuse with the latest version of goofys.

The code takes input the fuse type and its version and, jobfile_path as input for fio test.
Then it will perform the fio test for each of the given fuse type at a specific version
and extract fio_metrics to output.txt file.
"""
import argparse
import os
from fio import fio_metrics

from absl import app

GOOFYS_REPO = 'https://github.com/kahing/goofys'
GCSFUSE_REPO = 'https://github.com/GoogleCloudPlatform/gcsfuse'
GCSFUSE_FLAGS="--implicit-dirs --max-conns-per-host 100 --disable-http2"

def _install_gcsfuse(version, gcs_bucket) -> None:
  """Install gcsfuse with Specific version.

  Args:
    version(str): Gcsfuse version to be installed.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  os.system(f'''curl -L -O https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v{version}/gcsfuse_{version}_amd64.deb
            sudo dpkg --install gcsfuse_{version}_amd64.deb
            mkdir gcs
            gcsfuse {GCSFUSE_FLAGS} {gcs_bucket} gcs
            ''')

def _install_gcsfuse_source(version, gcs_bucket) -> None:
  """Run gcsfuse from source code.

  Args:
    version(str): Gcsfuse version to be installed.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  os.system(f'''git clone {GCSFUSE_REPO}
            mkdir gcs
            cd gcsfuse
            go run . {GCSFUSE_FLAGS} {gcs_bucket} ../gcs
            cd ..
            ''')

def _remove_gcsfuse(version) -> None:
  """Remove gcsfuse with specific version.

  Args:
    version(str): Gcsfuse version to be removed.
  """
  os.system(f'''fusermount -u gcs
            rm -rf gcs
            rm -rf gcsfuse_{version}_amd64.deb
            sudo apt-get remove gcsfuse -y
            rm -rf gcsfuse
            ''')

def _install_goofys(gcs_bucket) -> None:
  """Install latest version of goofys.

  Args:
    gcs_bucket(str): GCS bucket to be mounted.
  """
  os.system(f'''git clone {GOOFYS_REPO}
            export GOPATH=$HOME/work
            mkdir gcs
            cd goofys
            go run . gs://{gcs_bucket} ../gcs
            cd ..
            ''')

def _remove_goofys() -> None:
  """Remove goofys.
  """
  os.system('''fusermount -u gcs
            rm -rf gcs
            rm -rf goofys
            ''')

def _run_fio_test(jobfile_path, fio_metrics_obj) -> None:
  """Run fio test and extract metrics to output.txt file.

  Args:
    jobfile(str): Path of the job file.
    fio_metrics_obj(str): Object for extracting fio metrics.
  """
  os.system(f'''fio {jobfile_path} --lat_percentiles 1 --output-format=json --output='output.json'
            ''')
  fio_metricss = fio_metrics_obj.get_metrics('output.json', False)
  os.system(f'''echo {fio_metricss} >> out/output.txt
            rm output.json
            ''')

def _gcsfuse_test(version, jobfile_path, fio_metrics_obj, gcs_bucket) -> None:
  """FIO test for gcsfuse of given version.

  Args:
    version(str): Gcsfuse version to perform fio test.
    jobfile(str): Path of the job file.
    fio_metrics_obj(str): Object for extracting fio metrics.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  if version == 'master':
    _install_gcsfuse_source(version, gcs_bucket)
  else:
    _install_gcsfuse(version, gcs_bucket)

  _run_fio_test(jobfile_path, fio_metrics_obj)
  _remove_gcsfuse(version)

def _goofys_test(jobfile_path, fio_metrics_obj, gcs_bucket) -> None:
  """FIO test for latest version of goofys.

  Args:
    jobfile(str): Path of the job file.
    fio_metrics_obj(str): Object for extracting fio metrics.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  _install_goofys(gcs_bucket)
  _run_fio_test(jobfile_path, fio_metrics_obj)
  _remove_goofys()

def _fuse_test(fuse_type, fuse_type_version, jobfile_path, fio_metric_obj, gcs_bucket) -> None:
  """FIO test for specific version of given fuse type.

  Args:
    fuse_type(str): Fuse type for fio test.
    fuse_type_version(str): Fuse type version for fio test.
    jobfile(str): Path of the job file.
    fio_metrics_obj(str): Object for extracting fio metrics.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  if fuse_type == 'gcsfuse':
    _gcsfuse_test(fuse_type_version, jobfile_path, fio_metric_obj, gcs_bucket)
  elif fuse_type == 'goofys':
    _goofys_test(jobfile_path, fio_metric_obj, gcs_bucket)
  else:
    app.UsageError('Unsupported fuse type!')

def main(argv) -> None:

  parser = argparse.ArgumentParser()
  parser.add_argument(
      'fuse_type_1',
      help='Provide reading fuse_type_1 for fio test')
  parser.add_argument(
      'fuse_type_1_version',
      help='Provid the Specific version of fuse_type_1 in approprite formate')
  parser.add_argument(
      'fuse_type_2',
      help='Provide reading fuse_type_2 for fio test')
  parser.add_argument(
      'fuse_type_2_version',
      help='Provid the Specific version of fuse_type_2 in approprite formate')
  parser.add_argument(
      'jobfile_path',
      help='Provid path of the jobfile')
  parser.add_argument(
      'gcs_bucket',
      help="Provide the gcs bucket name to be mounted")
  args = parser.parse_args(argv[1:])

  fio_metrics_obj = fio_metrics.FioMetrics()
  os.system('mkdir out')

  os.system(f'echo Fuse Type 1: {args.fuse_type_1} {args.fuse_type_1_version} >> out/output.txt')
  _fuse_test(args.fuse_type_1, args.fuse_type_1_version, args.jobfile_path, fio_metrics_obj, args.gcs_bucket)

  os.system(f'echo Fuse Type 2: {args.fuse_type_2} {args.fuse_type_2_version} >> out/output.txt')
  _fuse_test(args.fuse_type_2, args.fuse_type_2_version, args.jobfile_path, fio_metrics_obj, args.gcs_bucket)

if __name__ == '__main__':
  app.run(main)

<commit_msg>Update compare_fuse_types_using_fio.py<commit_after>"""Python script to do FIO test for GCSFuse Vs GCSFuse/Goofys.

To run the script:
>> python3 compare_fuse_types_using_fio.py -- <fuse_type_1> <fuse_type_1_version> <fuser_type_2> <fuse_type_2_version> <jobfile_path> <gcs_bucket>

-> Supported fuse_types are gcsfuse and goofys.
-> Incase of gcsfuse you can specify version as any of its released version or master to run from the source.
-> Incase of goofys specify the version as latest, as we will be comparing gcsfuse with the latest version of goofys.

The code takes input the fuse type and its version and, jobfile_path as input for fio test.
Then it will perform the fio test for each of the given fuse type at a specific version
and extract fio_metrics to output.txt file.
"""
import argparse
import os
from fio import fio_metrics

from absl import app

GOOFYS_REPO = 'https://github.com/kahing/goofys'
GCSFUSE_REPO = 'https://github.com/GoogleCloudPlatform/gcsfuse'
GCSFUSE_FLAGS="--implicit-dirs --max-conns-per-host 100 --disable-http2"

def _install_gcsfuse(version, gcs_bucket) -> None:
  """Install gcsfuse with Specific version.

  Args:
    version(str): Gcsfuse version to be installed.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  os.system(f'''curl -L -O https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v{version}/gcsfuse_{version}_amd64.deb
            sudo dpkg --install gcsfuse_{version}_amd64.deb
            mkdir gcs
            gcsfuse {GCSFUSE_FLAGS} {gcs_bucket} gcs
            ''')

def _install_gcsfuse_source(version, gcs_bucket) -> None:
  """Run gcsfuse from source code.

  Args:
    version(str): Gcsfuse version to be installed.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  os.system(f'''git clone {GCSFUSE_REPO}
            mkdir gcs
            cd gcsfuse
            go run . {GCSFUSE_FLAGS} {gcs_bucket} ../gcs
            cd ..
            ''')

def _remove_gcsfuse(version) -> None:
  """Remove gcsfuse with specific version.

  Args:
    version(str): Gcsfuse version to be removed.
  """
  os.system(f'''fusermount -u gcs
            rm -rf gcs
            rm -rf gcsfuse_{version}_amd64.deb
            sudo apt-get remove gcsfuse -y
            rm -rf gcsfuse
            ''')

def _install_goofys(gcs_bucket) -> None:
  """Install latest version of goofys.

  Args:
    gcs_bucket(str): GCS bucket to be mounted.
  """
  os.system(f'''git clone {GOOFYS_REPO}
            export GOPATH=$HOME/work
            mkdir gcs
            cd goofys
            go run . gs://{gcs_bucket} ../gcs
            cd ..
            ''')

def _remove_goofys() -> None:
  """Remove goofys.
  """
  os.system('''fusermount -u gcs
            rm -rf gcs
            rm -rf goofys
            ''')

def _run_fio_test(jobfile_path, fio_metrics_obj) -> None:
  """Run fio test and extract metrics to output.txt file.

  Args:
    jobfile(str): Path of the job file.
    fio_metrics_obj(str): Object for extracting fio metrics.
  """
  os.system(f'''fio {jobfile_path} --lat_percentiles 1 --output-format=json --output='output.json'
            ''')
  fio_metricss = fio_metrics_obj.get_metrics('output.json', False)
  os.system(f'''echo {fio_metricss} >> out/output.txt
            rm output.json
            ''')

def _gcsfuse_test(version, jobfile_path, fio_metrics_obj, gcs_bucket) -> None:
  """FIO test for gcsfuse of given version.

  Args:
    version(str): Gcsfuse version to perform fio test.
    jobfile(str): Path of the job file.
    fio_metrics_obj(str): Object for extracting fio metrics.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  if version == 'master':
    _install_gcsfuse_source(version, gcs_bucket)
  else:
    _install_gcsfuse(version, gcs_bucket)

  _run_fio_test(jobfile_path, fio_metrics_obj)
  _remove_gcsfuse(version)

def _goofys_test(jobfile_path, fio_metrics_obj, gcs_bucket) -> None:
  """FIO test for latest version of goofys.

  Args:
    jobfile(str): Path of the job file.
    fio_metrics_obj(str): Object for extracting fio metrics.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  _install_goofys(gcs_bucket)
  _run_fio_test(jobfile_path, fio_metrics_obj)
  _remove_goofys()

def _fuse_test(fuse_type, fuse_type_version, jobfile_path, fio_metric_obj, gcs_bucket) -> None:
  """FIO test for specific version of given fuse type.

  Args:
    fuse_type(str): Fuse type for fio test.
    fuse_type_version(str): Fuse type version for fio test.
    jobfile(str): Path of the job file.
    fio_metrics_obj(str): Object for extracting fio metrics.
    gcs_bucket(str): GCS bucket to be mounted.
  """
  if fuse_type == 'gcsfuse':
    _gcsfuse_test(fuse_type_version, jobfile_path, fio_metric_obj, gcs_bucket)
  elif fuse_type == 'goofys':
    _goofys_test(jobfile_path, fio_metric_obj, gcs_bucket)
  else:
    app.UsageError('Unsupported fuse type!')

def main(argv) -> None:

  parser = argparse.ArgumentParser()
  parser.add_argument(
      'fuse_type_1',
      help='Provide reading fuse_type_1 for fio test')
  parser.add_argument(
      'fuse_type_1_version',
      help='Provid the Specific version of fuse_type_1 in approprite formate')
  parser.add_argument(
      'fuse_type_2',
      help='Provide reading fuse_type_2 for fio test')
  parser.add_argument(
      'fuse_type_2_version',
      help='Provid the Specific version of fuse_type_2 in approprite formate')
  parser.add_argument(
      'jobfile_path',
      help='Provid path of the jobfile')
  parser.add_argument(
      'gcs_bucket',
      help="Provide the gcs bucket name to be mounted")
  args = parser.parse_args(argv[1:])

  fio_metrics_obj = fio_metrics.FioMetrics()
  os.system('mkdir out')

  os.system(f'echo Fuse Type 1: {args.fuse_type_1} {args.fuse_type_1_version} >> out/output.txt')
  _fuse_test(args.fuse_type_1, args.fuse_type_1_version, args.jobfile_path, fio_metrics_obj, args.gcs_bucket)

  os.system(f'echo Fuse Type 2: {args.fuse_type_2} {args.fuse_type_2_version} >> out/output.txt')
  _fuse_test(args.fuse_type_2, args.fuse_type_2_version, args.jobfile_path, fio_metrics_obj, args.gcs_bucket)

if __name__ == '__main__':
  app.run(main)

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

import codecs
import difflib
import os
import re
import shutil
import sys
import time
import urlparse

from distutils.dir_util import mkpath
from xml.dom.minidom import Document

from subprocess import PIPE
from subprocess import Popen as execute

from lib.core.common import dataToStdout
from lib.core.common import getUnicode
from lib.core.common import readInput
from lib.core.data import conf
from lib.core.data import logger
from lib.core.data import paths
from lib.core.exception import sqlmapConnectionException
from lib.core.exception import sqlmapFilePathException
from lib.core.settings import MSSQL_VERSIONS_URL
from lib.core.settings import UNICODE_ENCODING
from lib.core.subprocessng import pollProcess
from lib.request.connect import Connect as Request

def __updateMSSQLXML():
    infoMsg = "updating Microsoft SQL Server XML versions file"
    logger.info(infoMsg)

    try:
        mssqlVersionsHtmlString, _ = Request.getPage(url=MSSQL_VERSIONS_URL, direct=True)
    except sqlmapConnectionException, _:
        __mssqlPath     = urlparse.urlsplit(MSSQL_VERSIONS_URL)
        __mssqlHostname = __mssqlPath[1]

        warnMsg  = "sqlmap was unable to connect to %s," % __mssqlHostname
        warnMsg += " check your Internet connection and retry"
        logger.warn(warnMsg)

        return

    releases      = re.findall("class=\"BCC_DV_01DarkBlueTitle\">SQL Server ([\d\.]+) Builds", mssqlVersionsHtmlString, re.I | re.M)
    releasesCount = len(releases)

    # Create the minidom document
    doc = Document()

    # Create the <root> base element
    root = doc.createElement("root")
    doc.appendChild(root)

    for index in range(0, releasesCount):
        release = releases[index]

        # Skip Microsoft SQL Server 6.5 because the HTML
        # table is in another format
        if release == "6.5":
            continue

        # Create the <signatures> base element
        signatures = doc.createElement("signatures")
        signatures.setAttribute("release", release)
        root.appendChild(signatures)

        startIdx = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index])

        if index == releasesCount - 1:
            stopIdx  = len(mssqlVersionsHtmlString)
        else:
            stopIdx  = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index + 1])

        mssqlVersionsReleaseString = mssqlVersionsHtmlString[startIdx:stopIdx]
        servicepackVersion = re.findall("	[7\.0|2000|2005|2008]*(.*?)	<td.*?([\d\.]+)	[\r]*\n", mssqlVersionsReleaseString, re.I | re.M)

        for servicePack, version in servicepackVersion:
            if servicePack.startswith(" "):
                servicePack = servicePack[1:]
            if "/" in servicePack:
                servicePack = servicePack[:servicePack.index("/")]
            if "(" in servicePack:
                servicePack = servicePack[:servicePack.index("(")]
            if "-" in servicePack:
                servicePack = servicePack[:servicePack.index("-")]
            if "*" in servicePack:
                servicePack = servicePack[:servicePack.index("*")]
            if servicePack.startswith("+"):
                servicePack = "0%s" % servicePack

            servicePack = servicePack.replace("\t", " ")
            servicePack = servicePack.replace("No SP", "0")
            servicePack = servicePack.replace("RTM", "0")
            servicePack = servicePack.replace("SP", "")
            servicePack = servicePack.replace("Service Pack", "")
            servicePack = servicePack.replace("<a href=\"http:", "")
            servicePack = servicePack.replace("  ", " ")

            if servicePack.endswith(" "):
                servicePack = servicePack[:-1]

            if servicePack and version:
                # Create the main <card> element
                signature = doc.createElement("signature")
                signatures.appendChild(signature)

                # Create a <version> element
                versionElement = doc.createElement("version")
                signature.appendChild(versionElement)

                # Give the <version> elemenet some text
                versionText = doc.createTextNode(version)
                versionElement.appendChild(versionText)

                # Create a <servicepack> element
                servicepackElement = doc.createElement("servicepack")
                signature.appendChild(servicepackElement)

                # Give the <servicepack> elemenet some text
                servicepackText = doc.createTextNode(servicePack)
                servicepackElement.appendChild(servicepackText)

    # Get the XML old file content to a local variable
    mssqlXml = codecs.open(paths.MSSQL_XML, "r", UNICODE_ENCODING)
    oldMssqlXml = mssqlXml.read()
    oldMssqlXmlSignatures = oldMssqlXml.count("<signature>")
    oldMssqlXmlList = oldMssqlXml.splitlines(1)
    mssqlXml.close()

    # Backup the XML old file
    shutil.copy(paths.MSSQL_XML, "%s.bak" % paths.MSSQL_XML)

    # Save our newly created XML to the signatures file
    mssqlXml = codecs.open(paths.MSSQL_XML, "w", UNICODE_ENCODING)
    doc.writexml(writer=mssqlXml, addindent="    ", newl="\n")
    mssqlXml.close()

    # Get the XML new file content to a local variable
    mssqlXml = codecs.open(paths.MSSQL_XML, "r", UNICODE_ENCODING)
    newMssqlXml = mssqlXml.read()
    newMssqlXmlSignatures = newMssqlXml.count("<signature>")
    newMssqlXmlList = newMssqlXml.splitlines(1)
    mssqlXml.close()

    # If the new XML versions file differs from the old one it probably
    # means that we have got new Microsoft SQL Server versions
    if oldMssqlXmlSignatures != newMssqlXmlSignatures:
        infoMsg  = "Microsoft SQL Server XML versions file updated successfully. "

        if oldMssqlXmlSignatures < newMssqlXmlSignatures:
            infoMsg += "%d " % (newMssqlXmlSignatures - oldMssqlXmlSignatures)
            infoMsg += "new signatures added since the last update"

        # NOTE: This should never happen, in this rare case it might
        # be that the Microsoft SQL Server versions database
        # (MSSQL_VERSIONS_URL) changed its structure
        else:
            infoMsg += "%d " % (oldMssqlXmlSignatures - newMssqlXmlSignatures)
            infoMsg += "signatures removed since the last update"

        logger.info(infoMsg)

        message = "Do you want to see the differences? [Y/n] "
        test = readInput(message, default="Y")

        if not test or test[0] in ("y", "Y"):
            infoMsg = "Differences:"
            logger.info(infoMsg)

            # Compare the old XML file with the new one
            diff = difflib.unified_diff(oldMssqlXmlList, newMssqlXmlList, "%s.bak" % paths.MSSQL_XML, paths.MSSQL_XML)
            sys.stdout.writelines(diff)
    else:
        infoMsg  = "no new Microsoft SQL Server versions since the "
        infoMsg += "last update"
        logger.info(infoMsg)

def __updateSqlmap():
    rootDir = paths.SQLMAP_ROOT_PATH

    infoMsg = "updating sqlmap to latest development version from the "
    infoMsg += "subversion repository"
    logger.info(infoMsg)

    try:
        import pysvn

        debugMsg = "sqlmap will update itself using installed python-svn "
        debugMsg += "third-party library, http://pysvn.tigris.org/"
        logger.debug(debugMsg)

        def notify(event_dict):
            action = getUnicode(event_dict['action'])
            index = action.find('_')
            prefix = action[index + 1].upper() if index != -1 else action.capitalize()

            if action.find('_update') != -1:
                return

            if action.find('_completed') == -1:
                dataToStdout("%s\t%s\n" % (prefix, event_dict['path']))
            else:
                revision = getUnicode(event_dict['revision'], UNICODE_ENCODING)
                index = revision.find('number ')

                if index != -1:
                    revision = revision[index+7:].strip('>')

                logger.info('updated to the latest revision %s' % revision)

        client = pysvn.Client()
        client.callback_notify = notify
        client.update(rootDir)
    except ImportError, _:
        debugMsg = "sqlmap will try to update itself using 'svn' command"
        logger.debug(debugMsg)

        process = execute("svn update %s" % rootDir, shell=True, stdout=PIPE, stderr=PIPE)

        dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
        pollProcess(process)
        svnStdout, svnStderr = process.communicate()

        if svnStderr:
            errMsg = svnStderr.strip()
            logger.error(errMsg)
        elif svnStdout:
            revision = re.search("revision\s+([\d]+)", svnStdout, re.I)
            if revision:
                logger.info('updated to the latest revision %s' % revision.group(1))

def update():
    if not conf.updateAll:
        return

    __updateSqlmap()
    __updateMSSQLXML()
<commit_msg>another minor cosmetic update<commit_after>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

import codecs
import difflib
import os
import re
import shutil
import sys
import time
import urlparse

from distutils.dir_util import mkpath
from xml.dom.minidom import Document

from subprocess import PIPE
from subprocess import Popen as execute

from lib.core.common import dataToStdout
from lib.core.common import getUnicode
from lib.core.common import readInput
from lib.core.data import conf
from lib.core.data import logger
from lib.core.data import paths
from lib.core.exception import sqlmapConnectionException
from lib.core.exception import sqlmapFilePathException
from lib.core.settings import MSSQL_VERSIONS_URL
from lib.core.settings import UNICODE_ENCODING
from lib.core.subprocessng import pollProcess
from lib.request.connect import Connect as Request

def __updateMSSQLXML():
    infoMsg = "updating Microsoft SQL Server XML versions file"
    logger.info(infoMsg)

    try:
        mssqlVersionsHtmlString, _ = Request.getPage(url=MSSQL_VERSIONS_URL, direct=True)
    except sqlmapConnectionException, _:
        __mssqlPath     = urlparse.urlsplit(MSSQL_VERSIONS_URL)
        __mssqlHostname = __mssqlPath[1]

        warnMsg  = "sqlmap was unable to connect to %s," % __mssqlHostname
        warnMsg += " check your Internet connection and retry"
        logger.warn(warnMsg)

        return

    releases      = re.findall("class=\"BCC_DV_01DarkBlueTitle\">SQL Server ([\d\.]+) Builds", mssqlVersionsHtmlString, re.I | re.M)
    releasesCount = len(releases)

    # Create the minidom document
    doc = Document()

    # Create the <root> base element
    root = doc.createElement("root")
    doc.appendChild(root)

    for index in range(0, releasesCount):
        release = releases[index]

        # Skip Microsoft SQL Server 6.5 because the HTML
        # table is in another format
        if release == "6.5":
            continue

        # Create the <signatures> base element
        signatures = doc.createElement("signatures")
        signatures.setAttribute("release", release)
        root.appendChild(signatures)

        startIdx = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index])

        if index == releasesCount - 1:
            stopIdx  = len(mssqlVersionsHtmlString)
        else:
            stopIdx  = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index + 1])

        mssqlVersionsReleaseString = mssqlVersionsHtmlString[startIdx:stopIdx]
        servicepackVersion = re.findall("	[7\.0|2000|2005|2008]*(.*?)	<td.*?([\d\.]+)	[\r]*\n", mssqlVersionsReleaseString, re.I | re.M)

        for servicePack, version in servicepackVersion:
            if servicePack.startswith(" "):
                servicePack = servicePack[1:]
            if "/" in servicePack:
                servicePack = servicePack[:servicePack.index("/")]
            if "(" in servicePack:
                servicePack = servicePack[:servicePack.index("(")]
            if "-" in servicePack:
                servicePack = servicePack[:servicePack.index("-")]
            if "*" in servicePack:
                servicePack = servicePack[:servicePack.index("*")]
            if servicePack.startswith("+"):
                servicePack = "0%s" % servicePack

            servicePack = servicePack.replace("\t", " ")
            servicePack = servicePack.replace("No SP", "0")
            servicePack = servicePack.replace("RTM", "0")
            servicePack = servicePack.replace("SP", "")
            servicePack = servicePack.replace("Service Pack", "")
            servicePack = servicePack.replace("<a href=\"http:", "")
            servicePack = servicePack.replace("  ", " ")
            servicePack = servicePack.replace("+ ", "+")
            servicePack = servicePack.replace(" +", "+")

            if servicePack.endswith(" "):
                servicePack = servicePack[:-1]

            if servicePack and version:
                # Create the main <card> element
                signature = doc.createElement("signature")
                signatures.appendChild(signature)

                # Create a <version> element
                versionElement = doc.createElement("version")
                signature.appendChild(versionElement)

                # Give the <version> elemenet some text
                versionText = doc.createTextNode(version)
                versionElement.appendChild(versionText)

                # Create a <servicepack> element
                servicepackElement = doc.createElement("servicepack")
                signature.appendChild(servicepackElement)

                # Give the <servicepack> elemenet some text
                servicepackText = doc.createTextNode(servicePack)
                servicepackElement.appendChild(servicepackText)

    # Get the XML old file content to a local variable
    mssqlXml = codecs.open(paths.MSSQL_XML, "r", UNICODE_ENCODING)
    oldMssqlXml = mssqlXml.read()
    oldMssqlXmlSignatures = oldMssqlXml.count("<signature>")
    oldMssqlXmlList = oldMssqlXml.splitlines(1)
    mssqlXml.close()

    # Backup the XML old file
    shutil.copy(paths.MSSQL_XML, "%s.bak" % paths.MSSQL_XML)

    # Save our newly created XML to the signatures file
    mssqlXml = codecs.open(paths.MSSQL_XML, "w", UNICODE_ENCODING)
    doc.writexml(writer=mssqlXml, addindent="    ", newl="\n")
    mssqlXml.close()

    # Get the XML new file content to a local variable
    mssqlXml = codecs.open(paths.MSSQL_XML, "r", UNICODE_ENCODING)
    newMssqlXml = mssqlXml.read()
    newMssqlXmlSignatures = newMssqlXml.count("<signature>")
    newMssqlXmlList = newMssqlXml.splitlines(1)
    mssqlXml.close()

    # If the new XML versions file differs from the old one it probably
    # means that we have got new Microsoft SQL Server versions
    if oldMssqlXmlSignatures != newMssqlXmlSignatures:
        infoMsg  = "Microsoft SQL Server XML versions file updated successfully. "

        if oldMssqlXmlSignatures < newMssqlXmlSignatures:
            infoMsg += "%d " % (newMssqlXmlSignatures - oldMssqlXmlSignatures)
            infoMsg += "new signatures added since the last update"

        # NOTE: This should never happen, in this rare case it might
        # be that the Microsoft SQL Server versions database
        # (MSSQL_VERSIONS_URL) changed its structure
        else:
            infoMsg += "%d " % (oldMssqlXmlSignatures - newMssqlXmlSignatures)
            infoMsg += "signatures removed since the last update"

        logger.info(infoMsg)

        message = "Do you want to see the differences? [Y/n] "
        test = readInput(message, default="Y")

        if not test or test[0] in ("y", "Y"):
            infoMsg = "Differences:"
            logger.info(infoMsg)

            # Compare the old XML file with the new one
            diff = difflib.unified_diff(oldMssqlXmlList, newMssqlXmlList, "%s.bak" % paths.MSSQL_XML, paths.MSSQL_XML)
            sys.stdout.writelines(diff)
    else:
        infoMsg  = "no new Microsoft SQL Server versions since the "
        infoMsg += "last update"
        logger.info(infoMsg)

def __updateSqlmap():
    rootDir = paths.SQLMAP_ROOT_PATH

    infoMsg = "updating sqlmap to latest development version from the "
    infoMsg += "subversion repository"
    logger.info(infoMsg)

    try:
        import pysvn

        debugMsg = "sqlmap will update itself using installed python-svn "
        debugMsg += "third-party library, http://pysvn.tigris.org/"
        logger.debug(debugMsg)

        def notify(event_dict):
            action = getUnicode(event_dict['action'])
            index = action.find('_')
            prefix = action[index + 1].upper() if index != -1 else action.capitalize()

            if action.find('_update') != -1:
                return

            if action.find('_completed') == -1:
                dataToStdout("%s\t%s\n" % (prefix, event_dict['path']))
            else:
                revision = getUnicode(event_dict['revision'], UNICODE_ENCODING)
                index = revision.find('number ')

                if index != -1:
                    revision = revision[index+7:].strip('>')

                logger.info('updated to the latest revision %s' % revision)

        client = pysvn.Client()
        client.callback_notify = notify
        client.update(rootDir)
    except ImportError, _:
        debugMsg = "sqlmap will try to update itself using 'svn' command"
        logger.debug(debugMsg)

        process = execute("svn update %s" % rootDir, shell=True, stdout=PIPE, stderr=PIPE)

        dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
        pollProcess(process)
        svnStdout, svnStderr = process.communicate()

        if svnStderr:
            errMsg = svnStderr.strip()
            logger.error(errMsg)
        elif svnStdout:
            revision = re.search("revision\s+([\d]+)", svnStdout, re.I)
            if revision:
                logger.info('updated to the latest revision %s' % revision.group(1))

def update():
    if not conf.updateAll:
        return

    __updateSqlmap()
    __updateMSSQLXML()
<|endoftext|>


<commit_before>import sys

from hwt.serializer.generic.indent import getIndent
from hwt.synthesizer.hObjList import HObjList
from hwt.synthesizer.interfaceLevel.mainBases import InterfaceBase

def pprintInterface(intf, prefix="", indent=0, file=sys.stdout):
    """
    Pretty print interface
    """
    try:
        s = intf._sig
    except AttributeError:
        s = ""
    if s is not "":
        s = " " + repr(s)

    file.write("".join([getIndent(indent), prefix, repr(intf._getFullName()),
                        s]))
    file.write("\n")

    if isinstance(intf, HObjList):
        for i, p in enumerate(intf):
            # interfaces have already name of this array and index in it's name
            pprintInterface(p, prefix=prefix, indent=indent + 1, file=file)
    else:
        for i in intf._interfaces:
            pprintInterface(i, indent=indent + 1, file=file)

def pprintAgents(unitOrIntf, indent=0, prefix="", file=sys.stdout):
    if isinstance(unitOrIntf, InterfaceBase):
        ag = unitOrIntf._ag
    elif isinstance(unitOrIntf, HObjList):
        for i, item in enumerate(unitOrIntf):
            item_prefix = "%s_%d" % (prefix, i)
            pprintAgents(item, indent=indent+1, prefix=item_prefix, file=file)
        return
    else:
        ag = None

    if ag is not None:
        file.write("%s%s%r\n" % (getIndent(indent), prefix, ag))

    for i in unitOrIntf._interfaces:
        pprintAgents(i, indent + 1, file=file)
<commit_msg>pprintInterface: update for intf._sig is None<commit_after>import sys

from hwt.serializer.generic.indent import getIndent
from hwt.synthesizer.hObjList import HObjList
from hwt.synthesizer.interfaceLevel.mainBases import InterfaceBase

def pprintInterface(intf, prefix="", indent=0, file=sys.stdout):
    """
    Pretty print interface
    """
    try:
        s = intf._sig
    except AttributeError:
        s = None
    if s is None:
        s = ""
    else:
        s = " " + repr(s)

    file.write("".join([getIndent(indent), prefix, repr(intf._getFullName()),
                        s]))
    file.write("\n")

    if isinstance(intf, HObjList):
        for i, p in enumerate(intf):
            # interfaces have already name of this array and index in it's name
            pprintInterface(p, prefix=prefix, indent=indent + 1, file=file)
    else:
        for i in intf._interfaces:
            pprintInterface(i, indent=indent + 1, file=file)

def pprintAgents(unitOrIntf, indent=0, prefix="", file=sys.stdout):
    if isinstance(unitOrIntf, InterfaceBase):
        ag = unitOrIntf._ag
    elif isinstance(unitOrIntf, HObjList):
        for i, item in enumerate(unitOrIntf):
            item_prefix = "%s_%d" % (prefix, i)
            pprintAgents(item, indent=indent+1, prefix=item_prefix, file=file)
        return
    else:
        ag = None

    if ag is not None:
        file.write("%s%s%r\n" % (getIndent(indent), prefix, ag))

    for i in unitOrIntf._interfaces:
        pprintAgents(i, indent + 1, file=file)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

import dbus, gobject, avahi
from dbus.mainloop.glib import DBusGMainLoop
import daemon, daemon.pidlockfile
import lockfile
import sys
import urllib
import simplejson as json

TYPE = "_hydra-agent._tcp"

def service_resolved(interface, protocol, name, stype, domain, host,
                     aprotocol, address, port, txt, flags):

    hostname = host[0:host.rfind(".")]

    u = urllib.urlopen('http://localhost/api/add_host/',
                       data=urllib.urlencode({'hostname': hostname}))
    j = json.load(u)

    if not j['success']:
        # what, oh what to do, really?
        print "adding host %s failed %s" % (hostname, j['errors'])

def print_error(err):
    print err

def myhandler(interface, protocol, name, stype, domain, flags):
    if flags & avahi.LOOKUP_RESULT_LOCAL:
        # local service, skip
        pass

    server.ResolveService(interface, protocol, name, stype, domain,
                          avahi.PROTO_UNSPEC, dbus.UInt32(0),
                          reply_handler=service_resolved,
                          error_handler=print_error)

with daemon.DaemonContext(pidfile = \
           daemon.pidlockfile.PIDLockFile('/var/run/hydra-host-discover.pid')):

    loop = DBusGMainLoop()
    bus = dbus.SystemBus(mainloop=loop)
    server = dbus.Interface(bus.get_object(avahi.DBUS_NAME,
                                           avahi.DBUS_PATH_SERVER),
                            avahi.DBUS_INTERFACE_SERVER)
    b = dbus.Interface(bus.get_object(avahi.DBUS_NAME,
                                      server.ServiceBrowserNew(avahi.IF_UNSPEC,
                                                               avahi.PROTO_UNSPEC,
                                                               TYPE, 'local',
                                                               dbus.UInt32(0))),
                       avahi.DBUS_INTERFACE_SERVICE_BROWSER)
    b.connect_to_signal('ItemNew', myhandler)

    gobject.MainLoop().run()
<commit_msg>Use 'json' instead of 'simplejson'<commit_after>#!/usr/bin/env python
#
# ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

import dbus, gobject, avahi
from dbus.mainloop.glib import DBusGMainLoop
import daemon, daemon.pidlockfile
import lockfile
import sys
import urllib
import json

TYPE = "_hydra-agent._tcp"

def service_resolved(interface, protocol, name, stype, domain, host,
                     aprotocol, address, port, txt, flags):

    hostname = host[0:host.rfind(".")]

    u = urllib.urlopen('http://localhost/api/add_host/',
                       data=urllib.urlencode({'hostname': hostname}))
    j = json.load(u)

    if not j['success']:
        # what, oh what to do, really?
        print "adding host %s failed %s" % (hostname, j['errors'])

def print_error(err):
    print err

def myhandler(interface, protocol, name, stype, domain, flags):
    if flags & avahi.LOOKUP_RESULT_LOCAL:
        # local service, skip
        pass

    server.ResolveService(interface, protocol, name, stype, domain,
                          avahi.PROTO_UNSPEC, dbus.UInt32(0),
                          reply_handler=service_resolved,
                          error_handler=print_error)

with daemon.DaemonContext(pidfile = \
           daemon.pidlockfile.PIDLockFile('/var/run/hydra-host-discover.pid')):

    loop = DBusGMainLoop()
    bus = dbus.SystemBus(mainloop=loop)
    server = dbus.Interface(bus.get_object(avahi.DBUS_NAME,
                                           avahi.DBUS_PATH_SERVER),
                            avahi.DBUS_INTERFACE_SERVER)
    b = dbus.Interface(bus.get_object(avahi.DBUS_NAME,
                                      server.ServiceBrowserNew(avahi.IF_UNSPEC,
                                                               avahi.PROTO_UNSPEC,
                                                               TYPE, 'local',
                                                               dbus.UInt32(0))),
                       avahi.DBUS_INTERFACE_SERVICE_BROWSER)
    b.connect_to_signal('ItemNew', myhandler)

    gobject.MainLoop().run()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2009 Phil Sung
#
# This file is part of Zeya.
#
# Zeya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Zeya is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Zeya. If not, see <http://www.gnu.org/licenses/>.

# Console frontend to Zeya server.
#
# Usage:
#   zeyaclient.py http://server.local:8080
#
# The user is prompted for a query, and all songs matching it are played. The
# query may be matched against the title, artist, or album of the song.

import readline # Modifies the behavior of raw_input.
import subprocess
import sys
import urllib2

try:
    import json
    json.dumps
except (ImportError, AttributeError):
    import simplejson as json

def song_matches(query, song):
    """
    Return True if the query matches the given song.

    A query might look like "help, the beatles"

    A song is considered to match if each comma-separated component of the
    query appears somewhere in one of the song's metadata fields. Matching is
    case-insensitive.
    """
    parts = [part.strip() for part in query.lower().split(",")]
    return all(part in song['album'].lower() or part in song['title'].lower() \
                   or part in song['artist'].lower() for part in parts)

# TODO: refactor the parts that directly interact with the server into a
# separate module.
def run(server_path):
    library_file = urllib2.urlopen(server_path + "/getlibrary")
    library_data = json.loads(library_file.read())
    print "Loaded %d songs from library." % (len(library_data),)
    print 'You can issue queries like: "Beatles" or "help, the beatles"'
    while True:
        # Prompt user for a query...
        query = raw_input("Query? ")
        if not query:
            break
        # ...then play all the songs we can find that match the query.
        matching_songs = \
            [song for song in library_data if song_matches(query, song)]
        for song in matching_songs:
            print "%s - %s" % (song['title'], song['artist'])
            song_url = "%s/getcontent?key=%d" % (server_path, song['key'])
            p = subprocess.Popen(["/usr/bin/ogg123", "-q", song_url])
            try:
                p.communicate()
            except KeyboardInterrupt:
                # TODO: handle Ctrl-C here in a better way. Whenever an
                # interrupt is followed by a new query (raw_input above) we
                # seem to get a spurious EOFError.
                pass

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print "Usage: zeyaclient.py http://server:8080"
        sys.exit(1)
    run(sys.argv[1])
<commit_msg>zeyaclient.py: handle C-c gracefully.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2009 Phil Sung
#
# This file is part of Zeya.
#
# Zeya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Zeya is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Zeya. If not, see <http://www.gnu.org/licenses/>.

# Console frontend to Zeya server.
#
# Usage:
#   zeyaclient.py http://server.local:8080
#
# The user is prompted for a query, and all songs matching it are played. The
# query may be matched against the title, artist, or album of the song.

import os
import readline # Modifies the behavior of raw_input.
import signal
import subprocess
import sys
import time
import urllib2

try:
    import json
    json.dumps
except (ImportError, AttributeError):
    import simplejson as json

def song_matches(query, song):
    """
    Return True if the query matches the given song.

    A query might look like "help, the beatles"

    A song is considered to match if each comma-separated component of the
    query appears somewhere in one of the song's metadata fields. Matching is
    case-insensitive.
    """
    parts = [part.strip() for part in query.lower().split(",")]
    return all(part in song['album'].lower() or part in song['title'].lower() \
                   or part in song['artist'].lower() for part in parts)

# TODO: refactor the parts that directly interact with the server into a
# separate module.
def run(server_path):
    library_file = urllib2.urlopen(server_path + "/getlibrary")
    library_data = json.loads(library_file.read())
    print "Loaded %d songs from library." % (len(library_data),)
    print 'You can issue queries like: "Beatles" or "help, the beatles"'
    while True:
        # Prompt user for a query...
        try:
            query = raw_input("\rQuery? ")
        except:
            # User pressed C-d or C-c.
            print
            break
        if not query:
            break
        # ...then play all the songs we can find that match the query.
        matching_songs = \
            [song for song in library_data if song_matches(query, song)]
        for song in matching_songs:
            print "\r%s - %s" % (song['title'], song['artist'])
            song_url = "%s/getcontent?key=%d" % (server_path, song['key'])
            p = subprocess.Popen(["/usr/bin/ogg123", "-q", song_url])
            try:
                p.communicate()
            except KeyboardInterrupt:
                # After a single ^C, skip to the next song.
                os.kill(p.pid, signal.SIGTERM)
                try:
                    time.sleep(0.5)
                except KeyboardInterrupt:
                    # If ^C^C is typed (within 0.5 sec) then break out back to
                    # the prompt.
                    break

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print "Usage: zeyaclient.py http://server:8080"
        sys.exit(1)
    run(sys.argv[1])
<|endoftext|>


<commit_before>import importlib
import logging

def init():
    """
    Initialize nazs environment, setup logging, processes and all
    needed stuff for running nazs
    """
    from django.core import management
    # Sync volatile db, TODO set correct permissions
    management.call_command('syncdb', database='volatile', interactive=False)

    from .sudo import set_euid
    set_euid()

    # Load all modules
    from django.conf import settings
    for app in settings.INSTALLED_APPS:
        try:
            importlib.import_module(app + '.module')
        except:
            pass

def modules():
    """
    Return a list of instances of all present modules
    """
    from .module import Module
    return [cls() for cls in Module.MODULES]

def changed():
    """
    Return True if there is any change in any of the available modules
    """
    for module in modules():
        if module.changed:
            return True
    return False

def save():
    """
    Apply configuration changes on all the modules
    """
    from .models import ModuleInfo

    logger = logging.getLogger(__name__)
    logger.info("Saving changes")

    # Save + restart
    for module in modules():
        if module.enabled:
            if module.changed:
                module.save()
                module.restart()
                module.commit()
            else:
                logger.debug("Not saving unchanged module: %s" % module.name)
        else:
            logger.debug("Not saving disabled module: %s" % module.name)

    # Commit
    ModuleInfo.commit()

    logger.info("Changes saved")

def menu():
    """
    Return global menu composed from all modules
    """
    from . import module

    root = module.MenuItem('')

    for mod in modules():
        if mod.installed:
            mod.menu(root)

    return root
<commit_msg>Fail when importing a broken module<commit_after>import importlib
import logging
import traceback
import sys
import six

def init():
    """
    Initialize nazs environment, setup logging, processes and all
    needed stuff for running nazs
    """
    from django.core import management
    # Sync volatile db, TODO set correct permissions
    management.call_command('syncdb', database='volatile', interactive=False)

    from .sudo import set_euid
    set_euid()

    # Load all modules
    from django.conf import settings
    for app in settings.INSTALLED_APPS:
        module_path = app + '.module'
        if six.PY2:
            try:
                importlib.import_module(module_path)
            except ImportError:
                tb = sys.exc_info()[2]
                stack = traceback.extract_tb(tb, 3)
                if len(stack) > 2:
                    raise
        else:
            from importlib import find_loader
            if find_loader(module_path):
                import_module(module_path)

def modules():
    """
    Return a list of instances of all present modules
    """
    from .module import Module
    return [cls() for cls in Module.MODULES]

def changed():
    """
    Return True if there is any change in any of the available modules
    """
    for module in modules():
        if module.changed:
            return True
    return False

def save():
    """
    Apply configuration changes on all the modules
    """
    from .models import ModuleInfo

    logger = logging.getLogger(__name__)
    logger.info("Saving changes")

    # Save + restart
    for module in modules():
        if module.enabled:
            if module.changed:
                module.save()
                module.restart()
                module.commit()
            else:
                logger.debug("Not saving unchanged module: %s" % module.name)
        else:
            logger.debug("Not saving disabled module: %s" % module.name)

    # Commit
    ModuleInfo.commit()

    logger.info("Changes saved")

def menu():
    """
    Return global menu composed from all modules
    """
    from . import module

    root = module.MenuItem('')

    for mod in modules():
        if mod.installed:
            mod.menu(root)

    return root
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2009 Phil Sung
#
# This file is part of Zeya.
#
# Zeya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Zeya is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Zeya. If not, see <http://www.gnu.org/licenses/>.

# Console frontend to Zeya server.
#
# Usage:
#   zeyaclient.py http://server.local:8080
#
# The user is prompted for a query, and all songs matching it are played. The
# query may be matched against the title, artist, or album of the song.

import getopt
import os
import readline # Modifies the behavior of raw_input.
import signal
import subprocess
import sys
import time
import urllib2

try:
    import json
    json.dumps
except (ImportError, AttributeError):
    import simplejson as json

class BadArgsError(Exception):
    """
    Error due to incorrect command-line invocation of this program.
    """
    def __init__(self, message):
        self.error_message = message
    def __str__(self):
        return "Error: %s" % (self.error_message,)

def song_matches(query, song):
    """
    Return True if the query matches the given song.

    A query might look like "help, the beatles"

    A song is considered to match if each comma-separated component of the
    query appears somewhere in one of the song's metadata fields. Matching is
    case-insensitive.
    """
    parts = [part.strip() for part in query.lower().split(",")]
    return all(part in song['album'].lower() or part in song['title'].lower() \
                   or part in song['artist'].lower() for part in parts)

# TODO: refactor the parts that directly interact with the server into a
# separate module.
def run(server_path):
    library_file = urllib2.urlopen(server_path + "/getlibrary")
    library_data = json.loads(library_file.read())
    print "Loaded %d songs from library." % (len(library_data),)
    print 'You can issue queries like: "Beatles" or "help, the beatles"'
    while True:
        # Prompt user for a query...
        try:
            query = raw_input("\rQuery? ")
        except:
            # User pressed C-d or C-c.
            print
            break
        if not query:
            break
        # ...then play all the songs we can find that match the query.
        matching_songs = \
            [song for song in library_data if song_matches(query, song)]
        for song in matching_songs:
            print "\r%s - %s" % (song['title'], song['artist'])
            song_url = "%s/getcontent?key=%d" % (server_path, song['key'])
            p = subprocess.Popen(["/usr/bin/ogg123", "-q", song_url])
            try:
                p.communicate()
            except KeyboardInterrupt:
                # After a single ^C, skip to the next song.
                os.kill(p.pid, signal.SIGTERM)
                try:
                    time.sleep(0.5)
                except KeyboardInterrupt:
                    # If ^C^C is typed (within 0.5 sec) then break out back to
                    # the prompt.
                    break

def get_options(remaining_args):
    """
    Parse the arguments and return a tuple (show_help, server), or raise
    BadArgsError if the invocation was not valid.

    show_help: whether user requested help information
    server: Zeya server to connect to
    """
    help_msg = False
    try:
        opts, file_list = getopt.getopt(remaining_args, "h", ["help"])
    except getopt.GetoptError, e:
        raise BadArgsError(e.msg)
    for flag, value in opts:
        if flag in ("-h", "--help"):
            help_msg = True
    if help_msg:
        # With --help, it's ok if the user enters no server path.
        return (help_msg, None)
    # file_list should contain be a singleton list with the remote server.
    if len(file_list) == 0:
        raise BadArgsError("Expected server path")
    if len(file_list) > 1:
        raise BadArgsError("Unexpected argument after server")
    return (help_msg, file_list[0])

def print_usage():
    print "Usage: %s http://server:8080" % (os.path.basename(sys.argv[0]),)

if __name__ == "__main__":
    try:
        show_help, server = get_options(sys.argv[1:])
    except BadArgsError, e:
        print e
        print_usage()
        sys.exit(1)
    if show_help:
        print_usage()
        sys.exit(0)
    run(server)
<commit_msg>zeyaclient.py: update comment.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2009 Phil Sung
#
# This file is part of Zeya.
#
# Zeya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Zeya is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Zeya. If not, see <http://www.gnu.org/licenses/>.

# Console frontend to Zeya server.
#
# Usage:
#   zeyaclient.py http://server.local:8080
#
# zeyaclient prompts the user for a query, and all songs matching the query are
# played. The query may be matched against the title, artist, or album of the
# song.

import getopt
import os
import readline # Modifies the behavior of raw_input.
import signal
import subprocess
import sys
import time
import urllib2

try:
    import json
    json.dumps
except (ImportError, AttributeError):
    import simplejson as json

class BadArgsError(Exception):
    """
    Error due to incorrect command-line invocation of this program.
    """
    def __init__(self, message):
        self.error_message = message
    def __str__(self):
        return "Error: %s" % (self.error_message,)

def song_matches(query, song):
    """
    Return True if the query matches the given song.

    A query might look like "help, the beatles"

    A song is considered to match if each comma-separated component of the
    query appears somewhere in one of the song's metadata fields. Matching is
    case-insensitive.
    """
    parts = [part.strip() for part in query.lower().split(",")]
    return all(part in song['album'].lower() or part in song['title'].lower() \
                   or part in song['artist'].lower() for part in parts)

# TODO: refactor the parts that directly interact with the server into a
# separate module.
def run(server_path):
    library_file = urllib2.urlopen(server_path + "/getlibrary")
    library_data = json.loads(library_file.read())
    print "Loaded %d songs from library." % (len(library_data),)
    print 'You can issue queries like: "Beatles" or "help, the beatles"'
    while True:
        # Prompt user for a query...
        try:
            query = raw_input("\rQuery? ")
        except:
            # User pressed C-d or C-c.
            print
            break
        if not query:
            break
        # ...then play all the songs we can find that match the query.
        matching_songs = \
            [song for song in library_data if song_matches(query, song)]
        for song in matching_songs:
            print "\r%s - %s" % (song['title'], song['artist'])
            song_url = "%s/getcontent?key=%d" % (server_path, song['key'])
            p = subprocess.Popen(["/usr/bin/ogg123", "-q", song_url])
            try:
                p.communicate()
            except KeyboardInterrupt:
                # After a single ^C, skip to the next song.
                os.kill(p.pid, signal.SIGTERM)
                try:
                    time.sleep(0.5)
                except KeyboardInterrupt:
                    # If ^C^C is typed (within 0.5 sec) then break out back to
                    # the prompt.
                    break

def get_options(remaining_args):
    """
    Parse the arguments and return a tuple (show_help, server), or raise
    BadArgsError if the invocation was not valid.

    show_help: whether user requested help information
    server: Zeya server to connect to
    """
    help_msg = False
    try:
        opts, file_list = getopt.getopt(remaining_args, "h", ["help"])
    except getopt.GetoptError, e:
        raise BadArgsError(e.msg)
    for flag, value in opts:
        if flag in ("-h", "--help"):
            help_msg = True
    if help_msg:
        # With --help, it's ok if the user enters no server path.
        return (help_msg, None)
    # file_list should contain be a singleton list with the remote server.
    if len(file_list) == 0:
        raise BadArgsError("Expected server path")
    if len(file_list) > 1:
        raise BadArgsError("Unexpected argument after server")
    return (help_msg, file_list[0])

def print_usage():
    print "Usage: %s http://server:8080" % (os.path.basename(sys.argv[0]),)

if __name__ == "__main__":
    try:
        show_help, server = get_options(sys.argv[1:])
    except BadArgsError, e:
        print e
        print_usage()
        sys.exit(1)
    if show_help:
        print_usage()
        sys.exit(0)
    run(server)
<|endoftext|>