<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    The refactoring about the OpenSSL support come from Tryton
#    Copyright (C) 2007-2009 Cédric Krier.
#    Copyright (C) 2007-2009 Bertrand Chenal.
#    Copyright (C) 2008 B2CK SPRL.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import errno
import logging
import logging.handlers
import os
import socket
import sys
import threading
import time
import release
from pprint import pformat
import warnings

class Service(object):
    """ Base class for *Local* services

        Functionality here is trusted, no authentication.
    """
    _services = {}
    def __init__(self, name, audience=''):
        Service._services[name] = self
        self.__name = name
        self._methods = {}

    def joinGroup(self, name):
        raise Exception("No group for local services")
        #GROUPS.setdefault(name, {})[self.__name] = self

    @classmethod
    def exists(cls, name):
        return name in cls._services

    @classmethod
    def remove(cls, name):
        if cls.exists(name):
            cls._services.pop(name)

    def exportMethod(self, method):
        if callable(method):
            self._methods[method.__name__] = method

    def abortResponse(self, error, description, origin, details):
        if not tools.config['debug_mode']:
            raise Exception("%s -- %s\n\n%s"%(origin, description, details))
        else:
            raise

class LocalService(object):
    """ Proxy for local services.

        Any instance of this class will behave like the single instance
        of Service(name)
    """
    __logger = logging.getLogger('service')
    def __init__(self, name):
        self.__name = name
        try:
            self._service = Service._services[name]
            for method_name, method_definition in self._service._methods.items():
                setattr(self, method_name, method_definition)
        except KeyError, keyError:
            self.__logger.error('This service does not exist: %s' % (str(keyError),) )
            raise

    def __call__(self, method, *params):
        return getattr(self, method)(*params)

class ExportService(object):
    """ Proxy for exported services.

    All methods here should take an AuthProxy as their first parameter. It
    will be appended by the calling framework.

    Note that this class has no direct proxy, capable of calling
    eservice.method(). Rather, the proxy should call
    dispatch(method,auth,params)
    """

    _services = {}
    _groups = {}

    def __init__(self, name, audience=''):
        ExportService._services[name] = self
        self.__name = name

    def joinGroup(self, name):
        ExportService._groups.setdefault(name, {})[self.__name] = self

    @classmethod
    def getService(cls,name):
        return cls._services[name]

    def dispatch(self, method, auth, params):
        raise Exception("stub dispatch at %s" % self.__name)

    def new_dispatch(self,method,auth,params):
        raise Exception("stub dispatch at %s" % self.__name)

    def abortResponse(self, error, description, origin, details):
        if not tools.config['debug_mode']:
            raise Exception("%s -- %s\n\n%s"%(origin, description, details))
        else:
            raise

LOG_NOTSET = 'notset'
LOG_DEBUG_RPC = 'debug_rpc'
LOG_DEBUG = 'debug'
LOG_TEST = 'test'
LOG_INFO = 'info'
LOG_WARNING = 'warn'
LOG_ERROR = 'error'
LOG_CRITICAL = 'critical'

logging.DEBUG_RPC = logging.DEBUG - 2
logging.addLevelName(logging.DEBUG_RPC, 'DEBUG_RPC')

logging.TEST = logging.INFO - 5
logging.addLevelName(logging.TEST, 'TEST')

BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, _NOTHING, DEFAULT = range(10)
#The background is set with 40 plus the number of the color, and the foreground with 30
#These are the sequences need to get colored ouput
RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
BOLD_SEQ = "\033[1m"
COLOR_PATTERN = "%s%s%%s%s" % (COLOR_SEQ, COLOR_SEQ, RESET_SEQ)
LEVEL_COLOR_MAPPING = {
    logging.DEBUG_RPC: (BLUE, WHITE),
    logging.DEBUG: (BLUE, DEFAULT),
    logging.INFO: (GREEN, DEFAULT),
    logging.TEST: (WHITE, BLUE),
    logging.WARNING: (YELLOW, DEFAULT),
    logging.ERROR: (RED, DEFAULT),
    logging.CRITICAL: (WHITE, RED),
}

class ColoredFormatter(logging.Formatter):
    def format(self, record):
        fg_color, bg_color = LEVEL_COLOR_MAPPING[record.levelno]
        record.levelname = COLOR_PATTERN % (30 + fg_color, 40 + bg_color, record.levelname)
        return logging.Formatter.format(self, record)

def init_logger():
    import os
    from tools.translate import resetlocale
    resetlocale()

    logger = logging.getLogger()
    # create a format for log messages and dates
    format = '[%(asctime)s] %(levelname)s:%(name)s:%(message)s'

    if tools.config['syslog']:
        # SysLog Handler
        if os.name == 'nt':
            handler = logging.handlers.NTEventLogHandler("%s %s" % (release.description, release.version))
        else:
            handler = logging.handlers.SysLogHandler('/dev/log')
        format = '%s %s' % (release.description, release.version) \
               + ':%(levelname)s:%(name)s:%(message)s'

    elif tools.config['logfile']:
        # LogFile Handler
        logf = tools.config['logfile']
        try:
            dirname = os.path.dirname(logf)
            if dirname and not os.path.isdir(dirname):
                os.makedirs(dirname)
            if tools.config['logrotate'] is not False:
                handler = logging.handlers.TimedRotatingFileHandler(logf,'D',1,30)
            elif os.name == 'posix':
                handler = logging.handlers.WatchedFileHandler(logf)
            else:
                handler = logging.handlers.FileHandler(logf)
        except Exception, ex:
            sys.stderr.write("ERROR: couldn't create the logfile directory. Logging to the standard output.\n")
            handler = logging.StreamHandler(sys.stdout)
    else:
        # Normal Handler on standard output
        handler = logging.StreamHandler(sys.stdout)

    if isinstance(handler, logging.StreamHandler) and os.isatty(handler.stream.fileno()):
        formatter = ColoredFormatter(format)
    else:
        formatter = logging.Formatter(format)
    handler.setFormatter(formatter)

    # add the handler to the root logger
    logger.addHandler(handler)
    logger.setLevel(int(tools.config['log_level'] or '0'))

class Logger(object):
    def __init__(self):
        warnings.warn("The netsvc.Logger API shouldn't be used anymore, please "
                      "use the standard `logging.getLogger` API instead",
                      PendingDeprecationWarning, stacklevel=2)
        super(Logger, self).__init__()

    def notifyChannel(self, name, level, msg):
        warnings.warn("notifyChannel API shouldn't be used anymore, please use "
                      "the standard `logging` module instead",
                      PendingDeprecationWarning, stacklevel=2)
        from service.web_services import common

        log = logging.getLogger(tools.ustr(name))

        if level in [LOG_DEBUG_RPC, LOG_TEST] and not hasattr(log, level):
            fct = lambda msg, *args, **kwargs: log.log(getattr(logging, level.upper()), msg, *args, **kwargs)
            setattr(log, level, fct)

        level_method = getattr(log, level)

        if isinstance(msg, Exception):
            msg = tools.exception_to_unicode(msg)

        try:
            msg = tools.ustr(msg).strip()
            if level in (LOG_ERROR, LOG_CRITICAL) and tools.config.get_misc('debug','env_info',False):
                msg = common().exp_get_server_environment() + "\n" + msg

            result = msg.split('\n')
        except UnicodeDecodeError:
            result = msg.strip().split('\n')
        try:
            if len(result)>1:
                for idx, s in enumerate(result):
                    level_method('[%02d]: %s' % (idx+1, s,))
            elif result:
                level_method(result[0])
        except IOError,e:
            # TODO: perhaps reset the logger streams?
            #if logrotate closes our files, we end up here..
            pass
        except:
            # better ignore the exception and carry on..
            pass

    def set_loglevel(self, level):
        log = logging.getLogger()
        log.setLevel(logging.INFO) # make sure next msg is printed
        log.info("Log level changed to %s" % logging.getLevelName(level))
        log.setLevel(level)

    def shutdown(self):
        logging.shutdown()

import tools
init_logger()

class Agent(object):
    _timers = {}
    _logger = Logger()

    __logger = logging.getLogger('timer')

    def setAlarm(self, fn, dt, db_name, *args, **kwargs):
        wait = dt - time.time()
        if wait > 0:
            self.__logger.debug("Job scheduled in %.3g seconds for %s.%s" % (wait, fn.im_class.__name__, fn.func_name))
            timer = threading.Timer(wait, fn, args, kwargs)
            timer.start()
            self._timers.setdefault(db_name, []).append(timer)

        for db in self._timers:
            for timer in self._timers[db]:
                if not timer.isAlive():
                    self._timers[db].remove(timer)

    @classmethod
    def cancel(cls, db_name):
        """Cancel all timers for a given database. If None passed, all timers are cancelled"""
        for db in cls._timers:
            if db_name is None or db == db_name:
                for timer in cls._timers[db]:
                    timer.cancel()

    @classmethod
    def quit(cls):
        cls.cancel(None)

import traceback

class Server:
    """ Generic interface for all servers with an event loop etc.
        Override this to impement http, net-rpc etc. servers.

        Servers here must have threaded behaviour. start() must not block,
        there is no run().
    """
    __is_started = False
    __servers = []
    __starter_threads = []

    # we don't want blocking server calls (think select()) to
    # wait forever and possibly prevent exiting the process,
    # but instead we want a form of polling/busy_wait pattern, where
    # _server_timeout should be used as the default timeout for
    # all I/O blocking operations
    _busywait_timeout = 0.5

    __logger = logging.getLogger('server')

    def __init__(self):
        Server.__servers.append(self)
        if Server.__is_started:
            # raise Exception('All instances of servers must be inited before the startAll()')
            # Since the startAll() won't be called again, allow this server to
            # init and then start it after 1sec (hopefully). Register that
            # timer thread in a list, so that we can abort the start if quitAll
            # is called in the meantime
            t = threading.Timer(1.0, self._late_start)
            Server.__starter_threads.append(t)
            t.start()

    def start(self):
        self.__logger.debug("called stub Server.start")

    def _late_start(self):
        self.start()
        for thr in Server.__starter_threads:
            if thr.finished.is_set():
                Server.__starter_threads.remove(thr)

    def stop(self):
        self.__logger.debug("called stub Server.stop")

    def stats(self):
        """ This function should return statistics about the server """
        return "%s: No statistics" % str(self.__class__)

    @classmethod
    def startAll(cls):
        if cls.__is_started:
            return
        cls.__logger.info("Starting %d services" % len(cls.__servers))
        for srv in cls.__servers:
            srv.start()
        cls.__is_started = True

    @classmethod
    def quitAll(cls):
        if not cls.__is_started:
            return
        cls.__logger.info("Stopping %d services" % len(cls.__servers))
        for thr in cls.__starter_threads:
            if not thr.finished.is_set():
                thr.cancel()
            cls.__starter_threads.remove(thr)

        for srv in cls.__servers:
            srv.stop()
        cls.__is_started = False

    @classmethod
    def allStats(cls):
        res = ["Servers %s" % ('stopped', 'started')[cls.__is_started]]
        res.extend(srv.stats() for srv in cls.__servers)
        return '\n'.join(res)

    def _close_socket(self):
        if os.name != 'nt':
            try:
                self.socket.shutdown(getattr(socket, 'SHUT_RDWR', 2))
            except socket.error, e:
                if e.errno != errno.ENOTCONN: raise
                # OSX, socket shutdowns both sides if any side closes it
                # causing an error 57 'Socket is not connected' on shutdown
                # of the other side (or something), see
                # http://bugs.python.org/issue4397
                self.__logger.debug(
                    '"%s" when shutting down server socket, '
                    'this is normal under OS X', e)
        self.socket.close()

class OpenERPDispatcherException(Exception):
    def __init__(self, exception, traceback):
        self.exception = exception
        self.traceback = traceback

class OpenERPDispatcher:
    def log(self, title, msg):
        if tools.config['log_level'] == logging.DEBUG_RPC:
            Logger().notifyChannel('%s' % title, LOG_DEBUG_RPC, pformat(msg))

    def dispatch(self, service_name, method, params):
        try:
            self.log('service', service_name)
            self.log('method', method)
            self.log('params', params)
            auth = getattr(self, 'auth_provider', None)
            result = ExportService.getService(service_name).dispatch(method, auth, params)
            self.log('result', result)
            # We shouldn't marshall None,
            if result == None:
                result = False
            return result
        except Exception, e:
            self.log('exception', tools.exception_to_unicode(e))
            tb = getattr(e, 'traceback', sys.exc_info())
            tb_s = "".join(traceback.format_exception(*tb))
            if tools.config['debug_mode']:
                import pdb
                pdb.post_mortem(tb[2])
            raise OpenERPDispatcherException(e, tb_s)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>Netsvc: try to have a name for late-start threads.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    The refactoring about the OpenSSL support come from Tryton
#    Copyright (C) 2007-2009 Cédric Krier.
#    Copyright (C) 2007-2009 Bertrand Chenal.
#    Copyright (C) 2008 B2CK SPRL.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import errno
import logging
import logging.handlers
import os
import socket
import sys
import threading
import time
import release
from pprint import pformat
import warnings

class Service(object):
    """ Base class for *Local* services

        Functionality here is trusted, no authentication.
    """
    _services = {}
    def __init__(self, name, audience=''):
        Service._services[name] = self
        self.__name = name
        self._methods = {}

    def joinGroup(self, name):
        raise Exception("No group for local services")
        #GROUPS.setdefault(name, {})[self.__name] = self

    @classmethod
    def exists(cls, name):
        return name in cls._services

    @classmethod
    def remove(cls, name):
        if cls.exists(name):
            cls._services.pop(name)

    def exportMethod(self, method):
        if callable(method):
            self._methods[method.__name__] = method

    def abortResponse(self, error, description, origin, details):
        if not tools.config['debug_mode']:
            raise Exception("%s -- %s\n\n%s"%(origin, description, details))
        else:
            raise

class LocalService(object):
    """ Proxy for local services.

        Any instance of this class will behave like the single instance
        of Service(name)
    """
    __logger = logging.getLogger('service')
    def __init__(self, name):
        self.__name = name
        try:
            self._service = Service._services[name]
            for method_name, method_definition in self._service._methods.items():
                setattr(self, method_name, method_definition)
        except KeyError, keyError:
            self.__logger.error('This service does not exist: %s' % (str(keyError),) )
            raise

    def __call__(self, method, *params):
        return getattr(self, method)(*params)

class ExportService(object):
    """ Proxy for exported services.

    All methods here should take an AuthProxy as their first parameter. It
    will be appended by the calling framework.

    Note that this class has no direct proxy, capable of calling
    eservice.method(). Rather, the proxy should call
    dispatch(method,auth,params)
    """

    _services = {}
    _groups = {}

    def __init__(self, name, audience=''):
        ExportService._services[name] = self
        self.__name = name

    def joinGroup(self, name):
        ExportService._groups.setdefault(name, {})[self.__name] = self

    @classmethod
    def getService(cls,name):
        return cls._services[name]

    def dispatch(self, method, auth, params):
        raise Exception("stub dispatch at %s" % self.__name)

    def new_dispatch(self,method,auth,params):
        raise Exception("stub dispatch at %s" % self.__name)

    def abortResponse(self, error, description, origin, details):
        if not tools.config['debug_mode']:
            raise Exception("%s -- %s\n\n%s"%(origin, description, details))
        else:
            raise

LOG_NOTSET = 'notset'
LOG_DEBUG_RPC = 'debug_rpc'
LOG_DEBUG = 'debug'
LOG_TEST = 'test'
LOG_INFO = 'info'
LOG_WARNING = 'warn'
LOG_ERROR = 'error'
LOG_CRITICAL = 'critical'

logging.DEBUG_RPC = logging.DEBUG - 2
logging.addLevelName(logging.DEBUG_RPC, 'DEBUG_RPC')

logging.TEST = logging.INFO - 5
logging.addLevelName(logging.TEST, 'TEST')

BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, _NOTHING, DEFAULT = range(10)
#The background is set with 40 plus the number of the color, and the foreground with 30
#These are the sequences need to get colored ouput
RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
BOLD_SEQ = "\033[1m"
COLOR_PATTERN = "%s%s%%s%s" % (COLOR_SEQ, COLOR_SEQ, RESET_SEQ)
LEVEL_COLOR_MAPPING = {
    logging.DEBUG_RPC: (BLUE, WHITE),
    logging.DEBUG: (BLUE, DEFAULT),
    logging.INFO: (GREEN, DEFAULT),
    logging.TEST: (WHITE, BLUE),
    logging.WARNING: (YELLOW, DEFAULT),
    logging.ERROR: (RED, DEFAULT),
    logging.CRITICAL: (WHITE, RED),
}

class ColoredFormatter(logging.Formatter):
    def format(self, record):
        fg_color, bg_color = LEVEL_COLOR_MAPPING[record.levelno]
        record.levelname = COLOR_PATTERN % (30 + fg_color, 40 + bg_color, record.levelname)
        return logging.Formatter.format(self, record)

def init_logger():
    import os
    from tools.translate import resetlocale
    resetlocale()

    logger = logging.getLogger()
    # create a format for log messages and dates
    format = '[%(asctime)s] %(levelname)s:%(name)s:%(message)s'

    if tools.config['syslog']:
        # SysLog Handler
        if os.name == 'nt':
            handler = logging.handlers.NTEventLogHandler("%s %s" % (release.description, release.version))
        else:
            handler = logging.handlers.SysLogHandler('/dev/log')
        format = '%s %s' % (release.description, release.version) \
               + ':%(levelname)s:%(name)s:%(message)s'

    elif tools.config['logfile']:
        # LogFile Handler
        logf = tools.config['logfile']
        try:
            dirname = os.path.dirname(logf)
            if dirname and not os.path.isdir(dirname):
                os.makedirs(dirname)
            if tools.config['logrotate'] is not False:
                handler = logging.handlers.TimedRotatingFileHandler(logf,'D',1,30)
            elif os.name == 'posix':
                handler = logging.handlers.WatchedFileHandler(logf)
            else:
                handler = logging.handlers.FileHandler(logf)
        except Exception, ex:
            sys.stderr.write("ERROR: couldn't create the logfile directory. Logging to the standard output.\n")
            handler = logging.StreamHandler(sys.stdout)
    else:
        # Normal Handler on standard output
        handler = logging.StreamHandler(sys.stdout)

    if isinstance(handler, logging.StreamHandler) and os.isatty(handler.stream.fileno()):
        formatter = ColoredFormatter(format)
    else:
        formatter = logging.Formatter(format)
    handler.setFormatter(formatter)

    # add the handler to the root logger
    logger.addHandler(handler)
    logger.setLevel(int(tools.config['log_level'] or '0'))

class Logger(object):
    def __init__(self):
        warnings.warn("The netsvc.Logger API shouldn't be used anymore, please "
                      "use the standard `logging.getLogger` API instead",
                      PendingDeprecationWarning, stacklevel=2)
        super(Logger, self).__init__()

    def notifyChannel(self, name, level, msg):
        warnings.warn("notifyChannel API shouldn't be used anymore, please use "
                      "the standard `logging` module instead",
                      PendingDeprecationWarning, stacklevel=2)
        from service.web_services import common

        log = logging.getLogger(tools.ustr(name))

        if level in [LOG_DEBUG_RPC, LOG_TEST] and not hasattr(log, level):
            fct = lambda msg, *args, **kwargs: log.log(getattr(logging, level.upper()), msg, *args, **kwargs)
            setattr(log, level, fct)

        level_method = getattr(log, level)

        if isinstance(msg, Exception):
            msg = tools.exception_to_unicode(msg)

        try:
            msg = tools.ustr(msg).strip()
            if level in (LOG_ERROR, LOG_CRITICAL) and tools.config.get_misc('debug','env_info',False):
                msg = common().exp_get_server_environment() + "\n" + msg

            result = msg.split('\n')
        except UnicodeDecodeError:
            result = msg.strip().split('\n')
        try:
            if len(result)>1:
                for idx, s in enumerate(result):
                    level_method('[%02d]: %s' % (idx+1, s,))
            elif result:
                level_method(result[0])
        except IOError,e:
            # TODO: perhaps reset the logger streams?
            #if logrotate closes our files, we end up here..
            pass
        except:
            # better ignore the exception and carry on..
            pass

    def set_loglevel(self, level):
        log = logging.getLogger()
        log.setLevel(logging.INFO) # make sure next msg is printed
        log.info("Log level changed to %s" % logging.getLevelName(level))
        log.setLevel(level)

    def shutdown(self):
        logging.shutdown()

import tools
init_logger()

class Agent(object):
    _timers = {}
    _logger = Logger()

    __logger = logging.getLogger('timer')

    def setAlarm(self, fn, dt, db_name, *args, **kwargs):
        wait = dt - time.time()
        if wait > 0:
            self.__logger.debug("Job scheduled in %.3g seconds for %s.%s" % (wait, fn.im_class.__name__, fn.func_name))
            timer = threading.Timer(wait, fn, args, kwargs)
            timer.start()
            self._timers.setdefault(db_name, []).append(timer)

        for db in self._timers:
            for timer in self._timers[db]:
                if not timer.isAlive():
                    self._timers[db].remove(timer)

    @classmethod
    def cancel(cls, db_name):
        """Cancel all timers for a given database. If None passed, all timers are cancelled"""
        for db in cls._timers:
            if db_name is None or db == db_name:
                for timer in cls._timers[db]:
                    timer.cancel()

    @classmethod
    def quit(cls):
        cls.cancel(None)

import traceback

class Server:
    """ Generic interface for all servers with an event loop etc.
        Override this to impement http, net-rpc etc. servers.

        Servers here must have threaded behaviour. start() must not block,
        there is no run().
    """
    __is_started = False
    __servers = []
    __starter_threads = []

    # we don't want blocking server calls (think select()) to
    # wait forever and possibly prevent exiting the process,
    # but instead we want a form of polling/busy_wait pattern, where
    # _server_timeout should be used as the default timeout for
    # all I/O blocking operations
    _busywait_timeout = 0.5

    __logger = logging.getLogger('server')

    def __init__(self):
        Server.__servers.append(self)
        if Server.__is_started:
            # raise Exception('All instances of servers must be inited before the startAll()')
            # Since the startAll() won't be called again, allow this server to
            # init and then start it after 1sec (hopefully). Register that
            # timer thread in a list, so that we can abort the start if quitAll
            # is called in the meantime
            t = threading.Timer(1.0, self._late_start)
            t.name = 'Late start timer for %s' % str(self.__class__)
            Server.__starter_threads.append(t)
            t.start()

    def start(self):
        self.__logger.debug("called stub Server.start")

    def _late_start(self):
        self.start()
        for thr in Server.__starter_threads:
            if thr.finished.is_set():
                Server.__starter_threads.remove(thr)

    def stop(self):
        self.__logger.debug("called stub Server.stop")

    def stats(self):
        """ This function should return statistics about the server """
        return "%s: No statistics" % str(self.__class__)

    @classmethod
    def startAll(cls):
        if cls.__is_started:
            return
        cls.__logger.info("Starting %d services" % len(cls.__servers))
        for srv in cls.__servers:
            srv.start()
        cls.__is_started = True

    @classmethod
    def quitAll(cls):
        if not cls.__is_started:
            return
        cls.__logger.info("Stopping %d services" % len(cls.__servers))
        for thr in cls.__starter_threads:
            if not thr.finished.is_set():
                thr.cancel()
            cls.__starter_threads.remove(thr)

        for srv in cls.__servers:
            srv.stop()
        cls.__is_started = False

    @classmethod
    def allStats(cls):
        res = ["Servers %s" % ('stopped', 'started')[cls.__is_started]]
        res.extend(srv.stats() for srv in cls.__servers)
        return '\n'.join(res)

    def _close_socket(self):
        if os.name != 'nt':
            try:
                self.socket.shutdown(getattr(socket, 'SHUT_RDWR', 2))
            except socket.error, e:
                if e.errno != errno.ENOTCONN: raise
                # OSX, socket shutdowns both sides if any side closes it
                # causing an error 57 'Socket is not connected' on shutdown
                # of the other side (or something), see
                # http://bugs.python.org/issue4397
                self.__logger.debug(
                    '"%s" when shutting down server socket, '
                    'this is normal under OS X', e)
        self.socket.close()

class OpenERPDispatcherException(Exception):
    def __init__(self, exception, traceback):
        self.exception = exception
        self.traceback = traceback

class OpenERPDispatcher:
    def log(self, title, msg):
        if tools.config['log_level'] == logging.DEBUG_RPC:
            Logger().notifyChannel('%s' % title, LOG_DEBUG_RPC, pformat(msg))

    def dispatch(self, service_name, method, params):
        try:
            self.log('service', service_name)
            self.log('method', method)
            self.log('params', params)
            auth = getattr(self, 'auth_provider', None)
            result = ExportService.getService(service_name).dispatch(method, auth, params)
            self.log('result', result)
            # We shouldn't marshall None,
            if result == None:
                result = False
            return result
        except Exception, e:
            self.log('exception', tools.exception_to_unicode(e))
            tb = getattr(e, 'traceback', sys.exc_info())
            tb_s = "".join(traceback.format_exception(*tb))
            if tools.config['debug_mode']:
                import pdb
                pdb.post_mortem(tb[2])
            raise OpenERPDispatcherException(e, tb_s)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2020, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import torch
import torch.nn.functional as F
from torch import nn
from torch.nn.modules.utils import _pair as pair
from torch.nn.parameter import Parameter

class MaskedLinear(nn.Module):
    """
    Masked weights remain at zero because their gradient is always zero.

    This is designed to be used as a finetuning layer with a fixed mask. It
    doesn't initialize its own weights, bias, or mask. The caller needs to
    provide this initialization, e.g. by loading a checkpoint generated by
    another dynamic mask class.

    If the caller prunes this class's weights dynamically during training, the
    caller needs to zero the optimizer's momentum for this weight.
    """
    def __init__(self, in_features, out_features, bias=True):
        super().__init__()
        self.in_features = in_features
        self.out_features = out_features

        self.weight = Parameter(torch.Tensor(out_features, in_features))
        self.register_buffer("weight_mask",
                             torch.Tensor(out_features, in_features))

        if bias:
            self.bias = Parameter(torch.Tensor(out_features))
        else:
            self.bias = None

    def extra_repr(self):
        s = f"{self.in_features}, {self.out_features}"
        if self.bias is None:
            s += ", bias=False"
        return s

    def forward(self, x):
        if self.training:
            w = self.weight * self.weight_mask
        else:
            w = self.weight

        return F.linear(x, w, self.bias)

class MaskedConv2d(nn.Module):
    """
    Masked weights remain at zero because their gradient is always zero.

    This is designed to be used as a finetuning layer with a fixed mask. It
    doesn't initialize its own weights, bias, or mask. The caller needs to
    provide this initialization, e.g. by loading a checkpoint generated by
    another dynamic mask class.

    If the caller prunes this class's weights dynamically during training, the
    caller needs to zero the optimizer's momentum for this weight.
    """
    def __init__(self, in_channels, out_channels, kernel_size, stride=1,
                 padding=0, dilation=1, groups=1, bias=True,
                 mask_mode="channel_to_channel"):
        """
        @param mask_mode (string)
        Determines how large the weight mask tensor needs to be.
        """
        super().__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.kernel_size = pair(kernel_size)
        self.stride = pair(stride)
        self.padding = pair(padding)
        self.dilation = pair(dilation)
        self.groups = groups

        self.weight = Parameter(torch.Tensor(out_channels, in_channels,
                                             *self.kernel_size))

        if mask_mode == "channel_to_channel":
            weight_mask = torch.Tensor(out_channels, in_channels, 1, 1)
        elif mask_mode == "weight_to_weight":
            weight_mask = torch.Tensor(out_channels, in_channels,
                                       *self.kernel_size)
        else:
            raise ValueError(f"Unrecognized mask_mode: {mask_mode}")
        self.register_buffer("weight_mask", weight_mask)

        if bias:
            self.bias = Parameter(torch.Tensor(out_channels))
        else:
            self.bias = None

    def extra_repr(self):
        s = (f"{self.in_channels}, {self.out_channels}, "
             f"kernel_size={self.kernel_size}, stride={self.stride}")
        if self.padding != (0,) * len(self.padding):
            s += f", padding={self.padding}"
        if self.dilation != (1,) * len(self.dilation):
            s += f", dilation={self.dilation}"
        if self.groups != 1:
            s += f", groups={self.groups}"
        if self.bias is None:
            s += ", bias=False"
        return s

    def forward(self, x):
        if self.training:
            w = self.weight * self.weight_mask
        else:
            w = self.weight

        return F.conv2d(
            x, w, self.bias, self.stride, self.padding, self.dilation,
            self.groups
        )

__all__ = [
    "MaskedLinear",
    "MaskedConv2d",
]
<commit_msg>Add maskedweight initialization functions<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2020, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import numpy as np
import torch
import torch.nn.functional as F
import torch.nn.init
from torch import nn
from torch.nn.modules.utils import _pair as pair
from torch.nn.parameter import Parameter

class MaskedLinear(nn.Module):
    """
    Masked weights remain at zero because their gradient is always zero.

    This is designed to be used as a finetuning layer with a fixed mask. It
    doesn't initialize its own weights, bias, or mask. The caller needs to
    provide this initialization, e.g. by loading a checkpoint generated by
    another dynamic mask class.

    If the caller prunes this class's weights dynamically during training, the
    caller needs to zero the optimizer's momentum for this weight.
    """
    def __init__(self, in_features, out_features, bias=True):
        super().__init__()
        self.in_features = in_features
        self.out_features = out_features

        self.weight = Parameter(torch.Tensor(out_features, in_features))
        self.register_buffer("weight_mask",
                             torch.Tensor(out_features, in_features))

        if bias:
            self.bias = Parameter(torch.Tensor(out_features))
        else:
            self.bias = None

    def extra_repr(self):
        s = f"{self.in_features}, {self.out_features}"
        if self.bias is None:
            s += ", bias=False"
        return s

    def forward(self, x):
        if self.training:
            w = self.weight * self.weight_mask
        else:
            w = self.weight

        return F.linear(x, w, self.bias)

class MaskedConv2d(nn.Module):
    """
    Masked weights remain at zero because their gradient is always zero.

    This is designed to be used as a finetuning layer with a fixed mask. It
    doesn't initialize its own weights, bias, or mask. The caller needs to
    provide this initialization, e.g. by loading a checkpoint generated by
    another dynamic mask class.

    If the caller prunes this class's weights dynamically during training, the
    caller needs to zero the optimizer's momentum for this weight.
    """
    def __init__(self, in_channels, out_channels, kernel_size, stride=1,
                 padding=0, dilation=1, groups=1, bias=True,
                 mask_mode="channel_to_channel"):
        """
        @param mask_mode (string)
        Determines how large the weight mask tensor needs to be.
        """
        super().__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.kernel_size = pair(kernel_size)
        self.stride = pair(stride)
        self.padding = pair(padding)
        self.dilation = pair(dilation)
        self.groups = groups

        self.weight = Parameter(torch.Tensor(out_channels, in_channels,
                                             *self.kernel_size))

        if mask_mode == "channel_to_channel":
            weight_mask = torch.Tensor(out_channels, in_channels, 1, 1)
        elif mask_mode == "weight_to_weight":
            weight_mask = torch.Tensor(out_channels, in_channels,
                                       *self.kernel_size)
        else:
            raise ValueError(f"Unrecognized mask_mode: {mask_mode}")
        self.register_buffer("weight_mask", weight_mask)

        if bias:
            self.bias = Parameter(torch.Tensor(out_channels))
        else:
            self.bias = None

    def extra_repr(self):
        s = (f"{self.in_channels}, {self.out_channels}, "
             f"kernel_size={self.kernel_size}, stride={self.stride}")
        if self.padding != (0,) * len(self.padding):
            s += f", padding={self.padding}"
        if self.dilation != (1,) * len(self.dilation):
            s += f", dilation={self.dilation}"
        if self.groups != 1:
            s += f", groups={self.groups}"
        if self.bias is None:
            s += ", bias=False"
        return s

    def forward(self, x):
        if self.training:
            w = self.weight * self.weight_mask
        else:
            w = self.weight

        return F.conv2d(
            x, w, self.bias, self.stride, self.padding, self.dilation,
            self.groups
        )

def maskedlinear_init(module, density):
    """
    Assign the same weight density for each out feature.
    """
    torch.nn.init.kaiming_normal_(module.weight, mode="fan_out")

    if density == 1:
        module.weight_mask[:] = 1
    else:
        # Generate random connectivity.
        on_features_per_out_feature = int(round(density * module.in_features))
        mask = torch.zeros(module.out_features, module.in_features,
                           dtype=torch.bool)
        for out_feature in range(module.out_features):
            in_features = np.random.choice(module.in_features,
                                           on_features_per_out_feature)
            mask[out_feature, in_features] = True
        module.weight_mask[:] = mask
        module.weight.data *= module.weight_mask

def maskedconv2d_init(module, density):
    """
    Assign the same weight density for each out channel.

    This uses channel to channel density. It assumes
    mask_mode="channel_to_channel".
    """
    torch.nn.init.kaiming_normal_(module.weight, mode="fan_out")

    if density == 1:
        module.weight_mask[:] = 1
    else:
        # Generate random connectivity.
        on_channels_per_out_channel = int(round(density * module.in_channels))
        mask = torch.zeros(module.out_channels, module.in_channels,
                           dtype=torch.bool)
        for out_channel in range(module.out_channels):
            in_channels = np.random.choice(module.in_channels,
                                           on_channels_per_out_channel)
            mask[out_channel, in_channels] = True

        # Expand the mask to shape (out_channels, in_channels, 1, 1) so that it
        # can be multiplied with the weight tensor.
        module.weight_mask[:] = mask.view(
            module.out_channels, module.in_channels, 1, 1
        ).float()
        module.weight.data *= module.weight_mask

__all__ = [
    "MaskedLinear",
    "MaskedConv2d",
    "maskedlinear_init",
    "maskedconv2d_init",
]
<|endoftext|>


<commit_before>import tempfile
from collections import OrderedDict

from memoized import memoized

from couchexport.models import Format
from couchexport.writers import Excel2007ExportWriter
from dimagi.utils.couch.loosechange import map_reduce
from django.db.models import Q
from soil import DownloadBase
from soil.util import expose_blob_download

from corehq.apps.commtrack.dbaccessors import (
    get_supply_point_ids_in_domain_by_location,
)
from corehq.apps.consumption.shortcuts import (
    build_consumption_dict,
    get_loaded_default_monthly_consumption,
)
from corehq.apps.domain.models import Domain
from corehq.apps.locations.const import (
    LOCATION_SHEET_HEADERS_BASE,
    LOCATION_SHEET_HEADERS_OPTIONAL,
    LOCATION_TYPE_SHEET_HEADERS,
)
from corehq.apps.locations.models import LocationType, SQLLocation
from corehq.apps.products.models import Product
from corehq.blobs import CODES, get_blob_db
from corehq.form_processor.interfaces.supply import SupplyInterface
from corehq.util.files import safe_filename_header

def load_locs_json(domain, selected_loc_id=None, include_archived=False,
        user=None, only_administrative=False):
    """initialize a json location tree for drill-down controls on
    the client. tree is only partially initialized and branches
    will be filled in on the client via ajax.

    what is initialized:
    * all top level locs
    * if a 'selected' loc is provided, that loc and its complete
      ancestry

    only_administrative - if False get all locations
                          if True get only administrative locations
    """
    def loc_to_json(loc):
        return {
            'name': loc.name,
            'location_type': loc.location_type.name,  # todo: remove when types aren't optional
            'uuid': loc.location_id,
            'is_archived': loc.is_archived,
            'can_edit': True
        }

    locations = SQLLocation.root_locations(
        domain, include_archive_ancestors=include_archived
    )

    if only_administrative:
        locations = locations.filter(location_type__administrative=True)

    loc_json = [loc_to_json(loc) for loc in locations]

    # if a location is selected, we need to pre-populate its location hierarchy
    # so that the data is available client-side to pre-populate the drop-downs
    if selected_loc_id:
        selected = SQLLocation.objects.get(
            domain=domain,
            location_id=selected_loc_id
        )

        lineage = selected.get_ancestors()

        parent = {'children': loc_json}
        for loc in lineage:
            children = loc.child_locations(include_archive_ancestors=include_archived)
            if only_administrative:
                children = children.filter(location_type__administrative=True)

            # find existing entry in the json tree that corresponds to this loc
            try:
                this_loc = [k for k in parent['children'] if k['uuid'] == loc.location_id][0]
            except IndexError:
                # if we couldn't find this location the view just break out of the loop.
                # there are some instances in viewing archived locations where we don't actually
                # support drilling all the way down.
                break
            this_loc['children'] = [loc_to_json(loc) for loc in children]
            parent = this_loc

    return loc_json

def location_hierarchy_config(domain):
    return [(loc_type.name, [loc_type.parent_type.name
                             if loc_type.parent_type else None])
            for loc_type in Domain.get_by_name(domain).location_types]

def parent_child(domain):
    """
    Returns a dict mapping from a location type to its possible
    child types
    """
    return map_reduce(lambda k_v: [(p, k_v[0]) for p in k_v[1]],
                      data=dict(location_hierarchy_config(domain)).items())

def get_location_data_model(domain):
    from .views import LocationFieldsView
    from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
    return CustomDataFieldsDefinition.get_or_create(
        domain,
        LocationFieldsView.field_type,
    )

class LocationExporter(object):

    def __init__(self, domain, include_consumption=False, root_location_id=None,
                 headers_only=False, async_task=None):
        self.domain = domain
        self.domain_obj = Domain.get_by_name(domain)
        self.include_consumption_flag = include_consumption
        self.root_location_id = root_location_id
        self.headers_only = headers_only
        self.data_model = get_location_data_model(domain)
        self.administrative_types = {}
        self.location_types = LocationType.objects.by_domain(domain)
        self.async_task = async_task
        self._location_count = None
        self._locations_exported = 0

    @property
    @memoized
    def consumption_dict(self):
        return build_consumption_dict(self.domain)

    def _increment_progress(self):
        if self._location_count is None:
            self._location_count = SQLLocation.active_objects.filter(domain=self.domain).count()
            self._progress_update_chunksize = max(10, self._location_count // 100)
        self._locations_exported += 1
        if self._locations_exported % self._progress_update_chunksize == 0:
            DownloadBase.set_progress(self.async_task, self._locations_exported, self._location_count)

    @property
    @memoized
    def include_consumption(self):
        if bool(
            self.include_consumption_flag and
            self.domain_obj.commtrack_settings.individual_consumption_defaults
        ):
            # we'll be needing these, so init 'em:
            self.products = Product.by_domain(self.domain)
            self.product_codes = [p.code for p in self.products]
            self.supply_point_map = get_supply_point_ids_in_domain_by_location(
                self.domain)
            self.administrative_types = {
                lt.name for lt in self.location_types
                if lt.administrative

            }
            return True
        return False

    def get_consumption(self, loc):
        if (
            not self.include_consumption or
            loc.location_type_name in self.administrative_types or
            not self.consumption_dict
        ):
            return {}
        if loc.location_id in self.supply_point_map:
            sp_id = self.supply_point_map[loc.location_id]
        else:
            # this only happens if the supply point case did
            # not already exist
            sp_id = SupplyInterface(self.domain).get_or_create_by_location(loc).location_id
        return {
            p.code: get_loaded_default_monthly_consumption(
                self.consumption_dict,
                self.domain,
                p._id,
                loc.location_type_name,
                sp_id
            ) or ''
            for p in self.products
        }

    def get_headers(self):
        headers = OrderedDict([
            ('types', [list(LOCATION_TYPE_SHEET_HEADERS.values())])
        ])
        for loc_type in self.location_types:
            additional_headers = []
            additional_headers.extend('data: {}'.format(f.slug) for f in self.data_model.fields)
            if self.include_consumption_flag and loc_type.name not in self.administrative_types:
                additional_headers.extend('consumption: {}'.format(code) for code in self.product_codes)
            additional_headers.append(LOCATION_SHEET_HEADERS_OPTIONAL['uncategorized_data'])
            additional_headers.append(LOCATION_SHEET_HEADERS_OPTIONAL['delete_uncategorized_data'])

            headers[loc_type.code] = [list(LOCATION_SHEET_HEADERS_BASE.values()) + additional_headers]

        return list(headers.items())

    def write_data(self, writer):
        self._write_type_sheet(writer)
        for location_type in self.location_types:
            self._write_locations(writer, location_type)

    def _write_locations(self, writer, location_type):
        include_consumption = self.include_consumption_flag and location_type.name not in self.administrative_types
        if self.headers_only:
            query = SQLLocation.objects.none()
        elif self.root_location_id:
            root_location = SQLLocation.objects.get(location_id=self.root_location_id)
            query = SQLLocation.active_objects.get_descendants(
                Q(domain=self.domain, id=root_location.id)
            ).filter(location_type=location_type)
        else:
            query = SQLLocation.active_objects.filter(
                domain=self.domain,
                location_type=location_type
            )

        def _row_generator(include_consumption=include_consumption):
            for loc in query:
                model_data, uncategorized_data = self.data_model.get_model_and_uncategorized(loc.metadata)
                row_data = {
                    'location_id': loc.location_id,
                    'site_code': loc.site_code,
                    'name': loc.name,
                    'parent_code': loc.parent.site_code if loc.parent else '',
                    'external_id': loc.external_id,
                    'latitude': loc.latitude or '',
                    'longitude': loc.longitude or '',
                    'do_delete': '',
                }
                row = [row_data[attr] for attr in LOCATION_SHEET_HEADERS_BASE.keys()]
                for field in self.data_model.fields:
                    row.append(model_data.get(field.slug, ''))

                if include_consumption:
                    consumption_data = self.get_consumption(loc)
                    row.extend([consumption_data[code] for code in self.product_codes])

                row.append(', '.join('{}: {}'.format(*d) for d in uncategorized_data.items()))

                yield row
                self._increment_progress()

        writer.write([(location_type.code, _row_generator())])

    def _write_type_sheet(self, writer):
        def foreign_code(lt, attr):
            val = getattr(lt, attr, None)
            if val:
                return val.code
            else:
                return None

        def coax_boolean(value):
            if isinstance(value, bool):
                value = 'yes' if value else 'no'
            return value

        rows = []
        for lt in self.location_types:
            rows.append([
                lt.code,
                lt.name,
                foreign_code(lt, 'parent_type'),
                '',  # do_delete
                coax_boolean(lt.shares_cases),
                coax_boolean(lt.view_descendants),
            ])

        writer.write([('types', rows)])

def dump_locations(domain, download_id, include_consumption, headers_only, root_location_id=None, task=None):
    exporter = LocationExporter(domain, include_consumption=include_consumption, root_location_id=root_location_id,
                                headers_only=headers_only, async_task=task)

    fd, path = tempfile.mkstemp()
    writer = Excel2007ExportWriter()
    writer.open(header_table=exporter.get_headers(), file=path)
    with writer:
        exporter.write_data(writer)

    with open(path, 'rb') as file_:
        db = get_blob_db()
        expiry_mins = 60
        db.put(
            file_,
            domain=domain,
            parent_id=domain,
            type_code=CODES.tempfile,
            key=download_id,
            timeout=expiry_mins,
        )

        file_format = Format.from_format(Excel2007ExportWriter.format)
        expose_blob_download(
            download_id,
            expiry=expiry_mins * 60,
            mimetype=file_format.mimetype,
            content_disposition=safe_filename_header('{}_locations'.format(domain), file_format.extension),
            download_id=download_id,
        )

def get_locations_from_ids(location_ids, domain, base_queryset=None):
    """
    Returns the SQLLocations with the given location_ids, ensuring
    that they belong to the given domain. Raises SQLLocation.DoesNotExist
    if any of the locations do not match the given domain or are not
    found.
    """
    if not base_queryset:
        base_queryset = SQLLocation.objects

    location_ids = list(set(location_ids))
    expected_count = len(location_ids)

    locations = base_queryset.filter(
        domain=domain,
        is_archived=False,
        location_id__in=location_ids,
    )
    if len(locations) != expected_count:
        raise SQLLocation.DoesNotExist('One or more of the locations was not found.')
    return locations
<commit_msg>Added root location name to filename<commit_after>import tempfile
from collections import OrderedDict

from memoized import memoized

from couchexport.models import Format
from couchexport.writers import Excel2007ExportWriter
from dimagi.utils.couch.loosechange import map_reduce
from django.db.models import Q
from soil import DownloadBase
from soil.util import expose_blob_download

from corehq.apps.commtrack.dbaccessors import (
    get_supply_point_ids_in_domain_by_location,
)
from corehq.apps.consumption.shortcuts import (
    build_consumption_dict,
    get_loaded_default_monthly_consumption,
)
from corehq.apps.domain.models import Domain
from corehq.apps.locations.const import (
    LOCATION_SHEET_HEADERS_BASE,
    LOCATION_SHEET_HEADERS_OPTIONAL,
    LOCATION_TYPE_SHEET_HEADERS,
)
from corehq.apps.locations.models import LocationType, SQLLocation
from corehq.apps.products.models import Product
from corehq.blobs import CODES, get_blob_db
from corehq.form_processor.interfaces.supply import SupplyInterface
from corehq.util.files import safe_filename_header

def load_locs_json(domain, selected_loc_id=None, include_archived=False,
        user=None, only_administrative=False):
    """initialize a json location tree for drill-down controls on
    the client. tree is only partially initialized and branches
    will be filled in on the client via ajax.

    what is initialized:
    * all top level locs
    * if a 'selected' loc is provided, that loc and its complete
      ancestry

    only_administrative - if False get all locations
                          if True get only administrative locations
    """
    def loc_to_json(loc):
        return {
            'name': loc.name,
            'location_type': loc.location_type.name,  # todo: remove when types aren't optional
            'uuid': loc.location_id,
            'is_archived': loc.is_archived,
            'can_edit': True
        }

    locations = SQLLocation.root_locations(
        domain, include_archive_ancestors=include_archived
    )

    if only_administrative:
        locations = locations.filter(location_type__administrative=True)

    loc_json = [loc_to_json(loc) for loc in locations]

    # if a location is selected, we need to pre-populate its location hierarchy
    # so that the data is available client-side to pre-populate the drop-downs
    if selected_loc_id:
        selected = SQLLocation.objects.get(
            domain=domain,
            location_id=selected_loc_id
        )

        lineage = selected.get_ancestors()

        parent = {'children': loc_json}
        for loc in lineage:
            children = loc.child_locations(include_archive_ancestors=include_archived)
            if only_administrative:
                children = children.filter(location_type__administrative=True)

            # find existing entry in the json tree that corresponds to this loc
            try:
                this_loc = [k for k in parent['children'] if k['uuid'] == loc.location_id][0]
            except IndexError:
                # if we couldn't find this location the view just break out of the loop.
                # there are some instances in viewing archived locations where we don't actually
                # support drilling all the way down.
                break
            this_loc['children'] = [loc_to_json(loc) for loc in children]
            parent = this_loc

    return loc_json

def location_hierarchy_config(domain):
    return [(loc_type.name, [loc_type.parent_type.name
                             if loc_type.parent_type else None])
            for loc_type in Domain.get_by_name(domain).location_types]

def parent_child(domain):
    """
    Returns a dict mapping from a location type to its possible
    child types
    """
    return map_reduce(lambda k_v: [(p, k_v[0]) for p in k_v[1]],
                      data=dict(location_hierarchy_config(domain)).items())

def get_location_data_model(domain):
    from .views import LocationFieldsView
    from corehq.apps.custom_data_fields.models import CustomDataFieldsDefinition
    return CustomDataFieldsDefinition.get_or_create(
        domain,
        LocationFieldsView.field_type,
    )

class LocationExporter(object):

    def __init__(self, domain, include_consumption=False, root_location_id=None,
                 headers_only=False, async_task=None):
        self.domain = domain
        self.domain_obj = Domain.get_by_name(domain)
        self.include_consumption_flag = include_consumption
        self.root_location_id = root_location_id
        self.headers_only = headers_only
        self.data_model = get_location_data_model(domain)
        self.administrative_types = {}
        self.location_types = LocationType.objects.by_domain(domain)
        self.async_task = async_task
        self._location_count = None
        self._locations_exported = 0

    @property
    @memoized
    def consumption_dict(self):
        return build_consumption_dict(self.domain)

    def _increment_progress(self):
        if self._location_count is None:
            self._location_count = SQLLocation.active_objects.filter(domain=self.domain).count()
            self._progress_update_chunksize = max(10, self._location_count // 100)
        self._locations_exported += 1
        if self._locations_exported % self._progress_update_chunksize == 0:
            DownloadBase.set_progress(self.async_task, self._locations_exported, self._location_count)

    @property
    @memoized
    def include_consumption(self):
        if bool(
            self.include_consumption_flag and
            self.domain_obj.commtrack_settings.individual_consumption_defaults
        ):
            # we'll be needing these, so init 'em:
            self.products = Product.by_domain(self.domain)
            self.product_codes = [p.code for p in self.products]
            self.supply_point_map = get_supply_point_ids_in_domain_by_location(
                self.domain)
            self.administrative_types = {
                lt.name for lt in self.location_types
                if lt.administrative

            }
            return True
        return False

    def get_consumption(self, loc):
        if (
            not self.include_consumption or
            loc.location_type_name in self.administrative_types or
            not self.consumption_dict
        ):
            return {}
        if loc.location_id in self.supply_point_map:
            sp_id = self.supply_point_map[loc.location_id]
        else:
            # this only happens if the supply point case did
            # not already exist
            sp_id = SupplyInterface(self.domain).get_or_create_by_location(loc).location_id
        return {
            p.code: get_loaded_default_monthly_consumption(
                self.consumption_dict,
                self.domain,
                p._id,
                loc.location_type_name,
                sp_id
            ) or ''
            for p in self.products
        }

    def get_headers(self):
        headers = OrderedDict([
            ('types', [list(LOCATION_TYPE_SHEET_HEADERS.values())])
        ])
        for loc_type in self.location_types:
            additional_headers = []
            additional_headers.extend('data: {}'.format(f.slug) for f in self.data_model.fields)
            if self.include_consumption_flag and loc_type.name not in self.administrative_types:
                additional_headers.extend('consumption: {}'.format(code) for code in self.product_codes)
            additional_headers.append(LOCATION_SHEET_HEADERS_OPTIONAL['uncategorized_data'])
            additional_headers.append(LOCATION_SHEET_HEADERS_OPTIONAL['delete_uncategorized_data'])

            headers[loc_type.code] = [list(LOCATION_SHEET_HEADERS_BASE.values()) + additional_headers]

        return list(headers.items())

    def write_data(self, writer):
        self._write_type_sheet(writer)
        for location_type in self.location_types:
            self._write_locations(writer, location_type)

    def _write_locations(self, writer, location_type):
        include_consumption = self.include_consumption_flag and location_type.name not in self.administrative_types
        if self.headers_only:
            query = SQLLocation.objects.none()
        elif self.root_location_id:
            root_location = SQLLocation.objects.get(location_id=self.root_location_id)
            query = SQLLocation.active_objects.get_descendants(
                Q(domain=self.domain, id=root_location.id)
            ).filter(location_type=location_type)
        else:
            query = SQLLocation.active_objects.filter(
                domain=self.domain,
                location_type=location_type
            )

        def _row_generator(include_consumption=include_consumption):
            for loc in query:
                model_data, uncategorized_data = self.data_model.get_model_and_uncategorized(loc.metadata)
                row_data = {
                    'location_id': loc.location_id,
                    'site_code': loc.site_code,
                    'name': loc.name,
                    'parent_code': loc.parent.site_code if loc.parent else '',
                    'external_id': loc.external_id,
                    'latitude': loc.latitude or '',
                    'longitude': loc.longitude or '',
                    'do_delete': '',
                }
                row = [row_data[attr] for attr in LOCATION_SHEET_HEADERS_BASE.keys()]
                for field in self.data_model.fields:
                    row.append(model_data.get(field.slug, ''))

                if include_consumption:
                    consumption_data = self.get_consumption(loc)
                    row.extend([consumption_data[code] for code in self.product_codes])

                row.append(', '.join('{}: {}'.format(*d) for d in uncategorized_data.items()))

                yield row
                self._increment_progress()

        writer.write([(location_type.code, _row_generator())])

    def _write_type_sheet(self, writer):
        def foreign_code(lt, attr):
            val = getattr(lt, attr, None)
            if val:
                return val.code
            else:
                return None

        def coax_boolean(value):
            if isinstance(value, bool):
                value = 'yes' if value else 'no'
            return value

        rows = []
        for lt in self.location_types:
            rows.append([
                lt.code,
                lt.name,
                foreign_code(lt, 'parent_type'),
                '',  # do_delete
                coax_boolean(lt.shares_cases),
                coax_boolean(lt.view_descendants),
            ])

        writer.write([('types', rows)])

def dump_locations(domain, download_id, include_consumption, headers_only, root_location_id=None, task=None):
    exporter = LocationExporter(domain, include_consumption=include_consumption, root_location_id=root_location_id,
                                headers_only=headers_only, async_task=task)

    fd, path = tempfile.mkstemp()
    writer = Excel2007ExportWriter()
    writer.open(header_table=exporter.get_headers(), file=path)
    with writer:
        exporter.write_data(writer)

    with open(path, 'rb') as file_:
        db = get_blob_db()
        expiry_mins = 60
        db.put(
            file_,
            domain=domain,
            parent_id=domain,
            type_code=CODES.tempfile,
            key=download_id,
            timeout=expiry_mins,
        )

        file_format = Format.from_format(Excel2007ExportWriter.format)
        filename = '{}_locations'.format(domain)
        if root_location_id:
            root_location = SQLLocation.objects.get(location_id=root_location_id)
            filename += '_{}'.format(root_location.name)
        expose_blob_download(
            download_id,
            expiry=expiry_mins * 60,
            mimetype=file_format.mimetype,
            content_disposition=safe_filename_header(filename, file_format.extension),
            download_id=download_id,
        )

def get_locations_from_ids(location_ids, domain, base_queryset=None):
    """
    Returns the SQLLocations with the given location_ids, ensuring
    that they belong to the given domain. Raises SQLLocation.DoesNotExist
    if any of the locations do not match the given domain or are not
    found.
    """
    if not base_queryset:
        base_queryset = SQLLocation.objects

    location_ids = list(set(location_ids))
    expected_count = len(location_ids)

    locations = base_queryset.filter(
        domain=domain,
        is_archived=False,
        location_id__in=location_ids,
    )
    if len(locations) != expected_count:
        raise SQLLocation.DoesNotExist('One or more of the locations was not found.')
    return locations
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright 2011 The Native Client Authors.  All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.

"""Do all the steps required to build and test against nacl."""

import optparse
import os.path
import subprocess
import sys

import chromebinaries

def FindChrome(src_dir, options):
  if options.browser_path:
    return options.browser_path

  # List of places that chrome could live.
  # In theory we should be more careful about what platform we're actually
  # building for.
  # As currently constructed, this will also hork people who have debug and
  # release builds sitting side by side who build locally.
  mode = options.mode
  chrome_locations = [
      'build/%s/chrome.exe' % mode,
      'chrome/%s/chrome.exe' % mode,
      'out/%s/chrome' % mode,
      'xcodebuild/%s/Chromium.app/Contents/MacOS/Chromium' % mode,
      'xcodebuild/%s/Chrome.app/Contents/MacOS/Chrome' % mode,
  ]

  # Pick the first one we find.
  for chrome in chrome_locations:
    chrome_filename = os.path.join(src_dir, chrome)
    if os.path.exists(chrome_filename):
      return chrome_filename
  raise Exception('Cannot find a chome binary - specify one with '
                  '--browser_path?')

def BuildAndTest(options):
  # Refuse to run under cygwin.
  if sys.platform == 'cygwin':
    raise Exception('I do not work under cygwin, sorry.')

  script_dir = os.path.dirname(os.path.abspath(__file__))
  nacl_dir = os.path.dirname(script_dir)
  src_dir = os.path.dirname(nacl_dir)

  # Load the nacl DEPS file.
  # TODO(ncbray): move this out of chromebinaries into a common library?
  deps = chromebinaries.EvalDepsFile(os.path.join(nacl_dir, 'DEPS'))

  # Get out the toolchain revs.
  x86_rev = deps['vars']['x86_toolchain_version']
  arm_rev = deps['vars']['arm_toolchain_version']

  # Download the toolchain.
  subprocess.check_call([sys.executable,
                        os.path.join(script_dir, 'download_toolchains.py'),
                        '--x86-version', x86_rev,
                        '--arm-version', arm_rev])

  # Decide platform specifics.
  env = dict(os.environ)
  if sys.platform in ['win32', 'cygwin']:
    shell = True
    if options.bits == 64:
      bits = 64
    elif options.bits == 32:
      bits = 32
    elif (os.environ.get('PROCESSOR_ARCHITECTURE', '').find('64') >= 0 or
          os.environ.get('PROCESSOR_ARCHITEW6432', '').find('64') >= 0):
      bits = 64
    else:
      bits = 32
    tools_bits = {32: 'x86', 64: 'x64'}[bits]
    msvs_path = ';'.join([
        r'c:\Program Files\Microsoft Visual Studio 9.0\VC',
        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\VC',
        r'c:\Program Files\Microsoft Visual Studio 9.0\Common7\Tools',
        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools',
        r'c:\Program Files\Microsoft Visual Studio 8\VC',
        r'c:\Program Files (x86)\Microsoft Visual Studio 8\VC',
        r'c:\Program Files\Microsoft Visual Studio 8\Common7\Tools',
        r'c:\Program Files (x86)\Microsoft Visual Studio 8\Common7\Tools',
    ])
    env['PATH'] += ';' + msvs_path
    scons = ['vcvarsall %s && scons.bat' % tools_bits]
  elif sys.platform == 'darwin':
    bits = 32
    scons = ['./scons']
    shell = False
  else:
    p = subprocess.Popen(
        'uname -m | '
        'sed -e "s/i.86/ia32/;s/x86_64/x64/;s/amd64/x64/;s/arm.*/arm/"',
        shell=True, stdout=subprocess.PIPE)
    (p_stdout, _) = p.communicate()
    assert p.returncode == 0
    if options.bits == 64:
      bits = 64
    elif options.bits == 32:
      bits = 32
    elif p_stdout.find('64') >= 0:
      bits = 64
    else:
      bits = 32
    scons = ['./scons']
    shell = False

  chrome_filename = FindChrome(src_dir, options)

  if options.jobs > 1:
    scons.append('-j%d' % options.jobs)

  if options.partial_sdk:
    # Build the partial sdk
    cmd = scons + ['platform=x86-%d' % bits, '--mode=nacl_extra_sdk',
                   'extra_sdk_update_header', 'install_libpthread',
                   'extra_sdk_update',
    ]
    subprocess.check_call(cmd, shell=shell, cwd=nacl_dir, env=env)

  # Run nacl/chrome integration tests.
  cmd = scons + ['-k', 'platform=x86-%d' % bits,
      'disable_dynamic_plugin_loading=1',
      'chrome_browser_path=' + chrome_filename,
      'chrome_browser_tests',
  ]
  subprocess.check_call(cmd, shell=shell, cwd=nacl_dir, env=env)

def MakeCommandLineParser():
  parser = optparse.OptionParser()
  parser.add_option('-m', '--mode', dest='mode', default='Debug',
                    help='Debug/Release mode')
  parser.add_option('--partial_sdk', dest='partial_sdk', default=False,
                    action='store_true', help='Build partial SDK')
  parser.add_option('-j', dest='jobs', default=1, type='int',
                    help='Number of parallel jobs')

  # Not used on the bots, but handy for running the script manually.
  parser.add_option('--bits', dest='bits', action='store',
                    type='int', default=None,
                    help='32/64')
  parser.add_option('--browser_path', dest='browser_path', action='store',
                    type='string', default=None,
                    help='Path to the chrome browser.')
  return parser

def Main():
  parser = MakeCommandLineParser()
  options, args = parser.parse_args()
  if args:
    parser.error('ERROR: invalid argument')
  BuildAndTest(options)

if __name__ == '__main__':
  Main()
<commit_msg>The Chrome integration bot now cleans its build directory.<commit_after>#!/usr/bin/python
# Copyright 2011 The Native Client Authors.  All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.

"""Do all the steps required to build and test against nacl."""

import optparse
import os.path
import shutil
import subprocess
import sys

import chromebinaries

def FindChrome(src_dir, options):
  if options.browser_path:
    return options.browser_path

  # List of places that chrome could live.
  # In theory we should be more careful about what platform we're actually
  # building for.
  # As currently constructed, this will also hork people who have debug and
  # release builds sitting side by side who build locally.
  mode = options.mode
  chrome_locations = [
      'build/%s/chrome.exe' % mode,
      'chrome/%s/chrome.exe' % mode,
      'out/%s/chrome' % mode,
      'xcodebuild/%s/Chromium.app/Contents/MacOS/Chromium' % mode,
      'xcodebuild/%s/Chrome.app/Contents/MacOS/Chrome' % mode,
  ]

  # Pick the first one we find.
  for chrome in chrome_locations:
    chrome_filename = os.path.join(src_dir, chrome)
    if os.path.exists(chrome_filename):
      return chrome_filename
  raise Exception('Cannot find a chome binary - specify one with '
                  '--browser_path?')

def BuildAndTest(options):
  # Refuse to run under cygwin.
  if sys.platform == 'cygwin':
    raise Exception('I do not work under cygwin, sorry.')

  script_dir = os.path.dirname(os.path.abspath(__file__))
  nacl_dir = os.path.dirname(script_dir)
  src_dir = os.path.dirname(nacl_dir)

  # Load the nacl DEPS file.
  # TODO(ncbray): move this out of chromebinaries into a common library?
  deps = chromebinaries.EvalDepsFile(os.path.join(nacl_dir, 'DEPS'))

  # Get out the toolchain revs.
  x86_rev = deps['vars']['x86_toolchain_version']
  arm_rev = deps['vars']['arm_toolchain_version']

  # Download the toolchain.
  subprocess.check_call([sys.executable,
                        os.path.join(script_dir, 'download_toolchains.py'),
                        '--x86-version', x86_rev,
                        '--arm-version', arm_rev])

  # Decide platform specifics.
  env = dict(os.environ)
  if sys.platform in ['win32', 'cygwin']:
    shell = True
    if options.bits == 64:
      bits = 64
    elif options.bits == 32:
      bits = 32
    elif (os.environ.get('PROCESSOR_ARCHITECTURE', '').find('64') >= 0 or
          os.environ.get('PROCESSOR_ARCHITEW6432', '').find('64') >= 0):
      bits = 64
    else:
      bits = 32
    tools_bits = {32: 'x86', 64: 'x64'}[bits]
    msvs_path = ';'.join([
        r'c:\Program Files\Microsoft Visual Studio 9.0\VC',
        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\VC',
        r'c:\Program Files\Microsoft Visual Studio 9.0\Common7\Tools',
        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools',
        r'c:\Program Files\Microsoft Visual Studio 8\VC',
        r'c:\Program Files (x86)\Microsoft Visual Studio 8\VC',
        r'c:\Program Files\Microsoft Visual Studio 8\Common7\Tools',
        r'c:\Program Files (x86)\Microsoft Visual Studio 8\Common7\Tools',
    ])
    env['PATH'] += ';' + msvs_path
    scons = ['vcvarsall %s && scons.bat' % tools_bits]
  elif sys.platform == 'darwin':
    bits = 32
    scons = ['./scons']
    shell = False
  else:
    p = subprocess.Popen(
        'uname -m | '
        'sed -e "s/i.86/ia32/;s/x86_64/x64/;s/amd64/x64/;s/arm.*/arm/"',
        shell=True, stdout=subprocess.PIPE)
    (p_stdout, _) = p.communicate()
    assert p.returncode == 0
    if options.bits == 64:
      bits = 64
    elif options.bits == 32:
      bits = 32
    elif p_stdout.find('64') >= 0:
      bits = 64
    else:
      bits = 32
    scons = ['./scons']
    shell = False

  chrome_filename = FindChrome(src_dir, options)

  if options.jobs > 1:
    scons.append('-j%d' % options.jobs)

  # Clean the output of the previous build.
  # Incremental builds can get wedged in wierd ways, so we're trading speed
  # for reliability.
  shutil.rmtree(os.path.join(nacl_dir, 'scons-out'), True)

  if options.partial_sdk:
    # Build the partial sdk
    cmd = scons + ['platform=x86-%d' % bits, '--mode=nacl_extra_sdk',
                   'extra_sdk_update_header', 'install_libpthread',
                   'extra_sdk_update',
    ]
    subprocess.check_call(cmd, shell=shell, cwd=nacl_dir, env=env)

  # Run nacl/chrome integration tests.
  cmd = scons + ['-k', 'platform=x86-%d' % bits,
      'disable_dynamic_plugin_loading=1',
      'chrome_browser_path=' + chrome_filename,
      'chrome_browser_tests',
  ]
  subprocess.check_call(cmd, shell=shell, cwd=nacl_dir, env=env)

def MakeCommandLineParser():
  parser = optparse.OptionParser()
  parser.add_option('-m', '--mode', dest='mode', default='Debug',
                    help='Debug/Release mode')
  parser.add_option('--partial_sdk', dest='partial_sdk', default=False,
                    action='store_true', help='Build partial SDK')
  parser.add_option('-j', dest='jobs', default=1, type='int',
                    help='Number of parallel jobs')

  # Not used on the bots, but handy for running the script manually.
  parser.add_option('--bits', dest='bits', action='store',
                    type='int', default=None,
                    help='32/64')
  parser.add_option('--browser_path', dest='browser_path', action='store',
                    type='string', default=None,
                    help='Path to the chrome browser.')
  return parser

def Main():
  parser = MakeCommandLineParser()
  options, args = parser.parse_args()
  if args:
    parser.error('ERROR: invalid argument')
  BuildAndTest(options)

if __name__ == '__main__':
  Main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
Generate starter kit downloads and a download page.
"""

import argparse
import glob
import json
import os
import zipfile

ENVIRONMENT_DIR_HELP = "Directory containing precompiled Halite environment " \
                       "executables, each named after their platform. "
VERSION_HELP = "The version string to embed in the downloads page."
IGNORED_EXTENSIONS = [".exe", ".class", ".pyc", ".obj"]
INCLUDED_EXTENSIONS = [".py", ".java", ".cpp", ".hpp", ".cs", ".csproj", ".scala", ".js", ".sh", ".bat", ".toml", ".rs",".go",".txt",".rb", ".kt", ".clj",".jl", ".ml", ".hs"]
INCLUDED_FILES = ["Makefile", "README", "REQUIRE"]
STARTER_KIT_DIR = "../airesources"
DOWNLOAD_DATA = "_data/downloads.json"
PLATFORM_AGNOSTIC = "None"

# Names of generated downloads
# Standard language + platform
OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_{language}_{platform}.zip"

# Platform only
ENVIRONMENT_OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_{platform}.zip"

# All languages + platform
ALL_LANGUAGES_OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_all_{platform}.zip"
SOURCE_FILE = "assets/downloads/Halite2Source.zip"

versions =  {"Python3" : "1.0", "C++" : "1.0", "Java" : "1.0", "CSharp" : "1.0" ,"JavaScript": "0.9-beta",
"ML-StarterBot-Python":"1.0", "Rust" : "1.0", "Scala" : "1.0-beta", "Go" : "1.0-beta", "Ruby" : "1.0-beta" , "Kotlin" : "0.9.0-beta", "Clojure" : "0.9.0-beta", "Julia" : "0.9.0-beta", "OCaml" : "0.9.0-beta", "Haskell" : "0.9.0-beta"  }

def detect_environments(directory):
    """Detect which platform binaries we have."""
    environments = [(PLATFORM_AGNOSTIC, None, None)]
    for filename in os.listdir(directory):
        platform, platform_ext = os.path.splitext(filename)

        if platform == ".DS_Store":
            # Dang it, MacOS
            continue

        print("Detected platform", platform)
        environments.append((platform,
                             os.path.join(directory, filename),
                             "halite" + platform_ext))

    return environments

def scan_directory(full_path):
    """Figure out what the starter kit files in a directory are."""

    included_files = []
    for containing_dir, _, possible_files in os.walk(full_path):
        for filename in possible_files:
            _, ext = os.path.splitext(filename)
            if ext.lower() in INCLUDED_EXTENSIONS or filename in INCLUDED_FILES:
                included_files.append(os.path.join(containing_dir, filename))

    included_files.append(os.path.join(STARTER_KIT_DIR, "README.MD"))
    return included_files

def make_archive(output, environment, base_path, included_files):
    """Create the output ZIP archive."""
    platform, source, target = environment
    # Get rid of duplicates
    included_files = list(set(included_files))
    with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive:
        if source is not None:
            # source is None <=> platform-agnostic archive
            zinfo = zipfile.ZipInfo.from_file(source, target)
            zinfo.compress_type = zipfile.ZIP_DEFLATED
            zinfo.external_attr = 0o0100755 << 16
            with open(source, 'rb') as source_file:
                archive.writestr(zinfo, source_file.read())

        for file in included_files:
            target_path = os.path.relpath(file, base_path)
            archive.write(file, target_path)

def make_source_download():
    included_files = []

    for directory, _, file_list in os.walk("../environment"):
        target_dir = os.path.relpath(directory, "../environment")
        for filename in file_list:
            _, ext = os.path.splitext(filename)
            if ext.lower() in {".cpp", ".c", ".hpp", ".h", ".bat"} or \
                    filename == "Makefile":
                source_path = os.path.join(directory, filename)
                target_path = os.path.normpath(
                    os.path.join("Halite/", target_dir, filename))

                included_files.append((source_path, target_path))

    with zipfile.ZipFile(SOURCE_FILE, "w", zipfile.ZIP_DEFLATED) as archive:
        for source_path, target_path in included_files:
            archive.write(source_path, target_path)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("version", help=VERSION_HELP)
    parser.add_argument("environment_dir", help=ENVIRONMENT_DIR_HELP)

    args = parser.parse_args()

    environments = detect_environments(args.environment_dir)
    generated_languages = []

    # Ensure output directories exists
    for location in [SOURCE_FILE, DOWNLOAD_DATA]:
        try:
            os.makedirs(os.path.dirname(location))
        except FileExistsError:
            pass

    make_source_download()

    # Keep paths of all source files around so we can make a single combined
    # download at the end
    all_files = []

    for directory in os.listdir(STARTER_KIT_DIR):
        full_path = os.path.join(STARTER_KIT_DIR, directory)
        if not os.path.isdir(full_path):
            continue

        if directory == "starterkitdocs":
            continue

        language = directory
        generated_languages.append(language)
        print("Language:", language)

        included_files = scan_directory(full_path)
        for file in included_files:
            print("\tIncluding:", file)

        print()

        all_files.extend(included_files)

        for (platform, source, target) in environments:
            output = "./" + OUTPUT_FILE_FORMAT.format(
                language=language, platform=platform)
            print("\tMaking:", output)
            make_archive(output, (platform, source, target),
                         full_path, included_files)

    panlanguage_kits = []
    for (platform, source, target) in environments:
        # Make downloads including all languages
        filename = ALL_LANGUAGES_OUTPUT_FILE_FORMAT.format(platform=platform)
        all_output = "./" + filename
        print("\tMaking:", all_output)
        make_archive(all_output, (platform, source, target), "../airesources", all_files)
        panlanguage_kits.append(filename)

        # Make downloads including no languages
        if source is None:
            continue
        output = "./" + ENVIRONMENT_OUTPUT_FILE_FORMAT.format(platform=platform)
        print("\tMaking:", output)
        make_archive(output, (platform, source, target), "", [])

    output = {
        "platforms": [environment[0] for environment in environments],
        "languages": [],
        "environments": [],
        "source": SOURCE_FILE,
        "version": args.version,
    }
    generated_languages.sort()
    for language in generated_languages:
        language_kits = []
        for (platform, _, _) in environments:
            language_kits.append(
                OUTPUT_FILE_FORMAT.format(
                    language=language, platform=platform))

        output["languages"].append({
            "language": language,
            "files": language_kits,
            "version": versions[language]
        })

    output["languages"].append({
        "language": "All Languages",
        "files": panlanguage_kits,
    })

    for (platform, source, _) in environments:
        if source is None:
            continue
        output["environments"].append(
            ENVIRONMENT_OUTPUT_FILE_FORMAT.format(platform=platform))

    with open(DOWNLOAD_DATA, "w") as output_file:
        json.dump(output, output_file, indent=2)

if __name__ == "__main__":
    main()
<commit_msg>Add Elixir Support<commit_after>#!/usr/bin/env python3
"""
Generate starter kit downloads and a download page.
"""

import argparse
import glob
import json
import os
import zipfile

ENVIRONMENT_DIR_HELP = "Directory containing precompiled Halite environment " \
                       "executables, each named after their platform. "
VERSION_HELP = "The version string to embed in the downloads page."
IGNORED_EXTENSIONS = [".exe", ".class", ".pyc", ".obj"]
INCLUDED_EXTENSIONS = [".py", ".java", ".cpp", ".hpp", ".cs", ".csproj", ".scala", ".js", ".sh", ".bat", ".toml", ".rs",".go",".txt",".rb", ".kt", ".clj",".jl", ".ml", ".hs", ".exs", ".ex", ".lock"]
INCLUDED_FILES = ["Makefile", "README", "REQUIRE"]
STARTER_KIT_DIR = "../airesources"
DOWNLOAD_DATA = "_data/downloads.json"
PLATFORM_AGNOSTIC = "None"

# Names of generated downloads
# Standard language + platform
OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_{language}_{platform}.zip"

# Platform only
ENVIRONMENT_OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_{platform}.zip"

# All languages + platform
ALL_LANGUAGES_OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_all_{platform}.zip"
SOURCE_FILE = "assets/downloads/Halite2Source.zip"

versions =  {"Python3" : "1.0", "C++" : "1.0", "Java" : "1.0", "CSharp" : "1.0" ,"JavaScript": "0.9-beta",
"ML-StarterBot-Python":"1.0", "Rust" : "1.0", "Scala" : "1.0-beta", "Go" : "1.0-beta", "Ruby" : "1.0-beta" , "Kotlin" : "0.9.0-beta", "Clojure" : "0.9.0-beta", "Julia" : "0.9.0-beta", "OCaml" : "0.9.0-beta", "Haskell" : "0.9.0-beta", "Elixir" : "0.9.0-beta" }

def detect_environments(directory):
    """Detect which platform binaries we have."""
    environments = [(PLATFORM_AGNOSTIC, None, None)]
    for filename in os.listdir(directory):
        platform, platform_ext = os.path.splitext(filename)

        if platform == ".DS_Store":
            # Dang it, MacOS
            continue

        print("Detected platform", platform)
        environments.append((platform,
                             os.path.join(directory, filename),
                             "halite" + platform_ext))

    return environments

def scan_directory(full_path):
    """Figure out what the starter kit files in a directory are."""

    included_files = []
    for containing_dir, _, possible_files in os.walk(full_path):
        for filename in possible_files:
            _, ext = os.path.splitext(filename)
            if ext.lower() in INCLUDED_EXTENSIONS or filename in INCLUDED_FILES:
                included_files.append(os.path.join(containing_dir, filename))

    included_files.append(os.path.join(STARTER_KIT_DIR, "README.MD"))
    return included_files

def make_archive(output, environment, base_path, included_files):
    """Create the output ZIP archive."""
    platform, source, target = environment
    # Get rid of duplicates
    included_files = list(set(included_files))
    with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive:
        if source is not None:
            # source is None <=> platform-agnostic archive
            zinfo = zipfile.ZipInfo.from_file(source, target)
            zinfo.compress_type = zipfile.ZIP_DEFLATED
            zinfo.external_attr = 0o0100755 << 16
            with open(source, 'rb') as source_file:
                archive.writestr(zinfo, source_file.read())

        for file in included_files:
            target_path = os.path.relpath(file, base_path)
            archive.write(file, target_path)

def make_source_download():
    included_files = []

    for directory, _, file_list in os.walk("../environment"):
        target_dir = os.path.relpath(directory, "../environment")
        for filename in file_list:
            _, ext = os.path.splitext(filename)
            if ext.lower() in {".cpp", ".c", ".hpp", ".h", ".bat"} or \
                    filename == "Makefile":
                source_path = os.path.join(directory, filename)
                target_path = os.path.normpath(
                    os.path.join("Halite/", target_dir, filename))

                included_files.append((source_path, target_path))

    with zipfile.ZipFile(SOURCE_FILE, "w", zipfile.ZIP_DEFLATED) as archive:
        for source_path, target_path in included_files:
            archive.write(source_path, target_path)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("version", help=VERSION_HELP)
    parser.add_argument("environment_dir", help=ENVIRONMENT_DIR_HELP)

    args = parser.parse_args()

    environments = detect_environments(args.environment_dir)
    generated_languages = []

    # Ensure output directories exists
    for location in [SOURCE_FILE, DOWNLOAD_DATA]:
        try:
            os.makedirs(os.path.dirname(location))
        except FileExistsError:
            pass

    make_source_download()

    # Keep paths of all source files around so we can make a single combined
    # download at the end
    all_files = []

    for directory in os.listdir(STARTER_KIT_DIR):
        full_path = os.path.join(STARTER_KIT_DIR, directory)
        if not os.path.isdir(full_path):
            continue

        if directory == "starterkitdocs":
            continue

        language = directory
        generated_languages.append(language)
        print("Language:", language)

        included_files = scan_directory(full_path)
        for file in included_files:
            print("\tIncluding:", file)

        print()

        all_files.extend(included_files)

        for (platform, source, target) in environments:
            output = "./" + OUTPUT_FILE_FORMAT.format(
                language=language, platform=platform)
            print("\tMaking:", output)
            make_archive(output, (platform, source, target),
                         full_path, included_files)

    panlanguage_kits = []
    for (platform, source, target) in environments:
        # Make downloads including all languages
        filename = ALL_LANGUAGES_OUTPUT_FILE_FORMAT.format(platform=platform)
        all_output = "./" + filename
        print("\tMaking:", all_output)
        make_archive(all_output, (platform, source, target), "../airesources", all_files)
        panlanguage_kits.append(filename)

        # Make downloads including no languages
        if source is None:
            continue
        output = "./" + ENVIRONMENT_OUTPUT_FILE_FORMAT.format(platform=platform)
        print("\tMaking:", output)
        make_archive(output, (platform, source, target), "", [])

    output = {
        "platforms": [environment[0] for environment in environments],
        "languages": [],
        "environments": [],
        "source": SOURCE_FILE,
        "version": args.version,
    }
    generated_languages.sort()
    for language in generated_languages:
        language_kits = []
        for (platform, _, _) in environments:
            language_kits.append(
                OUTPUT_FILE_FORMAT.format(
                    language=language, platform=platform))

        output["languages"].append({
            "language": language,
            "files": language_kits,
            "version": versions[language]
        })

    output["languages"].append({
        "language": "All Languages",
        "files": panlanguage_kits,
    })

    for (platform, source, _) in environments:
        if source is None:
            continue
        output["environments"].append(
            ENVIRONMENT_OUTPUT_FILE_FORMAT.format(platform=platform))

    with open(DOWNLOAD_DATA, "w") as output_file:
        json.dump(output, output_file, indent=2)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import math
import unittest

from moses import *

class TestMoses(unittest.TestCase):

  def setUp(self):
    self.ttable = PhraseTable("data/esen.ep.model.unfiltered/phrase-table",5,True)

  def test_hw(self):
    self.assertEqual(greet(), "hello, world")

  def test_get_translations(self):
    translations = self.ttable.get_translations("lados")
    self.assertEqual(len(translations),61)
    self.assertEqual(translations[0][1], [0.00015135499415919185, 0.00063730002148076892, 0.0018050499493256211, 0.011290299706161022, 2.7179999351501465])

  def test_get_translation_probability(self):
    fv = self.ttable.get_translation_probability("en ambos", "on both")
    expected_fv = [0.0908379,0.0213197,0.187399,0.0498198,2.718]
    for i in range(5):
      self.assertAlmostEqual(fv[i], expected_fv[i], 4)
    fv = self.ttable.get_translation_probability("kljhdklj!", "hkhdkj")
    for i in range(5):
      self.assertEqual(fv[i], 0)

if __name__ == "__main__":
  unittest.main()

suite = unittest.TestLoader().loadTestsFromTestCase(TestMoses)

<commit_msg>No longer required<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.11 on 2018-03-12 15:51
from __future__ import unicode_literals

from django.db import connection, migrations

def delete_ftl_duplicates(apps, schema):
    """
    In the past we fixed a bug that prevented duplicate FTL translations from
    being stored for the same entity and locale. However, we didn't run a data
    migration to clean up the old duplicates. That's why we're doing it now.

    See bug 1405256 for more details.
    """
    Translation = apps.get_model('base', 'Translation')
    TranslationMemoryEntry = apps.get_model('base', 'TranslationMemoryEntry')

    duplicates = []

    with connection.cursor() as cursor:
        cursor.execute("""
            WITH d AS (
                SELECT
                    entity_id,
                    locale_id,
                    t.string,
                    COUNT(t.id) AS duplicates,
                    ARRAY_AGG(t.id ORDER BY approved DESC, rejected ASC, date DESC) AS ids
                FROM base_translation t
                JOIN base_entity e ON (t.entity_id=e.id)
                JOIN base_resource r ON (e.resource_id=r.id)
                WHERE r.format = 'ftl'
                GROUP BY locale_id, entity_id, t.string
                ORDER BY COUNT(t.id) DESC
            )
            SELECT UNNEST(ids[2:100]) FROM d;
        """)
        # Note: Postgres does not support the [2:] syntax.

        duplicates = [duplicate[0] for duplicate in cursor.fetchall()]

        translations = Translation.objects.filter(pk__in=duplicates)
        tms = TranslationMemoryEntry.objects.filter(translation__pk__in=duplicates)

        translations.delete()
        tms.delete()

class Migration(migrations.Migration):

    dependencies = [
        ('base', '0116_bug_1441020_ftl_0_6_4'),
    ]

    operations = [
        migrations.RunPython(
            delete_ftl_duplicates,
            migrations.RunPython.noop
        )
    ]
<commit_msg>Fix bug 1405256: Fix migration (#895)<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.11 on 2018-03-12 15:51
from __future__ import unicode_literals

from django.db import connection, migrations

def delete_ftl_duplicates(apps, schema):
    """
    In the past we fixed a bug that prevented duplicate FTL translations from
    being stored for the same entity and locale. However, we didn't run a data
    migration to clean up the old duplicates. That's why we're doing it now.

    See bug 1405256 for more details.
    """
    Translation = apps.get_model('base', 'Translation')
    TranslationMemoryEntry = apps.get_model('base', 'TranslationMemoryEntry')

    duplicates = []

    with connection.cursor() as cursor:
        cursor.execute("""
            WITH d AS (
                SELECT
                    entity_id,
                    locale_id,
                    t.string,
                    COUNT(t.id) AS duplicates,
                    ARRAY_AGG(t.id ORDER BY approved DESC, rejected ASC, date DESC) AS ids
                FROM base_translation t
                JOIN base_entity e ON (t.entity_id=e.id)
                JOIN base_resource r ON (e.resource_id=r.id)
                WHERE r.format = 'ftl'
                GROUP BY locale_id, entity_id, t.string
                ORDER BY COUNT(t.id) DESC
            )
            SELECT UNNEST(ids[2:100]) FROM d;
        """)
        # Note: Postgres does not support the [2:] syntax.

        duplicates = [duplicate[0] for duplicate in cursor.fetchall()]

        translations = Translation.objects.filter(pk__in=duplicates)
        tms = TranslationMemoryEntry.objects.filter(translation__pk__in=duplicates)

        # Delete TranslationMemoryEntry instances first, because the tms QuerySet
        # gets empty after the Translation instances are deleted.
        tms.delete()
        translations.delete()

class Migration(migrations.Migration):

    dependencies = [
        ('base', '0116_bug_1441020_ftl_0_6_4'),
    ]

    operations = [
        migrations.RunPython(
            delete_ftl_duplicates,
            migrations.RunPython.noop
        )
    ]
<|endoftext|>