<commit_before>"""
Incoming lambda request objectifying
"""
import logging
from typing import Dict, Optional, Type
from enum import Enum, auto

LOG = logging.getLogger(__name__)
SUPPORTED_SCHEMA_VERSION = '1.0'

class _RequestApplication(object):
    application_id: str
    """A string representing the appliation ID for your skill."""

    def __init__(self, a: dict):
        self.application_id = a['applicationId']

class _RequestUser(object):
    user_id: str
    """
    A string that represents a unique identifier for the user who made the request. The length of this identifier
    can vary, but is never more than 255 characters. The userId is automatically generated when a user enables the
    skill in the Alexa app.  Note: Disabling and re-enabling a skill generates a new identifier."""

    def __init__(self, u: dict):
        self.user_id = u['userId']

class _RequestSession(object):
    """Standard request types (LaunchRequest, IntentRequest, and SessionEndedRequest) include the session object."""

    new: bool
    """
    A boolean value indicating whether this is a new session.
    Returns true for a new session or false for an existing session.
    """

    session_id: str
    """A string that represents a unique identifier per a user’s active session."""

    attributes: Optional[Dict[str, dict]] = None
    """A map of key-value pairs. The attributes map is empty for requests where a
    new session has started with the property new set to true."""

    application: _RequestApplication
    """An object containing an application ID.
    This is used to verify that the request was intended for your service.

    This information is also available in the context.System.application property."""

    user: _RequestUser
    """An object that describes the user making the request."""

    def __init__(self, s: dict):
        """
        :param s: session as dictionary
        """
        self.new = s['new']
        self.session_id = s['sessionId']
        self.attributes = s.get('attributes')
        self.application = _RequestApplication(s['application'])
        self.user = _RequestUser(s['user'])

class _RequestContextSystem(object):
    class Device(object):
        device_id: Optional[str]
        supported_interfaces: Dict[str, Dict]

        def __init__(self, d: dict):
            self.device_id = d.get('deviceId')
            self.supported_interfaces = d.get('supportedInterfaces', {})

    application: _RequestApplication
    user: _RequestUser
    device: Device
    api_endpoint: Optional[dict] = None

    def __init__(self, s: dict):
        self.application = _RequestApplication(s['application'])
        self.user = _RequestUser(s['user'])
        self.device = self.Device(s['device'])
        self.api_endpoint = s.get('apiEndpoint')

class _RequestContext(object):
    class AudioPlayer(object):
        token: Optional[str] = None
        offset_ms: Optional[int] = None
        activity: str

        def __init__(self, p:dict):
            self.token = p.get('token')
            self.offset_ms = p.get('offsetInMilliseconds')
            self.activity = p['playerActivity']

    system: _RequestContextSystem
    audio_player: AudioPlayer

    def __init__(self, c: dict):
        self.system = _RequestContextSystem(c['System'])

        # FireTV requests dont seem to have AudioPlayer
        # But Original Echo and Echo Show Do.
        self.audio_player = self.AudioPlayer(c.get('AudioPlayer', {}))

class _BaseAlexaRequest(object):
    """
    Base class for all Alexa Request Types
    """
    type = None # type: RequestTypes
    locale: str
    request_id: str
    timestamp: str

    def __init__(self, r: dict):
        self.locale = r['locale']
        self.request_id = r['requestId']
        self.timestamp = r['timestamp']

    def __repr__(self):
        return '<{} type={}>'.format(self.__class__.__name__, self.type)

class _AlexaLaunchRequest(_BaseAlexaRequest):
    """A LaunchRequest is an object that represents that a user made a request to an Alexa skill,
    but did not provide a specific intent."""
    def __init__(self, r: dict):
        super().__init__(r)
        self.type = RequestTypes.LaunchRequest

class Slot(object):
    name: str
    has_value: bool
    value: Optional[str]

    def __init__(self, s:dict):
        self.name = s['name']
        self.has_value = 'value' in s
        self.value = s.get('value')

    def __repr__(self):
        return '<{} {}={}>'.format(self.__class__.__name__, self.name, self.value)

    def __str__(self):
        if not self.value:
            if self.has_value:
                return repr(self.value)
            return '<NOT_SET>'
        return self.value

class Intent(object):
    name: str

    slots: Optional[Dict[str, Slot]] = None
    """
    A map of key-value pairs that further describes what the user meant based on a predefined intent schema.
    The map can be empty.
    """

    def __init__(self, i: dict):
        self.name = i['name']
        if 'slots' in i:
            self.slots = {
                k: Slot(v)
                for k, v in i['slots'].items()
            }

    def __repr__(self):
        return '<{} "{}">'.format(self.__class__.__name__, self.name)

    def __str__(self):
        slotstr = ''
        if self.slots:
            slotstr = ', '.join(
                '{}={}'.format(slot_name, slot)
                for slot_name, slot in self.slots.items())
        return '{}({})'.format(self.name, slotstr)

class _AlexaIntentRequest(_BaseAlexaRequest):
    """An IntentRequest is an object that represents a request made to a skill based on what the user wants to do."""

    intent: Intent
    """An object that represents what the user wants"""

    def __init__(self, r: dict):
        super().__init__(r)
        self.type = RequestTypes.IntentRequest
        self.intent = Intent(r['intent'])

    def __repr__(self):
        return '<{} type={} intent={}>'.format(self.__class__.__name__, self.type, self.intent)

    def __str__(self):
        return 'Intent: %s' % self.intent

class _AlexaSessionEndedRequest(_BaseAlexaRequest):
    """
    A SessionEndedRequest is an object that represents a request made to an Alexa skill to notify that a session was
    ended. Your service receives a SessionEndedRequest when a currently open session is closed for one of the
    following reasons:

    - The user says “exit”.
    - The user does not respond or says something that does not match an intent defined in your voice interface
      while the device is listening for the user’s response.
    - An error occurs.
    """

    class SessionEndError(object):
        type: str
        """
        a string indicating the type of error that occurred (INVALID_RESPONSE, DEVICE_COMMUNICATION_ERROR,
        INTERNAL_ERROR).
        """

        msg: Optional[str]
        """a string providing more information about the error. """

        def __init__(self, e: dict):
            self.type = e['type']
            self.msg = e.get('message')

        def __repr__(self):
            return '<{} "{}">'.format(self.__class__.__name__, self.type)

        def __str__(self):
            return '{}: {}'.format(self.type, self.msg)

    reason: str
    """
    Describes why the session ended. Possible values:

    - USER_INITIATED: The user explicitly ended the session.
    - ERROR: An error occurred that caused the session to end.
    - EXCEEDED_MAX_REPROMPTS: The user either did not respond or responded with an utterance that did not match
      any of the intents defined in your voice interface.
    """

    error: Optional[SessionEndError] = None
    """An error object providing more information about the error that occurred."""

    def __init__(self, r: dict):
        super().__init__(r)
        self.type = RequestTypes.SessionEndedRequest
        self.reason = r['reason']
        if 'error' in r:
            self.error = self.SessionEndError(r['error'])
        LOG.debug('SESSION_END reason: "%s", error: %s', self.reason, self.error)

class RequestTypes(Enum):
    LaunchRequest = (_AlexaLaunchRequest,)
    IntentRequest = (_AlexaIntentRequest,)
    SessionEndedRequest = (_AlexaSessionEndedRequest,)

    def __init__(self, cls_type: Type[_BaseAlexaRequest]):
        self.cls_type = cls_type

    def __repr__(self):
        return '<%s.%s>' % (self.__class__.__name__, self.name)

def _build_alexa_request(request: dict) -> _BaseAlexaRequest:
    req_type = RequestTypes[request['type']]
    LOG.debug('Request Type: %s', req_type)
    return req_type.cls_type(request)

class LambdaEvent(object):
    """
    All requests include the version, context, and request objects at the top level.
    The session object is included for all standard requests, but it is not included
    for AudioPlayer, VideoApp, or PlaybackController requests."""

    version: str
    """The version specifier for the request with the value defined as: 1.0"""

    session: Optional[_RequestSession] = None
    """The session object provides additional context associated with the request."""

    context: Optional[_RequestContext] = None
    """
    The context object provides your skill with information about the current state of the Alexa
    service and device at the time the request is sent to your service.
    This is included on all requests. For requests sent in the context of a session
    (LaunchRequest and IntentRequest), the context object duplicates the user and
    application information that is also available in the session."""

    request: _BaseAlexaRequest
    """A request object that provides the details of the user’s request.
    There are several different request types available."""

    def __init__(self, e: dict):
        self.version = e['version']
        LOG.debug('Request version: %s', self.version)
        assert self.version == SUPPORTED_SCHEMA_VERSION
        if e['session']:
            self.session = _RequestSession(e['session'])
        if 'context' in e:
            self.context = _RequestContext(e['context'])
        self.request = _build_alexa_request(e['request'])
<commit_msg>Fix #1: Crashes on FireTV when there's no playerActivity<commit_after>"""
Incoming lambda request objectifying
"""
import logging
from typing import Dict, Optional, Type
from enum import Enum, auto

LOG = logging.getLogger(__name__)
SUPPORTED_SCHEMA_VERSION = '1.0'

class _RequestApplication(object):
    application_id: str
    """A string representing the appliation ID for your skill."""

    def __init__(self, a: dict):
        self.application_id = a['applicationId']

class _RequestUser(object):
    user_id: str
    """
    A string that represents a unique identifier for the user who made the request. The length of this identifier
    can vary, but is never more than 255 characters. The userId is automatically generated when a user enables the
    skill in the Alexa app.  Note: Disabling and re-enabling a skill generates a new identifier."""

    def __init__(self, u: dict):
        self.user_id = u['userId']

class _RequestSession(object):
    """Standard request types (LaunchRequest, IntentRequest, and SessionEndedRequest) include the session object."""

    new: bool
    """
    A boolean value indicating whether this is a new session.
    Returns true for a new session or false for an existing session.
    """

    session_id: str
    """A string that represents a unique identifier per a user’s active session."""

    attributes: Optional[Dict[str, dict]] = None
    """A map of key-value pairs. The attributes map is empty for requests where a
    new session has started with the property new set to true."""

    application: _RequestApplication
    """An object containing an application ID.
    This is used to verify that the request was intended for your service.

    This information is also available in the context.System.application property."""

    user: _RequestUser
    """An object that describes the user making the request."""

    def __init__(self, s: dict):
        """
        :param s: session as dictionary
        """
        self.new = s['new']
        self.session_id = s['sessionId']
        self.attributes = s.get('attributes')
        self.application = _RequestApplication(s['application'])
        self.user = _RequestUser(s['user'])

class _RequestContextSystem(object):
    class Device(object):
        device_id: Optional[str]
        supported_interfaces: Dict[str, Dict]

        def __init__(self, d: dict):
            self.device_id = d.get('deviceId')
            self.supported_interfaces = d.get('supportedInterfaces', {})

    application: _RequestApplication
    user: _RequestUser
    device: Device
    api_endpoint: Optional[dict] = None

    def __init__(self, s: dict):
        self.application = _RequestApplication(s['application'])
        self.user = _RequestUser(s['user'])
        self.device = self.Device(s['device'])
        self.api_endpoint = s.get('apiEndpoint')

class _RequestContext(object):
    class AudioPlayer(object):
        token: Optional[str] = None
        offset_ms: Optional[int] = None
        activity: str

        def __init__(self, p: dict):
            self.token = p.get('token')
            self.offset_ms = p.get('offsetInMilliseconds')
            self.activity = p.get('playerActivity')

    system: _RequestContextSystem
    audio_player: AudioPlayer

    def __init__(self, c: dict):
        self.system = _RequestContextSystem(c['System'])

        # FireTV requests dont seem to have AudioPlayer
        # But Original Echo and Echo Show Do.
        self.audio_player = self.AudioPlayer(c.get('AudioPlayer', {}))

class _BaseAlexaRequest(object):
    """
    Base class for all Alexa Request Types
    """
    type = None # type: RequestTypes
    locale: str
    request_id: str
    timestamp: str

    def __init__(self, r: dict):
        self.locale = r['locale']
        self.request_id = r['requestId']
        self.timestamp = r['timestamp']

    def __repr__(self):
        return '<{} type={}>'.format(self.__class__.__name__, self.type)

class _AlexaLaunchRequest(_BaseAlexaRequest):
    """A LaunchRequest is an object that represents that a user made a request to an Alexa skill,
    but did not provide a specific intent."""
    def __init__(self, r: dict):
        super().__init__(r)
        self.type = RequestTypes.LaunchRequest

class Slot(object):
    name: str
    has_value: bool
    value: Optional[str]

    def __init__(self, s:dict):
        self.name = s['name']
        self.has_value = 'value' in s
        self.value = s.get('value')

    def __repr__(self):
        return '<{} {}={}>'.format(self.__class__.__name__, self.name, self.value)

    def __str__(self):
        if not self.value:
            if self.has_value:
                return repr(self.value)
            return '<NOT_SET>'
        return self.value

class Intent(object):
    name: str

    slots: Optional[Dict[str, Slot]] = None
    """
    A map of key-value pairs that further describes what the user meant based on a predefined intent schema.
    The map can be empty.
    """

    def __init__(self, i: dict):
        self.name = i['name']
        if 'slots' in i:
            self.slots = {
                k: Slot(v)
                for k, v in i['slots'].items()
            }

    def __repr__(self):
        return '<{} "{}">'.format(self.__class__.__name__, self.name)

    def __str__(self):
        slotstr = ''
        if self.slots:
            slotstr = ', '.join(
                '{}={}'.format(slot_name, slot)
                for slot_name, slot in self.slots.items())
        return '{}({})'.format(self.name, slotstr)

class _AlexaIntentRequest(_BaseAlexaRequest):
    """An IntentRequest is an object that represents a request made to a skill based on what the user wants to do."""

    intent: Intent
    """An object that represents what the user wants"""

    def __init__(self, r: dict):
        super().__init__(r)
        self.type = RequestTypes.IntentRequest
        self.intent = Intent(r['intent'])

    def __repr__(self):
        return '<{} type={} intent={}>'.format(self.__class__.__name__, self.type, self.intent)

    def __str__(self):
        return 'Intent: %s' % self.intent

class _AlexaSessionEndedRequest(_BaseAlexaRequest):
    """
    A SessionEndedRequest is an object that represents a request made to an Alexa skill to notify that a session was
    ended. Your service receives a SessionEndedRequest when a currently open session is closed for one of the
    following reasons:

    - The user says “exit”.
    - The user does not respond or says something that does not match an intent defined in your voice interface
      while the device is listening for the user’s response.
    - An error occurs.
    """

    class SessionEndError(object):
        type: str
        """
        a string indicating the type of error that occurred (INVALID_RESPONSE, DEVICE_COMMUNICATION_ERROR,
        INTERNAL_ERROR).
        """

        msg: Optional[str]
        """a string providing more information about the error. """

        def __init__(self, e: dict):
            self.type = e['type']
            self.msg = e.get('message')

        def __repr__(self):
            return '<{} "{}">'.format(self.__class__.__name__, self.type)

        def __str__(self):
            return '{}: {}'.format(self.type, self.msg)

    reason: str
    """
    Describes why the session ended. Possible values:

    - USER_INITIATED: The user explicitly ended the session.
    - ERROR: An error occurred that caused the session to end.
    - EXCEEDED_MAX_REPROMPTS: The user either did not respond or responded with an utterance that did not match
      any of the intents defined in your voice interface.
    """

    error: Optional[SessionEndError] = None
    """An error object providing more information about the error that occurred."""

    def __init__(self, r: dict):
        super().__init__(r)
        self.type = RequestTypes.SessionEndedRequest
        self.reason = r['reason']
        if 'error' in r:
            self.error = self.SessionEndError(r['error'])
        LOG.debug('SESSION_END reason: "%s", error: %s', self.reason, self.error)

class RequestTypes(Enum):
    LaunchRequest = (_AlexaLaunchRequest,)
    IntentRequest = (_AlexaIntentRequest,)
    SessionEndedRequest = (_AlexaSessionEndedRequest,)

    def __init__(self, cls_type: Type[_BaseAlexaRequest]):
        self.cls_type = cls_type

    def __repr__(self):
        return '<%s.%s>' % (self.__class__.__name__, self.name)

def _build_alexa_request(request: dict) -> _BaseAlexaRequest:
    req_type = RequestTypes[request['type']]
    LOG.debug('Request Type: %s', req_type)
    return req_type.cls_type(request)

class LambdaEvent(object):
    """
    All requests include the version, context, and request objects at the top level.
    The session object is included for all standard requests, but it is not included
    for AudioPlayer, VideoApp, or PlaybackController requests."""

    version: str
    """The version specifier for the request with the value defined as: 1.0"""

    session: Optional[_RequestSession] = None
    """The session object provides additional context associated with the request."""

    context: Optional[_RequestContext] = None
    """
    The context object provides your skill with information about the current state of the Alexa
    service and device at the time the request is sent to your service.
    This is included on all requests. For requests sent in the context of a session
    (LaunchRequest and IntentRequest), the context object duplicates the user and
    application information that is also available in the session."""

    request: _BaseAlexaRequest
    """A request object that provides the details of the user’s request.
    There are several different request types available."""

    def __init__(self, e: dict):
        self.version = e['version']
        LOG.debug('Request version: %s', self.version)
        assert self.version == SUPPORTED_SCHEMA_VERSION
        if e['session']:
            self.session = _RequestSession(e['session'])
        if 'context' in e:
            self.context = _RequestContext(e['context'])
        self.request = _build_alexa_request(e['request'])
<|endoftext|>


<commit_before>def _get_feature(name):
    import __future__
    try:
        return getattr(__future__, name)
    except AttributeError:
        # unique fake object for earlier Python versions or Python 3
        return object()

unicode_literals = _get_feature("unicode_literals")
with_statement = _get_feature("with_statement")
division = _get_feature("division")
print_function = _get_feature("print_function")
absolute_import = _get_feature("absolute_import")
nested_scopes = _get_feature("nested_scopes")  # dummy
generators = _get_feature("generators")  # dummy

del _get_feature
<commit_msg>minor code simplification<commit_after>def _get_feature(name):
    import __future__
    # fall back to a unique fake object for earlier Python versions or Python 3
    return getattr(__future__, name, object())

unicode_literals = _get_feature("unicode_literals")
with_statement = _get_feature("with_statement")
division = _get_feature("division")
print_function = _get_feature("print_function")
absolute_import = _get_feature("absolute_import")
nested_scopes = _get_feature("nested_scopes")  # dummy
generators = _get_feature("generators")  # dummy

del _get_feature
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import fnmatch
import glob
import logging
import os
import pyinotify
import queue
import sys
import threading

logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s: %(message)s')

class Captor(pyinotify.ProcessEvent, threading.Thread):
    def __init__(self, directory, pattern):
        super(Captor, self).__init__()
        self.directory = directory
        self.pattern = pattern
        self.spec = os.path.join(self.directory, self.pattern)
        self.logger = logging.getLogger('captor')
        self.logger.setLevel(logging.DEBUG)
        wm = pyinotify.WatchManager()
        mask = pyinotify.IN_CLOSE_NOWRITE | pyinotify.IN_CLOSE_WRITE
        wm.add_watch(self.directory, mask)
        self.notifier = pyinotify.Notifier(wm, self)
        self.queue = queue.Queue()
        self.collect()
        self.daemon = True
        self.start()

    def collect(self):
        for found in glob.glob(os.path.join(self.directory, self.pattern)):
            self.logger.info('Collected ({}): {}'.format(self.spec, found))
            self.queue.put(found)

    def run(self):
        self.logger.info('Captor (%s) ready', self.spec)
        try:
            self.notifier.loop()
        except Exception:
            self.logger.exception('Captor ({}) failed'.format(self.spec))

    def process_IN_CLOSE_WRITE(self, event):
        directory, name = os.path.split(event.pathname)
        if fnmatch.fnmatch(name, self.pattern):
            self.logger.info('Captured ({}): {}'.format(self.spec, name))
            self.queue.put(event.pathname)

    def empty(self):
        return self.queue.empty()

    def grab(self):
        return self.queue.get(block=True, timeout=None)

if __name__ == '__main__':
    if len(sys.argv) < 3:
        print('Usage: captor.py DIRECTORY PATTERN')
    else:
        Captor(sys.argv[1], sys.argv[2]).join()
<commit_msg>Using the right inherited constructor in Captor<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import fnmatch
import glob
import logging
import os
import pyinotify
import queue
import sys
import threading

logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s: %(message)s')

class Captor(pyinotify.ProcessEvent, threading.Thread):
    def __init__(self, directory, pattern):
        threading.Thread.__init__(self)
        self.directory = directory
        self.pattern = pattern
        self.spec = os.path.join(self.directory, self.pattern)
        self.logger = logging.getLogger('captor')
        self.logger.setLevel(logging.DEBUG)
        wm = pyinotify.WatchManager()
        mask = pyinotify.IN_CLOSE_NOWRITE | pyinotify.IN_CLOSE_WRITE
        wm.add_watch(self.directory, mask)
        self.notifier = pyinotify.Notifier(wm, self)
        self.queue = queue.Queue()
        self.collect()
        self.daemon = True
        self.start()

    def collect(self):
        for found in glob.glob(os.path.join(self.directory, self.pattern)):
            self.logger.info('Collected ({}): {}'.format(self.spec, found))
            self.queue.put(found)

    def run(self):
        self.logger.info('Captor (%s) ready', self.spec)
        try:
            self.notifier.loop()
        except Exception:
            self.logger.exception('Captor ({}) failed'.format(self.spec))

    def process_IN_CLOSE_WRITE(self, event):
        directory, name = os.path.split(event.pathname)
        if fnmatch.fnmatch(name, self.pattern):
            self.logger.info('Captured ({}): {}'.format(self.spec, name))
            self.queue.put(event.pathname)

    def empty(self):
        return self.queue.empty()

    def grab(self):
        return self.queue.get(block=True, timeout=None)

if __name__ == '__main__':
    if len(sys.argv) < 3:
        print('Usage: captor.py DIRECTORY PATTERN')
    else:
        Captor(sys.argv[1], sys.argv[2]).join()
<|endoftext|>


<commit_before>#
#   Pyrex - C naming conventions
#
#
#   Prefixes for generating C names.
#   Collected here to facilitate ensuring uniqueness.
#

pyrex_prefix    = "__pyx_"

builtin_prefix    = pyrex_prefix + "builtin_"
arg_prefix        = pyrex_prefix + "arg_"
funcdoc_prefix    = pyrex_prefix + "doc_"
enum_prefix       = pyrex_prefix + "e_"
func_prefix       = pyrex_prefix + "f_"
pyfunc_prefix     = pyrex_prefix + "pf_"
gstab_prefix      = pyrex_prefix + "getsets_"
prop_get_prefix   = pyrex_prefix + "getprop_"
const_prefix      = pyrex_prefix + "k"
label_prefix      = pyrex_prefix + "L"
pymethdef_prefix  = pyrex_prefix + "mdef_"
methtab_prefix    = pyrex_prefix + "methods_"
memtab_prefix     = pyrex_prefix + "members_"
interned_prefix   = pyrex_prefix + "n_"
interned_num_prefix = pyrex_prefix + "num_"
objstruct_prefix  = pyrex_prefix + "obj_"
typeptr_prefix    = pyrex_prefix + "ptype_"
prop_set_prefix   = pyrex_prefix + "setprop_"
type_prefix       = pyrex_prefix + "t_"
typeobj_prefix    = pyrex_prefix + "type_"
var_prefix        = pyrex_prefix + "v_"
vtable_prefix     = pyrex_prefix + "vtable_"
vtabptr_prefix    = pyrex_prefix + "vtabptr_"
vtabstruct_prefix = pyrex_prefix + "vtabstruct_"
opt_arg_prefix    = pyrex_prefix + "opt_args_"

args_cname       = pyrex_prefix + "args"
kwdlist_cname    = pyrex_prefix + "argnames"
obj_base_cname   = pyrex_prefix + "base"
builtins_cname   = pyrex_prefix + "b"
preimport_cname  = pyrex_prefix + "i"
moddict_cname    = pyrex_prefix + "d"
dummy_cname      = pyrex_prefix + "dummy"
filename_cname   = pyrex_prefix + "filename"
filetable_cname  = pyrex_prefix + "f"
filenames_cname  = pyrex_prefix + "filenames"
fileinit_cname   = pyrex_prefix + "init_filenames"
intern_tab_cname = pyrex_prefix + "intern_tab"
kwds_cname       = pyrex_prefix + "kwds"
lineno_cname     = pyrex_prefix + "lineno"
module_cname     = pyrex_prefix + "m"
moddoc_cname     = pyrex_prefix + "mdoc"
methtable_cname  = pyrex_prefix + "methods"
retval_cname     = pyrex_prefix + "r"
reqd_kwds_cname  = pyrex_prefix + "reqd_kwds"
self_cname       = pyrex_prefix + "self"
stringtab_cname  = pyrex_prefix + "string_tab"
vtabslot_cname   = pyrex_prefix + "vtab"
c_api_tab_cname  = pyrex_prefix + "c_api_tab"
gilstate_cname   = pyrex_prefix + "state"
skip_dispatch_cname = pyrex_prefix + "skip_dispatch"
empty_tuple      = pyrex_prefix + "empty_tuple"
cleanup_cname    = pyrex_prefix + "module_cleanup"
optional_args_cname = pyrex_prefix + "optional_args"
no_opt_args      = pyrex_prefix + "no_opt_args"

extern_c_macro  = pyrex_prefix.upper() + "EXTERN_C"

exc_type_name   = pyrex_prefix + "exc_type"
exc_value_name  = pyrex_prefix + "exc_value"
exc_tb_name     = pyrex_prefix + "exc_tb"
exc_lineno_name = pyrex_prefix + "exc_lineno"

exc_vars = (exc_type_name, exc_value_name, exc_tb_name)

api_name        = pyrex_prefix + "capi__"

h_guard_prefix   = "__PYX_HAVE__"
api_guard_prefix = "__PYX_HAVE_API__"
api_func_guard   = "__PYX_HAVE_API_FUNC_"

def py_version_hex(major, minor=0, micro=0, release_level=0, release_serial=0):
    return (major << 24) | (minor << 16) | (micro << 8) | (release_level << 4) | (release_serial)
<commit_msg>num -> int for int constants<commit_after>#
#   Pyrex - C naming conventions
#
#
#   Prefixes for generating C names.
#   Collected here to facilitate ensuring uniqueness.
#

pyrex_prefix    = "__pyx_"

builtin_prefix    = pyrex_prefix + "builtin_"
arg_prefix        = pyrex_prefix + "arg_"
funcdoc_prefix    = pyrex_prefix + "doc_"
enum_prefix       = pyrex_prefix + "e_"
func_prefix       = pyrex_prefix + "f_"
pyfunc_prefix     = pyrex_prefix + "pf_"
gstab_prefix      = pyrex_prefix + "getsets_"
prop_get_prefix   = pyrex_prefix + "getprop_"
const_prefix      = pyrex_prefix + "k"
label_prefix      = pyrex_prefix + "L"
pymethdef_prefix  = pyrex_prefix + "mdef_"
methtab_prefix    = pyrex_prefix + "methods_"
memtab_prefix     = pyrex_prefix + "members_"
interned_prefix   = pyrex_prefix + "n_"
interned_num_prefix = pyrex_prefix + "int_"
objstruct_prefix  = pyrex_prefix + "obj_"
typeptr_prefix    = pyrex_prefix + "ptype_"
prop_set_prefix   = pyrex_prefix + "setprop_"
type_prefix       = pyrex_prefix + "t_"
typeobj_prefix    = pyrex_prefix + "type_"
var_prefix        = pyrex_prefix + "v_"
vtable_prefix     = pyrex_prefix + "vtable_"
vtabptr_prefix    = pyrex_prefix + "vtabptr_"
vtabstruct_prefix = pyrex_prefix + "vtabstruct_"
opt_arg_prefix    = pyrex_prefix + "opt_args_"

args_cname       = pyrex_prefix + "args"
kwdlist_cname    = pyrex_prefix + "argnames"
obj_base_cname   = pyrex_prefix + "base"
builtins_cname   = pyrex_prefix + "b"
preimport_cname  = pyrex_prefix + "i"
moddict_cname    = pyrex_prefix + "d"
dummy_cname      = pyrex_prefix + "dummy"
filename_cname   = pyrex_prefix + "filename"
filetable_cname  = pyrex_prefix + "f"
filenames_cname  = pyrex_prefix + "filenames"
fileinit_cname   = pyrex_prefix + "init_filenames"
intern_tab_cname = pyrex_prefix + "intern_tab"
kwds_cname       = pyrex_prefix + "kwds"
lineno_cname     = pyrex_prefix + "lineno"
module_cname     = pyrex_prefix + "m"
moddoc_cname     = pyrex_prefix + "mdoc"
methtable_cname  = pyrex_prefix + "methods"
retval_cname     = pyrex_prefix + "r"
reqd_kwds_cname  = pyrex_prefix + "reqd_kwds"
self_cname       = pyrex_prefix + "self"
stringtab_cname  = pyrex_prefix + "string_tab"
vtabslot_cname   = pyrex_prefix + "vtab"
c_api_tab_cname  = pyrex_prefix + "c_api_tab"
gilstate_cname   = pyrex_prefix + "state"
skip_dispatch_cname = pyrex_prefix + "skip_dispatch"
empty_tuple      = pyrex_prefix + "empty_tuple"
cleanup_cname    = pyrex_prefix + "module_cleanup"
optional_args_cname = pyrex_prefix + "optional_args"
no_opt_args      = pyrex_prefix + "no_opt_args"

extern_c_macro  = pyrex_prefix.upper() + "EXTERN_C"

exc_type_name   = pyrex_prefix + "exc_type"
exc_value_name  = pyrex_prefix + "exc_value"
exc_tb_name     = pyrex_prefix + "exc_tb"
exc_lineno_name = pyrex_prefix + "exc_lineno"

exc_vars = (exc_type_name, exc_value_name, exc_tb_name)

api_name        = pyrex_prefix + "capi__"

h_guard_prefix   = "__PYX_HAVE__"
api_guard_prefix = "__PYX_HAVE_API__"
api_func_guard   = "__PYX_HAVE_API_FUNC_"

def py_version_hex(major, minor=0, micro=0, release_level=0, release_serial=0):
    return (major << 24) | (minor << 16) | (micro << 8) | (release_level << 4) | (release_serial)
<|endoftext|>


<commit_before>from data_importers.github_importer import BaseGitHubImporter

class Command(BaseGitHubImporter):

    srid = 4326
    districts_srid = 4326
    council_id = "W06000009"
    elections = ["parl.2019-12-12"]
    scraper_name = "wdiv-scrapers/DC-PollingStations-Pembrokeshire"
    geom_type = "geojson"
    seen_stations = set()

    def district_record_to_dict(self, record):
        poly = self.extract_geometry(record, self.geom_type, self.get_srid("districts"))
        return {
            "internal_council_id": record["DistrictRef"],
            "name": record["DistrictName"],
            "area": poly,
        }

    def station_record_to_dict(self, record):
        if record["County"] == "Carmarthenshire":
            return None

        location = self.extract_geometry(
            record, self.geom_type, self.get_srid("stations")
        )
        codes = record["DistricttRef"].split(";")
        stations = []
        for code in codes:
            if code not in self.seen_stations:
                self.seen_stations.add(code)
                stations.append(
                    {
                        "internal_council_id": code,
                        "postcode": "",
                        "address": record["StationName"],
                        "location": location,
                        "polling_district_id": code,
                    }
                )
            else:
                print(f"Discarding duplicate station for code {code}")

        return stations
<commit_msg>Import script for Pembrokeshire (closes #3441)<commit_after>from data_importers.github_importer import BaseGitHubImporter

class Command(BaseGitHubImporter):

    srid = 4326
    districts_srid = 4326
    council_id = "PEM"
    elections = ["2021-05-06"]
    scraper_name = "wdiv-scrapers/DC-PollingStations-Pembrokeshire"
    geom_type = "geojson"
    seen_stations = set()

    def district_record_to_dict(self, record):
        poly = self.extract_geometry(record, self.geom_type, self.get_srid("districts"))
        return {
            "internal_council_id": record["DistrictRef"],
            "name": record["DistrictName"],
            "area": poly,
        }

    def station_record_to_dict(self, record):
        if record["County"] == "Carmarthenshire":
            return None

        location = self.extract_geometry(
            record, self.geom_type, self.get_srid("stations")
        )
        codes = record["DistricttRef"].split(";")
        stations = []
        for code in codes:
            if code not in self.seen_stations:
                self.seen_stations.add(code)
                stations.append(
                    {
                        "internal_council_id": code,
                        "postcode": "",
                        "address": record["StationName"],
                        "location": location,
                        "polling_district_id": code,
                    }
                )
            else:
                print(f"Discarding duplicate station for code {code}")

        return stations
<|endoftext|>


<commit_before>import sys
import os
path = os.path.realpath(__file__)
sys.path.insert(0, path[0: path.rfind('ixnetwork')])

from ixnetwork.samples.Config import Config
from ixnetwork.IxnHttp import IxnHttp
from ixnetwork.IxnConfigManagement import IxnConfigManagement

# get an IxnHttp instance using the samples Config object
ixnhttp = Config.get_IxnHttp_instance()

# management objects
config_mgmt = IxnConfigManagement(ixnhttp)

# clear the current configuration
config_mgmt.new_config()

# create two virtual ports
vports = ixnhttp.root.create_vport(count=2)

# setup topology west
w_topology = ixnhttp.root.create_topology()
w_topology.attributes.vports.value = [vports[0].href]
w_topology.attributes.name.value = 'West'
w_topology.update()
w_ipv4 = w_topology.create_deviceGroup().create_ethernet().create_ipv4()

# change the address and gw multivalue
w_ipv4.attributes.address.value.singleValue = '1.1.1.1'
w_ipv4.attributes.gatewayIp.value.singleValue = '1.1.1.2'

# setup topology east
e_topology = ixnhttp.root.create_topology()
e_topology.attributes.vports.value = [vports[1].href]
e_topology.attributes.name.value = 'East'
e_topology.update()
e_ipv4 = e_topology.create_deviceGroup().create_ethernet().create_ipv4()

# change the address and gw multivalue
e_ipv4.attributes.address.value.singleValue = '1.1.1.2'
e_ipv4.attributes.gatewayIp.value.singleValue = '1.1.1.1'
<commit_msg>add gatewayIp to sample<commit_after>import sys
import os
path = os.path.realpath(__file__)
sys.path.insert(0, path[0: path.rfind('ixnetwork')])

from ixnetwork.samples.Config import Config
from ixnetwork.IxnHttp import IxnHttp
from ixnetwork.IxnConfigManagement import IxnConfigManagement

# get an IxnHttp instance using the samples Config object
ixnhttp = Config.get_IxnHttp_instance()

# management objects
config_mgmt = IxnConfigManagement(ixnhttp)

# clear the current configuration
config_mgmt.new_config()

# create two virtual ports
vports = ixnhttp.root.create_vport(count=2)

# setup topology west

w_topology = ixnhttp.root.create_topology()
w_topology.attributes.vports.value = [vports[0].href]
w_topology.attributes.name.value = 'West'
w_topology.update()
w_ipv4 = w_topology.create_deviceGroup().create_ethernet().create_ipv4()

# change the address and gw multivalue
w_ipv4.attributes.address.value.singleValue = '1.1.1.1'
w_ipv4.attributes.gatewayIp.value.singleValue = '1.1.1.2'

# setup topology east
e_topology = ixnhttp.root.create_topology()
e_topology.attributes.vports.value = [vports[1].href]
e_topology.attributes.name.value = 'East'
e_topology.update()
e_ipv4 = e_topology.create_deviceGroup().create_ethernet().create_ipv4()

# change the address and gw multivalue
e_ipv4.attributes.address.value.singleValue = '1.1.1.2'
e_ipv4.attributes.gatewayIp.value.singleValue = '1.1.1.1'
<|endoftext|>


<commit_before>import unittest
import pandas as pd

import healthcareai.tests.helpers as helpers
from healthcareai.common.healthcareai_error import HealthcareAIError
from healthcareai.supervised_model_trainer import SupervisedModelTrainer

class TestTopFactors(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        """ Load a dataframe, train a linear model and prepare a prediction dataframe for assertions """
        training_df = helpers.load_factors_dataframe()

        hcai = SupervisedModelTrainer(
            training_df,
            'has_diabetes',
            'classification',
            impute=True,
            grain_column='id')

        # Train the logistic regression model
        cls.trained_lr = hcai.logistic_regression()

        # Load a new df for predicting
        cls.prediction_df = helpers.load_factors_dataframe()

        # Create various outputs
        cls.factors = cls.trained_lr.make_factors(cls.prediction_df, number_top_features=3)

    def test_top_k_factors_raises_error_on_more_features_than_model_has(self):
        self.assertRaises(HealthcareAIError, self.trained_lr.make_factors, self.prediction_df, 10)

    def test_factors_return_is_dataframe(self):
        self.assertIsInstance(self.factors, pd.DataFrame)

    def test_factors_are_same_length_as_input(self):
        self.assertEqual(len(self.factors), len(self.prediction_df))

    def test_factors_columns(self):
        expected = ['id', 'Factor1TXT', 'Factor2TXT', 'Factor3TXT']
        results = self.factors.columns.values
        self.assertTrue(set(expected) == set(results))

    def test_factors_are_correct_on_new_predictions(self):
        new_data = pd.DataFrame(
            {'id': [99, 98],
             'age': [50, 30],
             'gender': ['m', 'f'],
             'weight': [250, 120],
             'a1c': [8, 6],
             'fbg': [50, 80],
             'has_diabetes': ['Y', 'N']})

        expected = pd.DataFrame({
            'id': [99, 98],
            'Factor1TXT': ['age', 'fbg'],
            'Factor2TXT': ['a1c', 'a1c'],
            'Factor3TXT': ['fbg', 'age']})

        results = self.trained_lr.make_factors(new_data)

        # Sort each because column order matters for equality checks
        expected = expected.sort(axis=1)
        result = results.sort(axis=1)

        self.assertTrue(result.equals(expected))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Replaced top_factors.csv with new data set<commit_after>import unittest
import pandas as pd
import numpy as np

from healthcareai.common.healthcareai_error import HealthcareAIError
from healthcareai.supervised_model_trainer import SupervisedModelTrainer

class TestTopFactors(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        """ Load a dataframe, train a linear model and prepare a prediction dataframe for assertions """

        # Build a dataframe
        # TODO Modify probs and cat_weights (and expected df) once top factors is made independent of dummification
        probs = (0.334, 0.333, 0.333)  # distribution of categorical
        cat_weights = (-3, -4, 7)
        # TODO Modify mu and sigma once feature scaling is built into the logistic regression
        mu = 0  # mean of negative_corr
        sigma = 1  # standard deviation of negative_corr
        noise = 0.5  # standard deviation of noise
        rows = 200
        # Set seed to guarantee data set is always the same
        np.random.seed(1066)
        factors_df = pd.DataFrame({'id': range(rows),  # grain col
                                   'positive_corr': np.random.normal(size=rows),
                                   'categorical': np.random.choice(['Common', 'Medium', 'Rare'], p=probs, size=rows),
                                   'negative_corr': np.random.normal(loc=mu, scale=sigma, size=rows),
                                   'useless_pred_1': np.random.normal(size=rows),
                                   'useless_pred_2': np.random.choice(['Y', 'N'], size=rows)},
                                  columns=['id', 'positive_corr', 'categorical', 'negative_corr', 'useless_pred_1',
                                           'useless_pred_2'])

        # Set true decision boundary using importance
        factors_df['dot_product'] = 4 * factors_df['positive_corr']
        factors_df.loc[factors_df['categorical'] == 'Common', 'dot_product'] += cat_weights[0]
        factors_df.loc[factors_df['categorical'] == 'Medium', 'dot_product'] += cat_weights[1]
        factors_df.loc[factors_df['categorical'] == 'Rare', 'dot_product'] += cat_weights[2]
        factors_df['dot_product'] += -2 * (factors_df['negative_corr'] - mu) / sigma

        # Add noise
        factors_df['dot_product'] += np.random.normal(scale=noise, size=rows)

        # Add labels
        factors_df['response'] = 'N'
        factors_df.loc[factors_df['dot_product'] > 0, 'response'] = 'Y'

        # Remove column defining decision boundary
        factors_df.drop('dot_product', axis=1, inplace=True)
        # Reset random seed
        np.random.seed()

        training_df = factors_df.copy()

        hcai = SupervisedModelTrainer(
            training_df,
            'response',
            'classification',
            impute=True,
            grain_column='id')

        # Train the logistic regression model
        cls.trained_lr = hcai.logistic_regression()

        # Load a new df for predicting
        cls.prediction_df = factors_df.copy()

        # Create various outputs
        cls.factors = cls.trained_lr.make_factors(cls.prediction_df, number_top_features=3)

    def test_top_k_factors_raises_error_on_more_features_than_model_has(self):
        self.assertRaises(HealthcareAIError, self.trained_lr.make_factors, self.prediction_df, 10)

    def test_factors_return_is_dataframe(self):
        self.assertIsInstance(self.factors, pd.DataFrame)

    def test_factors_are_same_length_as_input(self):
        self.assertEqual(len(self.factors), len(self.prediction_df))

    def test_factors_columns(self):
        expected = ['id', 'Factor1TXT', 'Factor2TXT', 'Factor3TXT']
        results = self.factors.columns.values
        self.assertTrue(set(expected) == set(results))

    def test_factors_are_correct_on_new_predictions(self):
        new_data = pd.DataFrame({'id': [1001, 1002, 1003],
                                 'positive_corr': [3, 0.3, -2],
                                 'categorical': ['Common', 'Rare', 'Medium'],
                                 'negative_corr': [2, -1.5, 0],
                                 'useless_pred_1': [0, 0.01, -0.01],
                                 'useless_pred_2': ['Y', 'N', 'Y'],
                                 'response': ['Y', 'Y', 'N']})

        expected = pd.DataFrame({'id': [1001, 1002, 1003],
                                 'Factor1TXT': ['positive_corr', 'categorical.Rare', 'positive_corr'],
                                 'Factor2TXT': ['negative_corr', 'negative_corr', 'categorical.Medium'],
                                 'Factor3TXT': ['useless_pred_2.Y', 'positive_corr', 'useless_pred_2.Y']})

        results = self.trained_lr.make_factors(new_data)

        # Sort each because column order matters for equality checks
        expected = expected.sort_index(axis=1)
        result = results.sort_index(axis=1)

        self.assertTrue(result.equals(expected))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import tempfile
import os

class TempFileContext:
    remove_at_exit = True
    removable_files = []

    """
    Base class to create 'with' contexts.

    The __init__ method must define:

        - self.removable_files <list>. This list will hold a list of filenames which will
        removed at the end of the context, or when calling self.delete().
        - self.tempfile <string>. Temporary file of interest, returned by "with" statement.
    """

    def __init__(self, file_extension="", remove_at_exit=True):
        self.remove_at_exit = remove_at_exit
        self.tempfile = tempfile.mktemp() + file_extension

    def __enter__(self):
        return self.tempfile

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.remove_at_exit:
            self.delete()
        else:
            self.remove_intermediate_files()

    def delete(self):
        self.remove_tempfile()
        self.remove_intermediate_files()

    def remove_tempfile(self):
        os.remove(self.tempfile)

    def remove_intermediate_files(self):
        map(os.remove, self.removable_files)

def create_context_class(core_function, output_extension=""):
    """
    Creates a context class, which implements the functionality provided
    by the function core_function.
    Inputs:
        - core_function: a function with the following restrictions.
            + it must take AT LEAST 2 arguments
            + the first argument is an input file to be processed
            + the second argument is the output file
        - output_extension: sometimes the output file of a process must have
                            a given extension. You can specify it here.
                            output_extension can be:
                    a) A string specifying the extension (with dot, eg. ".jpg")
                    b) A function that given the input file name, produces the
                       extension. Example: lambda file: file.split('.')[-1]
    """

    def get_extension(f_in):
        if hasattr(output_extension, '__call__'):
            return output_extension(f_in)
        return output_extension

    # This is our mold of Context Class :)
    class GenericContextClass(TempFileContext):

        def __init__(self, *args, **kwargs):
            if 'remove_at_exit' in kwargs:
                self.remove_at_exit = kwargs['remove_at_exit']
                kwargs.pop('remove_at_exit', None)
            else:
                self.remove_at_exit = True
            self.removable_files = []

            extension = get_extension(args[0])
            if extension:
                extension = "." + extension
            self.tempfile = tempfile.mktemp() + extension
            core_function(args[0], self.tempfile, *args[1:], **kwargs)

    return GenericContextClass
<commit_msg>conserve docstring from core_function and create class name (same as core_function but capitalized)<commit_after>import tempfile
import os

class TempFileContext:
    remove_at_exit = True
    removable_files = []

    """
    Base class to create 'with' contexts.

    The __init__ method must define:

        - self.removable_files <list>. This list will hold a list of filenames which will
        removed at the end of the context, or when calling self.delete().
        - self.tempfile <string>. Temporary file of interest, returned by "with" statement.
    """

    def __init__(self, file_extension="", remove_at_exit=True):
        self.remove_at_exit = remove_at_exit
        self.tempfile = tempfile.mktemp() + file_extension

    def __enter__(self):
        return self.tempfile

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.remove_at_exit:
            self.delete()
        else:
            self.remove_intermediate_files()

    def delete(self):
        self.remove_tempfile()
        self.remove_intermediate_files()

    def remove_tempfile(self):
        os.remove(self.tempfile)

    def remove_intermediate_files(self):
        map(os.remove, self.removable_files)

def create_context_class(core_function, output_extension=""):
    """
    Creates a context class, which implements the functionality provided
    by the function core_function.
    Inputs:
        - core_function: a function with the following restrictions.
            + it must take AT LEAST 2 arguments
            + the first argument is an input file to be processed
            + the second argument is the output file
        - output_extension: sometimes the output file of a process must have
                            a given extension. You can specify it here.
                            output_extension can be:
                    a) A string specifying the extension (with dot, eg. ".jpg")
                    b) A function that given the input file name, produces the
                       extension. Example: lambda file: file.split('.')[-1]
    """

    def get_extension(f_in):
        if hasattr(output_extension, '__call__'):
            return output_extension(f_in)
        return output_extension

    # This is the init of our new  Context Class

    def init(self, file_in, *args, **kwargs):
        if 'remove_at_exit' in kwargs:
            self.remove_at_exit = kwargs['remove_at_exit']
            kwargs.pop('remove_at_exit', None)
        else:
            self.remove_at_exit = True
        self.removable_files = []

        extension = get_extension(file_in)
        if extension:
            extension = "." + extension
        self.tempfile = tempfile.mktemp() + extension
        core_function(file_in, self.tempfile, *args, **kwargs)

    class_docstring = "This is the documentation from the core function.\n\
    Remind that when instantiating this class you can use the same arguments EXCEPT\n\
    the output file.\n{}".format(core_function.__doc__)

    dic = {
            '__init__': init,
            '__doc__': class_docstring
            }
    class_name = core_function.__name__.capitalize()
    GenericContextClass = type(class_name, (TempFileContext,), dic)
    return GenericContextClass
<|endoftext|>


<commit_before># Copyright 2016 The Closure Rules Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Common build definitions for Closure Compiler build definitions.
"""

CSS_FILE_TYPE = FileType([".css", ".gss"])
HTML_FILE_TYPE = FileType([".html"])
JS_FILE_TYPE = FileType([".js"])
JS_LANGUAGE_DEFAULT = "ECMASCRIPT5_STRICT"
JS_TEST_FILE_TYPE = FileType(["_test.js"])
SOY_FILE_TYPE = FileType([".soy"])

JS_LANGUAGE_IN = "ECMASCRIPT_2017"
JS_LANGUAGE_OUT_DEFAULT = "ECMASCRIPT5"
JS_LANGUAGES = depset([
    "ECMASCRIPT3",
    "ECMASCRIPT5",
    "ECMASCRIPT5_STRICT",
    "ECMASCRIPT_2015",
])

CLOSURE_LIBRARY_BASE_ATTR = attr.label(
    default=Label("@com_google_javascript_closure_library//:closure/goog/base.js"),
    allow_files=True,
    single_file=True)

CLOSURE_LIBRARY_DEPS_ATTR = attr.label(
    default=Label("@com_google_javascript_closure_library//:closure/goog/deps.js"),
    allow_files=True,
    single_file=True)

CLOSURE_WORKER_ATTR = attr.label(
    default=Label("//java/io/bazel/rules/closure:ClosureWorker"),
    executable=True,
    cfg="host")

def unfurl(deps, provider=""):
  """Returns deps as well as deps exported by parent rules."""
  res = []
  for dep in deps:
    if not provider or hasattr(dep, provider):
      res.append(dep)
    if hasattr(dep, "exports"):
      for edep in dep.exports:
        if not provider or hasattr(edep, provider):
          res.append(edep)
  return res

def collect_js(ctx, deps,
               has_direct_srcs=False,
               no_closure_library=False,
               css=None):
  """Aggregates transitive JavaScript source files from unfurled deps."""
  srcs = depset()
  ijs_files = depset()
  infos = depset()
  modules = depset()
  descriptors = depset()
  stylesheets = depset()
  js_module_roots = depset()
  has_closure_library = False
  for dep in deps:
    srcs += getattr(dep.closure_js_library, "srcs", [])
    ijs_files += getattr(dep.closure_js_library, "ijs_files", [])
    infos += getattr(dep.closure_js_library, "infos", [])
    modules += getattr(dep.closure_js_library, "modules", [])
    descriptors += getattr(dep.closure_js_library, "descriptors", [])
    stylesheets += getattr(dep.closure_js_library, "stylesheets", [])
    js_module_roots += getattr(dep.closure_js_library, "js_module_roots", [])
    has_closure_library = (
        has_closure_library or
        getattr(dep.closure_js_library, "has_closure_library", False))
  if no_closure_library:
    if has_closure_library:
      fail("no_closure_library can't be used when Closure Library is " +
           "already part of the transitive closure")
  elif has_direct_srcs and not has_closure_library:
    tmp = depset([ctx.file._closure_library_base,
               ctx.file._closure_library_deps])
    tmp += srcs
    srcs = tmp
    has_closure_library = True
  if css:
    tmp = depset([ctx.file._closure_library_base,
               css.closure_css_binary.renaming_map])
    tmp += srcs
    srcs = tmp
  return struct(
      srcs=srcs,
      js_module_roots=js_module_roots,
      ijs_files=ijs_files,
      infos=infos,
      modules=modules,
      descriptors=descriptors,
      stylesheets=stylesheets,
      has_closure_library=has_closure_library)

def collect_css(deps, orientation=None):
  """Aggregates transitive CSS source files from unfurled deps."""
  srcs = depset()
  labels = depset()
  for dep in deps:
    srcs += getattr(dep.closure_css_library, "srcs", [])
    labels += getattr(dep.closure_css_library, "labels", [])
    if orientation:
      if dep.closure_css_library.orientation != orientation:
        fail("%s does not have the same orientation" % dep.label)
    orientation = dep.closure_css_library.orientation
  return struct(
      srcs=srcs,
      labels=labels,
      orientation=orientation)

def collect_runfiles(targets):
  """Aggregates data runfiles from targets."""
  data = depset()
  for target in targets:
    if hasattr(target, "closure_legacy_js_runfiles"):
      data += target.closure_legacy_js_runfiles
      continue
    if hasattr(target, "runfiles"):
      data += target.runfiles.files
      continue
    if hasattr(target, "data_runfiles"):
      data += target.data_runfiles.files
    if hasattr(target, "default_runfiles"):
      data += target.default_runfiles.files
  return data

def find_js_module_roots(ctx, srcs):
  """Finds roots of JavaScript sources.

  This discovers --js_module_root paths for direct srcs that deviate from the
  working directory of ctx.action(). This is basically the cartesian product of
  generated roots, external repository roots, and includes prefixes.

  The includes attribute works the same way as it does in cc_library(). It
  contains a list of directories relative to the package. This feature is
  useful for third party libraries that weren't written with include paths
  relative to the root of a monolithic Bazel repository. Also, unlike the C++
  rules, there is no penalty for using includes in JavaScript compilation.
  """
  roots = depset([f.root.path for f in srcs if f.root.path])
  # Bazel started prefixing external repo paths with ../
  new_bazel_version = Label('@foo//bar').workspace_root.startswith('../')
  if ctx.workspace_name != "__main__":
    if new_bazel_version:
      roots += ["%s" % root for root in roots]
      roots += ["../%s" % ctx.workspace_name]
    else:
      roots += ["%s/external/%s" % (root, ctx.workspace_name) for root in roots]
      roots += ["external/%s" % ctx.workspace_name]
  if getattr(ctx.attr, "includes", []):
    for f in srcs:
      if f.owner.package != ctx.label.package:
        fail("Can't have srcs from a different package when using includes")
    magic_roots = []
    for include in ctx.attr.includes:
      if include == ".":
        prefix = ctx.label.package
      else:
        prefix = "%s/%s" % (ctx.label.package, include)
        found = False
        for f in srcs:
          if f.owner.name.startswith(include + "/"):
            found = True
            break
        if not found:
          fail("No srcs found beginning with '%s/'" % include)
      for root in roots:
        magic_roots.append("%s/%s" % (root, prefix))
    roots += magic_roots
  return roots

def sort_roots(roots):
  """Sorts roots with the most labels first."""
  return [r for _, r in sorted([(-len(r.split("/")), r) for r in roots])]

def convert_path_to_es6_module_name(path, roots):
  """Equivalent to JsCheckerHelper#convertPathToModuleName."""
  if not path.endswith(".js"):
    fail("Path didn't end with .js: %s" % path)
  module = path[:-3]
  for root in roots:
    if module.startswith(root + "/"):
      return module[len(root) + 1:]
  return module

def make_jschecker_progress_message(srcs, label):
  if srcs:
    return "Checking %d JS files in %s" % (len(srcs), label)
  else:
    return "Checking %s" % (label)

def difference(a, b):
  return [i for i in a if i not in b]

def long_path(ctx, file_):
  """Returns short_path relative to parent directory."""
  if file_.short_path.startswith("../"):
    return file_.short_path[3:]
  if file_.owner and file_.owner.workspace_root:
    return file_.owner.workspace_root + "/" + file_.short_path
  return ctx.workspace_name + "/" + file_.short_path

def create_argfile(ctx, args):
  bin_dir = ctx.configuration.bin_dir
  if hasattr(ctx, 'bin_dir'):
    bin_dir = ctx.bin_dir
  argfile = ctx.new_file(bin_dir, "%s_worker_input" % ctx.label.name)
  ctx.file_action(output=argfile, content="\n".join(args))
  return argfile

def library_level_checks(ctx, ijs_deps, srcs, executable, output, suppress = []):
  args = [
      "JsCompiler",
      "--checks_only",
      "--incremental_check_mode", "CHECK_IJS",
      "--warning_level", "VERBOSE",
      "--jscomp_off", "reportUnknownTypes",
      "--language_in", "ECMASCRIPT_NEXT",
      "--language_out", "ECMASCRIPT5",
      "--js_output_file", output.path,
  ]
  inputs = []
  for f in ijs_deps:
    args.append("--externs=%s" % f.path)
    inputs.append(f)
  for f in srcs:
    args.append("--js=%s" % f.path)
    inputs.append(f)
  for s in suppress:
    args.append("--suppress")
    args.append(s)
  ctx.action(
      inputs=inputs,
      outputs=[output],
      executable=executable,
      arguments=args,
      mnemonic="LibraryLevelChecks",
      progress_message="Doing library-level typechecking of " + str(ctx.label))
<commit_msg>Remove explicitly set language mode<commit_after># Copyright 2016 The Closure Rules Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Common build definitions for Closure Compiler build definitions.
"""

CSS_FILE_TYPE = FileType([".css", ".gss"])
HTML_FILE_TYPE = FileType([".html"])
JS_FILE_TYPE = FileType([".js"])
JS_LANGUAGE_DEFAULT = "ECMASCRIPT5_STRICT"
JS_TEST_FILE_TYPE = FileType(["_test.js"])
SOY_FILE_TYPE = FileType([".soy"])

JS_LANGUAGE_IN = "ECMASCRIPT_2017"
JS_LANGUAGE_OUT_DEFAULT = "ECMASCRIPT5"
JS_LANGUAGES = depset([
    "ECMASCRIPT3",
    "ECMASCRIPT5",
    "ECMASCRIPT5_STRICT",
    "ECMASCRIPT_2015",
])

CLOSURE_LIBRARY_BASE_ATTR = attr.label(
    default=Label("@com_google_javascript_closure_library//:closure/goog/base.js"),
    allow_files=True,
    single_file=True)

CLOSURE_LIBRARY_DEPS_ATTR = attr.label(
    default=Label("@com_google_javascript_closure_library//:closure/goog/deps.js"),
    allow_files=True,
    single_file=True)

CLOSURE_WORKER_ATTR = attr.label(
    default=Label("//java/io/bazel/rules/closure:ClosureWorker"),
    executable=True,
    cfg="host")

def unfurl(deps, provider=""):
  """Returns deps as well as deps exported by parent rules."""
  res = []
  for dep in deps:
    if not provider or hasattr(dep, provider):
      res.append(dep)
    if hasattr(dep, "exports"):
      for edep in dep.exports:
        if not provider or hasattr(edep, provider):
          res.append(edep)
  return res

def collect_js(ctx, deps,
               has_direct_srcs=False,
               no_closure_library=False,
               css=None):
  """Aggregates transitive JavaScript source files from unfurled deps."""
  srcs = depset()
  ijs_files = depset()
  infos = depset()
  modules = depset()
  descriptors = depset()
  stylesheets = depset()
  js_module_roots = depset()
  has_closure_library = False
  for dep in deps:
    srcs += getattr(dep.closure_js_library, "srcs", [])
    ijs_files += getattr(dep.closure_js_library, "ijs_files", [])
    infos += getattr(dep.closure_js_library, "infos", [])
    modules += getattr(dep.closure_js_library, "modules", [])
    descriptors += getattr(dep.closure_js_library, "descriptors", [])
    stylesheets += getattr(dep.closure_js_library, "stylesheets", [])
    js_module_roots += getattr(dep.closure_js_library, "js_module_roots", [])
    has_closure_library = (
        has_closure_library or
        getattr(dep.closure_js_library, "has_closure_library", False))
  if no_closure_library:
    if has_closure_library:
      fail("no_closure_library can't be used when Closure Library is " +
           "already part of the transitive closure")
  elif has_direct_srcs and not has_closure_library:
    tmp = depset([ctx.file._closure_library_base,
               ctx.file._closure_library_deps])
    tmp += srcs
    srcs = tmp
    has_closure_library = True
  if css:
    tmp = depset([ctx.file._closure_library_base,
               css.closure_css_binary.renaming_map])
    tmp += srcs
    srcs = tmp
  return struct(
      srcs=srcs,
      js_module_roots=js_module_roots,
      ijs_files=ijs_files,
      infos=infos,
      modules=modules,
      descriptors=descriptors,
      stylesheets=stylesheets,
      has_closure_library=has_closure_library)

def collect_css(deps, orientation=None):
  """Aggregates transitive CSS source files from unfurled deps."""
  srcs = depset()
  labels = depset()
  for dep in deps:
    srcs += getattr(dep.closure_css_library, "srcs", [])
    labels += getattr(dep.closure_css_library, "labels", [])
    if orientation:
      if dep.closure_css_library.orientation != orientation:
        fail("%s does not have the same orientation" % dep.label)
    orientation = dep.closure_css_library.orientation
  return struct(
      srcs=srcs,
      labels=labels,
      orientation=orientation)

def collect_runfiles(targets):
  """Aggregates data runfiles from targets."""
  data = depset()
  for target in targets:
    if hasattr(target, "closure_legacy_js_runfiles"):
      data += target.closure_legacy_js_runfiles
      continue
    if hasattr(target, "runfiles"):
      data += target.runfiles.files
      continue
    if hasattr(target, "data_runfiles"):
      data += target.data_runfiles.files
    if hasattr(target, "default_runfiles"):
      data += target.default_runfiles.files
  return data

def find_js_module_roots(ctx, srcs):
  """Finds roots of JavaScript sources.

  This discovers --js_module_root paths for direct srcs that deviate from the
  working directory of ctx.action(). This is basically the cartesian product of
  generated roots, external repository roots, and includes prefixes.

  The includes attribute works the same way as it does in cc_library(). It
  contains a list of directories relative to the package. This feature is
  useful for third party libraries that weren't written with include paths
  relative to the root of a monolithic Bazel repository. Also, unlike the C++
  rules, there is no penalty for using includes in JavaScript compilation.
  """
  roots = depset([f.root.path for f in srcs if f.root.path])
  # Bazel started prefixing external repo paths with ../
  new_bazel_version = Label('@foo//bar').workspace_root.startswith('../')
  if ctx.workspace_name != "__main__":
    if new_bazel_version:
      roots += ["%s" % root for root in roots]
      roots += ["../%s" % ctx.workspace_name]
    else:
      roots += ["%s/external/%s" % (root, ctx.workspace_name) for root in roots]
      roots += ["external/%s" % ctx.workspace_name]
  if getattr(ctx.attr, "includes", []):
    for f in srcs:
      if f.owner.package != ctx.label.package:
        fail("Can't have srcs from a different package when using includes")
    magic_roots = []
    for include in ctx.attr.includes:
      if include == ".":
        prefix = ctx.label.package
      else:
        prefix = "%s/%s" % (ctx.label.package, include)
        found = False
        for f in srcs:
          if f.owner.name.startswith(include + "/"):
            found = True
            break
        if not found:
          fail("No srcs found beginning with '%s/'" % include)
      for root in roots:
        magic_roots.append("%s/%s" % (root, prefix))
    roots += magic_roots
  return roots

def sort_roots(roots):
  """Sorts roots with the most labels first."""
  return [r for _, r in sorted([(-len(r.split("/")), r) for r in roots])]

def convert_path_to_es6_module_name(path, roots):
  """Equivalent to JsCheckerHelper#convertPathToModuleName."""
  if not path.endswith(".js"):
    fail("Path didn't end with .js: %s" % path)
  module = path[:-3]
  for root in roots:
    if module.startswith(root + "/"):
      return module[len(root) + 1:]
  return module

def make_jschecker_progress_message(srcs, label):
  if srcs:
    return "Checking %d JS files in %s" % (len(srcs), label)
  else:
    return "Checking %s" % (label)

def difference(a, b):
  return [i for i in a if i not in b]

def long_path(ctx, file_):
  """Returns short_path relative to parent directory."""
  if file_.short_path.startswith("../"):
    return file_.short_path[3:]
  if file_.owner and file_.owner.workspace_root:
    return file_.owner.workspace_root + "/" + file_.short_path
  return ctx.workspace_name + "/" + file_.short_path

def create_argfile(ctx, args):
  bin_dir = ctx.configuration.bin_dir
  if hasattr(ctx, 'bin_dir'):
    bin_dir = ctx.bin_dir
  argfile = ctx.new_file(bin_dir, "%s_worker_input" % ctx.label.name)
  ctx.file_action(output=argfile, content="\n".join(args))
  return argfile

def library_level_checks(ctx, ijs_deps, srcs, executable, output, suppress = []):
  args = [
      "JsCompiler",
      "--checks_only",
      "--incremental_check_mode", "CHECK_IJS",
      "--warning_level", "VERBOSE",
      "--jscomp_off", "reportUnknownTypes",
      "--language_in", "ECMASCRIPT_2017",
      "--language_out", "ECMASCRIPT5",
      "--js_output_file", output.path,
  ]
  inputs = []
  for f in ijs_deps:
    args.append("--externs=%s" % f.path)
    inputs.append(f)
  for f in srcs:
    args.append("--js=%s" % f.path)
    inputs.append(f)
  for s in suppress:
    args.append("--suppress")
    args.append(s)
  ctx.action(
      inputs=inputs,
      outputs=[output],
      executable=executable,
      arguments=args,
      mnemonic="LibraryLevelChecks",
      progress_message="Doing library-level typechecking of " + str(ctx.label))
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Morfessor 2.0 Categories-MAP variant.
"""

import collections
import logging
import math

import morfessor

_logger = logging.getLogger(__name__)

class CatmapIO(morfessor.MorfessorIO):
    """Extends data file formats to include category tags."""

    def __init__(self, encoding=None, construction_separator=' + ',
                 comment_start='#', compound_separator='\s+',
                 atom_separator=None, category_separator='/'):
        morfessor.MorfessorIO.__init__(
            self, encoding=encoding,
            construction_separator=construction_separator,
            comment_start=comment_start, compound_separator=compound_separator,
            atom_separator=atom_separator)
        self.category_separator = category_separator

class MorphContext:
    def __init__(self):
        self.rcount = 0
        self.left = collections.defaultdict(int)
        self.right = collections.defaultdict(int)

    @property
    def left_perplexity(self):
        return MorphContext._perplexity(self.left)

    @property
    def right_perplexity(self):
        return MorphContext._perplexity(self.right)

    @staticmethod
    def _perplexity(contexts):
        entropy = 0
        for c in contexts:
            p = float(contexts[c]) / float(len(contexts))
            entropy -= p * math.log(p)
        return math.exp(entropy)

CatProbs = collections.namedtuple('CatProbs', ['PRE', 'STM', 'SUF', 'ZZZ'])

class ClassProbs:
    def __init__(self, total_morph_tokens):
        self.total_morph_tokens = float(total_morph_tokens)
        self.probs = None

    def add(self, rcount, catprobs):
        if self.probs is None:
            self.probs = [0.0] * len(catprobs)
        freq = float(rcount) / self.total_morph_tokens
        for i, x in enumerate(catprobs):
            self.probs[i] += freq * float(x)

    def get(self):
        return CatProbs(self.probs)

class CatmapModel:
    """Morfessor Categories-MAP model class."""

    word_boundary = object()

    def __init__(self, ppl_treshold=100, ppl_slope=None, length_treshold=3,
                 length_slope=2):
        self.ppl_treshold = float(ppl_treshold)
        self.length_treshold = float(length_treshold)
        self.length_slope = float(length_slope)
        if ppl_slope is not None:
            self.ppl_slope = float(ppl_slope)
        else:
            self.ppl_slope = 10.0 / self.ppl_treshold

    def load_baseline(self, segmentations):
        self.contexts = collections.defaultdict(MorphContext)
        total_morph_tokens = 0
        for rcount, segments in segmentations:
            total_morph_tokens += len(segments)
            for (i, morph) in enumerate(segments):
                if i == 0:
                    neighbour = CatmapModel.word_boundary
                else:
                    neighbour = segments[i - 1]
                self.contexts[morph].left[neighbour] += rcount
                if i == len(segments) - 1:
                    neighbour = segments[i - 1]
                else:
                    neighbour = CatmapModel.word_boundary
                self.contexts[morph].right[neighbour] += rcount
                self.contexts[morph].rcount += rcount

        classprobs = ClassProbs(total_morph_tokens)
        for morph in self.contexts:
            catprobs = self._contextToProbability(morph, self.contexts[morph])
            # Scale by frequency and accumulate elementwise
            classprobs.add(self.contexts[morph].rcount, catprobs)
            print("%s: %s" % (morph, catprobs))         # FIXME debug
        print("classprobs: %s" % (classprobs.probs,))   # FIXME debug

    def _contextToProbability(self, morph, context):
        prelike = sigmoid(context.right_perplexity, self.ppl_treshold,
                            self.ppl_slope)
        suflike = sigmoid(context.left_perplexity, self.ppl_treshold,
                            self.ppl_slope)
        stmlike = sigmoid(len(morph), self.length_treshold,
                            self.length_slope)

        p_nonmorpheme = (1. - prelike) * (1. - suflike) * (1. - stmlike)

        if p_nonmorpheme == 1:
            p_pre = 0.0
            p_suf = 0.0
            p_stm = 0.0
        else:
            if p_nonmorpheme < 0.001:
                p_nonmorpheme = 0.001

            normcoeff = ((1.0 - p_nonmorpheme) /
                         ((prelike ** 2) + (suflike ** 2) + (stmlike ** 2)))
            p_pre = (prelike ** 2) * normcoeff
            p_suf = (suflike ** 2) * normcoeff
            p_stm = 1.0 - p_pre - p_suf - p_nonmorpheme

        return CatProbs(p_pre, p_stm, p_suf, p_nonmorpheme)

def sigmoid(value, treshold, slope):
    return 1.0 / (1.0 + math.exp(-slope * (value - treshold)))
<commit_msg>Debugging the perplexity calculations.<commit_after>#!/usr/bin/env python
"""
Morfessor 2.0 Categories-MAP variant.
"""

import collections
import logging
import math

import morfessor

_logger = logging.getLogger(__name__)

class CatmapIO(morfessor.MorfessorIO):
    """Extends data file formats to include category tags."""

    def __init__(self, encoding=None, construction_separator=' + ',
                 comment_start='#', compound_separator='\s+',
                 atom_separator=None, category_separator='/'):
        morfessor.MorfessorIO.__init__(
            self, encoding=encoding,
            construction_separator=construction_separator,
            comment_start=comment_start, compound_separator=compound_separator,
            atom_separator=atom_separator)
        self.category_separator = category_separator

class MorphContext:
    def __init__(self):
        self.rcount = 0
        self.left = collections.defaultdict(int)
        self.right = collections.defaultdict(int)

    @property
    def left_perplexity(self):
        return MorphContext._perplexity(self.left)

    @property
    def right_perplexity(self):
        return MorphContext._perplexity(self.right)

    @staticmethod
    def _perplexity(contexts):
        entropy = 0
        total_tokens = float(sum(contexts.values()))
        for c in contexts:
            p = float(contexts[c]) / total_tokens
            entropy -= p * math.log(p)
        return math.exp(entropy)

CatProbs = collections.namedtuple('CatProbs', ['PRE', 'STM', 'SUF', 'ZZZ'])

class ClassProbs:
    def __init__(self, total_morph_tokens):
        self.total_morph_tokens = float(total_morph_tokens)
        self.probs = None

    def add(self, rcount, catprobs):
        if self.probs is None:
            self.probs = [0.0] * len(catprobs)
        freq = float(rcount) / self.total_morph_tokens
        for i, x in enumerate(catprobs):
            self.probs[i] += freq * float(x)

    def get(self):
        return CatProbs(*self.probs)

class CatmapModel:
    """Morfessor Categories-MAP model class."""

    word_boundary = object()

    def __init__(self, ppl_treshold=100, ppl_slope=None, length_treshold=3,
                 length_slope=2, use_word_tokens=True, min_perplexity_length=4):
        self.ppl_treshold = float(ppl_treshold)
        self.length_treshold = float(length_treshold)
        self.length_slope = float(length_slope)
        self.use_word_tokens = bool(use_word_tokens)
        self.min_perplexity_length = int(min_perplexity_length)
        if ppl_slope is not None:
            self.ppl_slope = float(ppl_slope)
        else:
            self.ppl_slope = 10.0 / self.ppl_treshold

    def load_baseline(self, segmentations):
        self.contexts = collections.defaultdict(MorphContext)
        total_morph_tokens = 0
        for rcount, segments in segmentations:
            if not self.use_word_tokens:
                rcount = 1
            total_morph_tokens += len(segments)
            for (i, morph) in enumerate(segments):
                if i == 0:
                    neighbour = CatmapModel.word_boundary
                else:
                    neighbour = segments[i - 1]
                    if len(neighbour) < self.min_perplexity_length:
                        neighbour = None
                if neighbour is not None:
                    self.contexts[morph].left[neighbour] += rcount

                if i == len(segments) - 1:
                    neighbour = CatmapModel.word_boundary
                else:
                    neighbour = segments[i + 1]
                    if len(neighbour) < self.min_perplexity_length:
                        neighbour = None
                if neighbour is not None:
                    self.contexts[morph].right[neighbour] += rcount

                self.contexts[morph].rcount += rcount

        classprobs = ClassProbs(total_morph_tokens)
        for morph in sorted(self.contexts, cmp=lambda x, y: len(x) < len(y)):
            catprobs = self._contextToPrior(morph, self.contexts[morph])
            # Scale by frequency and accumulate elementwise
            classprobs.add(self.contexts[morph].rcount, catprobs)
            print(u'#P(Tag|"{0:s}")\t{1:.10f}\t{2:.10f}\t{3:.10f}\t{4:.10f}'.format(morph, *catprobs))  # FIXME debug
        tmp = classprobs.get()
        print('#PTag("PRE")\t{0:.10f}'.format(tmp.PRE))   # FIXME debug
        print('#PTag("STM")\t{0:.10f}'.format(tmp.STM))   # FIXME debug
        print('#PTag("SUF")\t{0:.10f}'.format(tmp.SUF))   # FIXME debug
        print('#PTag("ZZZ")\t{0:.10f}'.format(tmp.ZZZ))   # FIXME debug

    def _contextToPrior(self, morph, context):
        crap = u'#Features("{0:s}")\t{1:.4f}\t{2:.4f}\t{3:d}'.format(morph, context.right_perplexity, context.left_perplexity, len(morph)) # FIXME debug
        print(crap) # FIXME debug
        prelike = sigmoid(context.right_perplexity, self.ppl_treshold,
                            self.ppl_slope)
        suflike = sigmoid(context.left_perplexity, self.ppl_treshold,
                            self.ppl_slope)
        stmlike = sigmoid(len(morph), self.length_treshold,
                            self.length_slope)

        p_nonmorpheme = (1. - prelike) * (1. - suflike) * (1. - stmlike)

        if p_nonmorpheme == 1:
            p_pre = 0.0
            p_suf = 0.0
            p_stm = 0.0
        else:
            if p_nonmorpheme < 0.001:
                p_nonmorpheme = 0.001

            normcoeff = ((1.0 - p_nonmorpheme) /
                         ((prelike ** 2) + (suflike ** 2) + (stmlike ** 2)))
            p_pre = (prelike ** 2) * normcoeff
            p_suf = (suflike ** 2) * normcoeff
            p_stm = 1.0 - p_pre - p_suf - p_nonmorpheme

        return CatProbs(p_pre, p_stm, p_suf, p_nonmorpheme)

def sigmoid(value, treshold, slope):
    return 1.0 / (1.0 + math.exp(-slope * (value - treshold)))

def debug_trainbaseline():
    baseline = morfessor.BaselineModel()
    io = morfessor.MorfessorIO(encoding='latin-1')
    data = io.read_corpus_list_file('mydata.gz')
    c = baseline.load_data(data)
    e, c = baseline.train_batch('recursive')
    return baseline

# FIXME temporary rough test against old morfessor

baseline = morfessor.BaselineModel()
io = morfessor.MorfessorIO(encoding='latin-1')

baseline.load_segmentations(io.read_segmentation_file('/akulabra/home/t40511/sgronroo/Downloads/morfessor_catmap0.9.2/train/baselineseg.final.gz'))
model = CatmapModel(ppl_treshold=10, ppl_slope=1, length_treshold=3, length_slope=2, use_word_tokens=False)
model.load_baseline(baseline.get_segmentations())

<|endoftext|>


<commit_before>import os
import json
import superdesk
import pymongo

from superdesk import get_resource_service
from flask import current_app as app

"""
App initialization information, maps resource name to the file containing the data
and the index to be created for the resource and the boolean flag to update the
resource or not.
__entities__ = {
    "resource_name": ("file_name", "index_params", "do_patch")
}
"file_name" (str): name of the file containing seed data
"index_params list: List of key (field or List of tuple as required by pymongo create_index function.
http://api.mongodb.org/python/current/api/pymongo/collection.html
For example:
[[("first_name", pymongo.ASCENDING), ("last_name", pymongo.ASCENDING)], "username"] will create two indexes
- composite index of "first_name", "last_name" field.
- index on username field.
Alternatively index param can be specified as
[[("first_name", pymongo.ASCENDING), ("last_name", pymongo.ASCENDING)], [("username", pymongo.ASCENDING)]]
"""
__entities__ = {
    'roles': ('roles.json', ['name'], False),
    'users': ('users.json', [[('first_name', pymongo.ASCENDING),
                             ('last_name', pymongo.DESCENDING)],
                             'username'], False)
}

class AppInitializeWithDataCommand(superdesk.Command):
    """
    Initialize application with predefined data for various entities.
    Entities supported: [roles, users, desks, stages, groups, vocabularies, validators, content_templates].
    If no --entity-name parameter is supplied, all the entities are inserted.
    The entities [vocabularies, validators] will be updated with the predefined data if it already exists,
    no action will be taken for the other entities.
    """

    option_list = [
        superdesk.Option('--entity-name', '-n', dest='entity_name', default='')
    ]

    def run(self, entity_name=None, index_only='false'):
        self.logger.info('Starting data import')
        if entity_name:
            (file_name, index_params, do_patch) = __entities__[entity_name]
            self.import_file(entity_name, file_name, index_params, do_patch)
            return 0

        for name, (file_name, index_params, do_patch) in __entities__.items():
            print('name', name)
            try:
                self.import_file(name, file_name, index_params, do_patch)
            except Exception as ex:
                self.logger.info('Exception loading entity {} from {}'.format(name, file_name))
                self.logger.exception(ex)

        self.logger.info('Data import finished')
        return 0

    def import_file(self, entity_name, file_name, index_params, do_patch=False):
        """
        imports seed data based on the entity_name (resource name) from the file_name specified.
        index_params use to create index for that entity/resource
        :param str entity_name: name of the resource
        :param str file_name: file name that contains seed data
        :param list index_params: list of indexes that is created on that entity.
        For example:
        [[("first_name", pymongo.ASCENDING), ("last_name", pymongo.ASCENDING)], "username"] will create two indexes
        - composite index of "first_name", "last_name" field.
        - index on username field.
        Alternatively index param can be specified as
        [[("first_name", pymongo.ASCENDING), ("last_name", pymongo.ASCENDING)], [("username", pymongo.ASCENDING)]]
        Refer to pymongo create_index documentation for more information.
        http://api.mongodb.org/python/current/api/pymongo/collection.html
        :param bool do_patch: if True then patch the document else don't patch.
        """
        print('Config: ', app.config['APP_ABSPATH'])
        if file_name:
            file_path = os.path.join(app.config.get('APP_ABSPATH'), 'apps', 'prepopulate', 'data_initialization',
                                     file_name)
            print('Got file path: ', file_path)
            with open(file_path, 'rt') as app_prepopulation:
                json_data = json.loads(app_prepopulation.read())
                service = get_resource_service(entity_name)
                data = [app.data.mongo._mongotize(item, service.datasource) for item in json_data]
                existing_data = []
                existing = service.get_from_mongo(None, {})
                update_data = True
                if not do_patch and existing.count() > 0:
                    self.logger.info('Data already exists for {} none will be loaded'.format(entity_name))
                    update_data = False
                elif do_patch and existing.count() > 0:
                    self.logger.info('Data already exists for {} it will be updated'.format(entity_name))

                if update_data:
                    if do_patch:
                        for item in existing:
                            for loaded_item in data:
                                if '_id' in loaded_item and loaded_item['_id'] == item['_id']:
                                    existing_data.append(loaded_item)
                                    data.remove(loaded_item)

                    if data:
                        service.post(data)
                    if existing_data and do_patch:
                        for item in existing_data:
                            service.patch(item['_id'], item)

                self.logger.info('File {} imported successfully.'.format(file_name))

        if index_params:
            for index in index_params:
                index_name = app.data.mongo.pymongo(resource=entity_name).db[entity_name].create_index(index)
                self.logger.info('Index: {} for collection {} created successfully.'.format(index_name, entity_name))

superdesk.command('app:initialize_data', AppInitializeWithDataCommand())
<commit_msg>do_patch set to true<commit_after>import os
import json
import superdesk
import pymongo

from superdesk import get_resource_service
from flask import current_app as app

"""
App initialization information, maps resource name to the file containing the data
and the index to be created for the resource and the boolean flag to update the
resource or not.
__entities__ = {
    "resource_name": ("file_name", "index_params", "do_patch")
}
"file_name" (str): name of the file containing seed data
"index_params list: List of key (field or List of tuple as required by pymongo create_index function.
http://api.mongodb.org/python/current/api/pymongo/collection.html
For example:
[[("first_name", pymongo.ASCENDING), ("last_name", pymongo.ASCENDING)], "username"] will create two indexes
- composite index of "first_name", "last_name" field.
- index on username field.
Alternatively index param can be specified as
[[("first_name", pymongo.ASCENDING), ("last_name", pymongo.ASCENDING)], [("username", pymongo.ASCENDING)]]
"""
__entities__ = {
    'roles': ('roles.json', ['name'], True),
    'users': ('users.json', [[('first_name', pymongo.ASCENDING),
                             ('last_name', pymongo.DESCENDING)],
                             'username'], False)
}

class AppInitializeWithDataCommand(superdesk.Command):
    """
    Initialize application with predefined data for various entities.
    Entities supported: [roles, users, desks, stages, groups, vocabularies, validators, content_templates].
    If no --entity-name parameter is supplied, all the entities are inserted.
    The entities [vocabularies, validators] will be updated with the predefined data if it already exists,
    no action will be taken for the other entities.
    """

    option_list = [
        superdesk.Option('--entity-name', '-n', dest='entity_name', default='')
    ]

    def run(self, entity_name=None, index_only='false'):
        self.logger.info('Starting data import')
        if entity_name:
            (file_name, index_params, do_patch) = __entities__[entity_name]
            self.import_file(entity_name, file_name, index_params, do_patch)
            return 0

        for name, (file_name, index_params, do_patch) in __entities__.items():
            try:
                self.import_file(name, file_name, index_params, do_patch)
            except Exception as ex:
                self.logger.info('Exception loading entity {} from {}'.format(name, file_name))
                self.logger.exception(ex)

        self.logger.info('Data import finished')
        return 0

    def import_file(self, entity_name, file_name, index_params, do_patch=False):
        """
        imports seed data based on the entity_name (resource name) from the file_name specified.
        index_params use to create index for that entity/resource
        :param str entity_name: name of the resource
        :param str file_name: file name that contains seed data
        :param list index_params: list of indexes that is created on that entity.
        For example:
        [[("first_name", pymongo.ASCENDING), ("last_name", pymongo.ASCENDING)], "username"] will create two indexes
        - composite index of "first_name", "last_name" field.
        - index on username field.
        Alternatively index param can be specified as
        [[("first_name", pymongo.ASCENDING), ("last_name", pymongo.ASCENDING)], [("username", pymongo.ASCENDING)]]
        Refer to pymongo create_index documentation for more information.
        http://api.mongodb.org/python/current/api/pymongo/collection.html
        :param bool do_patch: if True then patch the document else don't patch.
        """
        print('Config: ', app.config['APP_ABSPATH'])
        if file_name:
            file_path = os.path.join(app.config.get('APP_ABSPATH'), 'apps', 'prepopulate', 'data_initialization',
                                     file_name)
            print('Got file path: ', file_path)
            with open(file_path, 'rt') as app_prepopulation:
                json_data = json.loads(app_prepopulation.read())
                service = get_resource_service(entity_name)
                data = [app.data.mongo._mongotize(item, service.datasource) for item in json_data]
                existing_data = []
                existing = service.get_from_mongo(None, {})
                update_data = True
                if not do_patch and existing.count() > 0:
                    self.logger.info('Data already exists for {} none will be loaded'.format(entity_name))
                    update_data = False
                elif do_patch and existing.count() > 0:
                    self.logger.info('Data already exists for {} it will be updated'.format(entity_name))

                if update_data:
                    if do_patch:
                        for item in existing:
                            for loaded_item in data:
                                if '_id' in loaded_item and loaded_item['_id'] == item['_id']:
                                    existing_data.append(loaded_item)
                                    data.remove(loaded_item)

                    if data:
                        service.post(data)
                    if existing_data and do_patch:
                        for item in existing_data:
                            service.patch(item['_id'], item)

                self.logger.info('File {} imported successfully.'.format(file_name))

        if index_params:
            for index in index_params:
                index_name = app.data.mongo.pymongo(resource=entity_name).db[entity_name].create_index(index)
                self.logger.info('Index: {} for collection {} created successfully.'.format(index_name, entity_name))

superdesk.command('app:initialize_data', AppInitializeWithDataCommand())
<|endoftext|>