<commit_before>#!/usr/bin/env python

##
# Copyright (c) 2010-2013 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
from __future__ import print_function

"""
This tool gets and sets Calendar Server configuration keys
"""

from getopt import getopt, GetoptError
import os
import plistlib
import signal
import sys
import xml

from twext.python.plistlib import readPlistFromString, writePlistToString
from twistedcaldav.config import config, ConfigDict, ConfigurationError, mergeData
from twistedcaldav.stdconfig import DEFAULT_CONFIG_FILE
WRITABLE_CONFIG_KEYS = [
    "Authentication.Basic.AllowedOverWireUnencrypted",
    "Authentication.Basic.Enabled",
    "Authentication.Digest.AllowedOverWireUnencrypted",
    "Authentication.Digest.Enabled",
    "Authentication.Kerberos.AllowedOverWireUnencrypted",
    "Authentication.Kerberos.Enabled",
    "Authentication.Wiki.Enabled",
    "DataRoot",
    "DefaultLogLevel",
    "EnableCalDAV",
    "EnableCardDAV",
    "EnableSearchAddressBook",
    "EnableSSL",
    "HTTPPort",
    "LogLevels",
    "Notifications.Services.APNS.CalDAV.AuthorityChainPath",
    "Notifications.Services.APNS.CalDAV.CertificatePath",
    "Notifications.Services.APNS.CalDAV.PrivateKeyPath",
    "Notifications.Services.APNS.CardDAV.AuthorityChainPath",
    "Notifications.Services.APNS.CardDAV.CertificatePath",
    "Notifications.Services.APNS.CardDAV.PrivateKeyPath",
    "Notifications.Services.APNS.Enabled",
    "RedirectHTTPToHTTPS",
    "Scheduling.iMIP.Enabled",
    "Scheduling.iMIP.Receiving.Port",
    "Scheduling.iMIP.Receiving.Server",
    "Scheduling.iMIP.Receiving.Type",
    "Scheduling.iMIP.Receiving.Username",
    "Scheduling.iMIP.Receiving.UseSSL",
    "Scheduling.iMIP.Sending.Address",
    "Scheduling.iMIP.Sending.Port",
    "Scheduling.iMIP.Sending.Server",
    "Scheduling.iMIP.Sending.Username",
    "Scheduling.iMIP.Sending.UseSSL",
    "ServerHostName",
    "SSLAuthorityChain",
    "SSLCertificate",
    "SSLPort",
    "SSLPrivateKey",
]

def usage(e=None):
    if e:
        print(e)
        print("")

    name = os.path.basename(sys.argv[0])
    print("usage: %s [options] config_key" % (name,))
    print("")
    print("Print the value of the given config key.")
    print("options:")
    print("  -h --help: print this help and exit")
    print("  -f --config: Specify caldavd.plist configuration path")
    print("  -w --writeconfig: Specify caldavd.plist configuration path for writing")

    if e:
        sys.exit(64)
    else:
        sys.exit(0)

def main():
    try:
        (optargs, args) = getopt(
            sys.argv[1:], "hf:w:", [
                "help",
                "config=",
                "writeconfig=",
            ],
        )
    except GetoptError, e:
        usage(e)

    configFileName = DEFAULT_CONFIG_FILE
    writeConfigFileName = ""

    for opt, arg in optargs:
        if opt in ("-h", "--help"):
            usage()

        elif opt in ("-f", "--config"):
            configFileName = arg

        elif opt in ("-w", "--writeconfig"):
            writeConfigFileName = arg

    try:
        config.load(configFileName)
    except ConfigurationError, e:
        sys.stdout.write("%s\n" % (e,))
        sys.exit(1)

    if not writeConfigFileName:
        # If --writeconfig was not passed, use WritableConfigFile from
        # main plist.  If that's an empty string, writes will happen to
        # the main file.
        writeConfigFileName = config.WritableConfigFile

    if not writeConfigFileName:
        writeConfigFileName = configFileName

    writable = WritableConfig(config, writeConfigFileName)
    writable.read()

    if args:
        for configKey in args:

            if "=" in configKey:
                # This is an assignment
                configKey, stringValue = configKey.split("=")
                value = writable.convertToValue(stringValue)
                writable.set({configKey:value})
            else:
                # This is a read
                c = config
                for subKey in configKey.split("."):
                    c = c.get(subKey, None)
                    if c is None:
                        sys.stderr.write("No such config key: %s\n" % configKey)
                        break
                sys.stdout.write("%s=%s\n" % (configKey, c))

        writable.save(restart=True)

    else:
        # Read plist commands from stdin
        rawInput = sys.stdin.read()
        try:
            plist = readPlistFromString(rawInput)
            # Note: values in plist will already be unicode
        except xml.parsers.expat.ExpatError, e:
            respondWithError(str(e))
            return

        # If the plist is an array, each element of the array is a separate
        # command dictionary.
        if isinstance(plist, list):
            commands = plist
        else:
            commands = [plist]

        runner = Runner(commands)
        runner.run()

class Runner(object):

    """
    A class which carries out commands, which are plist strings containing
    dictionaries with a "command" key, plus command-specific data.
    """

    def __init__(self, commands):
        """
        @param commands: the commands to run
        @type commands: list of plist strings
        """
        self.commands = commands

    def validate(self):
        """
        Validate all the commands by making sure this class implements
        all the command keys.
        @return: True if all commands are valid, False otherwise
        """
        # Make sure commands are valid
        for command in self.commands:
            if 'command' not in command:
                respondWithError("'command' missing from plist")
                return False
            commandName = command['command']
            methodName = "command_%s" % (commandName,)
            if not hasattr(self, methodName):
                respondWithError("Unknown command '%s'" % (commandName,))
                return False
        return True

    def run(self):
        """
        Find the appropriate method for each command and call them.
        """
        try:
            for command in self.commands:
                commandName = command['command']
                methodName = "command_%s" % (commandName,)
                if hasattr(self, methodName):
                    getattr(self, methodName)(command)
                else:
                    respondWithError("Unknown command '%s'" % (commandName,))

        except Exception, e:
            respondWithError("Command failed: '%s'" % (str(e),))
            raise

    def command_readConfig(self, command):
        """
        Return current configuration

        @param command: the dictionary parsed from the plist read from stdin
        @type command: C{dict}
        """
        result = {}
        for keyPath in WRITABLE_CONFIG_KEYS:
            value = getKeyPath(config, keyPath)
            if value is not None:
                # Note: config contains utf-8 encoded strings, but plistlib
                # wants unicode, so decode here:
                if isinstance(value, str):
                    value = value.decode("utf-8")
                setKeyPath(result, keyPath, value)
        respond(command, result)

    def command_writeConfig(self, command):
        """
        Write config to secondary, writable plist

        @param command: the dictionary parsed from the plist read from stdin
        @type command: C{dict}
        """
        writable = WritableConfig(config, config.WritableConfigFile)
        writable.read()
        valuesToWrite = command.get("Values", {})
        # Note: values are unicode if they contain non-ascii
        for keyPath, value in flattenDictionary(valuesToWrite):
            if keyPath in WRITABLE_CONFIG_KEYS:
                writable.set(setKeyPath(ConfigDict(), keyPath, value))
        try:
            writable.save(restart=False)
        except Exception, e:
            respond(command, {"error": str(e)})
        else:
            config.reload()
            self.command_readConfig(command)

def setKeyPath(parent, keyPath, value):
    """
    Allows the setting of arbitrary nested dictionary keys via a single
    dot-separated string.  For example, setKeyPath(parent, "foo.bar.baz",
    "xyzzy") would create any intermediate missing directories (or whatever
    class parent is, such as ConfigDict) so that the following structure
    results:  parent = { "foo" : { "bar" : { "baz" : "xyzzy } } }

    @param parent: the object to modify
    @type parent: any dict-like object
    @param keyPath: a dot-delimited string specifying the path of keys to
        traverse
    @type keyPath: C{str}
    @param value: the value to set
    @type value: c{object}
    @return: parent
    """
    original = parent
    parts = keyPath.split(".")
    for part in parts[:-1]:
        child = parent.get(part, None)
        if child is None:
            parent[part] = child = parent.__class__()
        parent = child
    parent[parts[-1]] = value
    return original

def getKeyPath(parent, keyPath):
    """
    Allows the getting of arbitrary nested dictionary keys via a single
    dot-separated string.  For example, getKeyPath(parent, "foo.bar.baz")
    would fetch parent["foo"]["bar"]["baz"].  If any of the keys don't
    exist, None is returned instead.

    @param parent: the object to traverse
    @type parent: any dict-like object
    @param keyPath: a dot-delimited string specifying the path of keys to
        traverse
    @type keyPath: C{str}
    @return: the value at keyPath
    """
    parts = keyPath.split(".")
    for part in parts[:-1]:
        child = parent.get(part, None)
        if child is None:
            return None
        parent = child
    return parent.get(parts[-1], None)

def flattenDictionary(dictionary, current=""):
    """
    Returns a generator of (keyPath, value) tuples for the given dictionary,
    where each keyPath is a dot-separated string representing the complete
    path to a nested key.

    @param dictionary: the dict object to traverse
    @type dictionary: C{dict}
    @param current: do not use; used internally for recursion
    @type current: C{str}
    @return: generator of (keyPath, value) tuples
    """
    for key, value in dictionary.iteritems():
        if isinstance(value, dict):
            for result in flattenDictionary(value, current + key + "."):
                yield result
        else:
            yield (current + key, value)

def restartService(pidFilename):
    """
    Given the path to a PID file, sends a HUP signal to the contained pid
    in order to cause calendar server to restart.

    @param pidFilename: an absolute path to a PID file
    @type pidFilename: C{str}
    """
    if os.path.exists(pidFilename):
        pidFile = open(pidFilename, "r")
        pid = pidFile.read().strip()
        pidFile.close()
        try:
            pid = int(pid)
        except ValueError:
            return
        try:
            os.kill(pid, signal.SIGHUP)
        except OSError:
            pass

class WritableConfig(object):
    """
    A wrapper around a Config object which allows writing of values.  The idea
    is a deployment could have a master plist which doesn't change, and have
    it include a plist file which does.  This class facilitates writing to that
    included plist.
    """

    def __init__(self, wrappedConfig, fileName):
        """
        @param wrappedConfig: the Config object to read from
        @type wrappedConfig: C{Config}
        @param fileName: the full path to the modifiable plist
        @type fileName: C{str}
        """
        self.config = wrappedConfig
        self.fileName = fileName
        self.changes = None
        self.currentConfigSubset = ConfigDict()
        self.dirty = False

    def set(self, data):
        """
        Merges data into a ConfigDict of changes intended to be saved to disk
        when save( ) is called.

        @param data: a dict containing new values
        @type data: C{dict}
        """
        if not isinstance(data, ConfigDict):
            data = ConfigDict(mapping=data)
        mergeData(self.currentConfigSubset, data)
        self.dirty = True

    def read(self):
        """
        Reads in the data contained in the writable plist file.

        @return: C{ConfigDict}
        """
        if os.path.exists(self.fileName):
            self.currentConfigSubset = ConfigDict(mapping=plistlib.readPlist(self.fileName))
        else:
            self.currentConfigSubset = ConfigDict()

    def toString(self):
        return plistlib.writePlistToString(self.currentConfigSubset)

    def save(self, restart=False):
        """
        Writes any outstanding changes to the writable plist file.  Optionally
        restart calendar server.

        @param restart: whether to restart the calendar server.
        @type restart: C{bool}
        """
        if self.dirty:
            plistlib.writePlist(self.currentConfigSubset, self.fileName)
            self.dirty = False
            if restart:
                restartService(self.config.PIDFile)

    @classmethod
    def convertToValue(cls, string):
        """
        Inspect string and convert the value into an appropriate Python data type
        TODO: change this to look at actual types definied within stdconfig
        """
        if "." in string:
            try:
                value = float(string)
            except ValueError:
                value = string
        else:
            try:
                value = int(string)
            except ValueError:
                if string == "True":
                    value = True
                elif string == "False":
                    value = False
                else:
                    value = string
        return value

def respond(command, result):
    sys.stdout.write(writePlistToString({'command' : command['command'], 'result' : result}))

def respondWithError(msg, status=1):
    sys.stdout.write(writePlistToString({'error' : msg, }))
<commit_msg>Allow enabling of directory search via config tool<commit_after>#!/usr/bin/env python

##
# Copyright (c) 2010-2013 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
from __future__ import print_function

"""
This tool gets and sets Calendar Server configuration keys
"""

from getopt import getopt, GetoptError
import os
import plistlib
import signal
import sys
import xml

from twext.python.plistlib import readPlistFromString, writePlistToString
from twistedcaldav.config import config, ConfigDict, ConfigurationError, mergeData
from twistedcaldav.stdconfig import DEFAULT_CONFIG_FILE
WRITABLE_CONFIG_KEYS = [
    "Authentication.Basic.AllowedOverWireUnencrypted",
    "Authentication.Basic.Enabled",
    "Authentication.Digest.AllowedOverWireUnencrypted",
    "Authentication.Digest.Enabled",
    "Authentication.Kerberos.AllowedOverWireUnencrypted",
    "Authentication.Kerberos.Enabled",
    "Authentication.Wiki.Enabled",
    "DataRoot",
    "DefaultLogLevel",
    "DirectoryAddressBook.params.queryUserRecords",
    "DirectoryAddressBook.params.queryPeopleRecords",
    "EnableCalDAV",
    "EnableCardDAV",
    "EnableSearchAddressBook",
    "EnableSSL",
    "HTTPPort",
    "LogLevels",
    "Notifications.Services.APNS.CalDAV.AuthorityChainPath",
    "Notifications.Services.APNS.CalDAV.CertificatePath",
    "Notifications.Services.APNS.CalDAV.PrivateKeyPath",
    "Notifications.Services.APNS.CardDAV.AuthorityChainPath",
    "Notifications.Services.APNS.CardDAV.CertificatePath",
    "Notifications.Services.APNS.CardDAV.PrivateKeyPath",
    "Notifications.Services.APNS.Enabled",
    "RedirectHTTPToHTTPS",
    "Scheduling.iMIP.Enabled",
    "Scheduling.iMIP.Receiving.Port",
    "Scheduling.iMIP.Receiving.Server",
    "Scheduling.iMIP.Receiving.Type",
    "Scheduling.iMIP.Receiving.Username",
    "Scheduling.iMIP.Receiving.UseSSL",
    "Scheduling.iMIP.Sending.Address",
    "Scheduling.iMIP.Sending.Port",
    "Scheduling.iMIP.Sending.Server",
    "Scheduling.iMIP.Sending.Username",
    "Scheduling.iMIP.Sending.UseSSL",
    "ServerHostName",
    "SSLAuthorityChain",
    "SSLCertificate",
    "SSLPort",
    "SSLPrivateKey",
]

def usage(e=None):
    if e:
        print(e)
        print("")

    name = os.path.basename(sys.argv[0])
    print("usage: %s [options] config_key" % (name,))
    print("")
    print("Print the value of the given config key.")
    print("options:")
    print("  -h --help: print this help and exit")
    print("  -f --config: Specify caldavd.plist configuration path")
    print("  -w --writeconfig: Specify caldavd.plist configuration path for writing")

    if e:
        sys.exit(64)
    else:
        sys.exit(0)

def main():
    try:
        (optargs, args) = getopt(
            sys.argv[1:], "hf:w:", [
                "help",
                "config=",
                "writeconfig=",
            ],
        )
    except GetoptError, e:
        usage(e)

    configFileName = DEFAULT_CONFIG_FILE
    writeConfigFileName = ""

    for opt, arg in optargs:
        if opt in ("-h", "--help"):
            usage()

        elif opt in ("-f", "--config"):
            configFileName = arg

        elif opt in ("-w", "--writeconfig"):
            writeConfigFileName = arg

    try:
        config.load(configFileName)
    except ConfigurationError, e:
        sys.stdout.write("%s\n" % (e,))
        sys.exit(1)

    if not writeConfigFileName:
        # If --writeconfig was not passed, use WritableConfigFile from
        # main plist.  If that's an empty string, writes will happen to
        # the main file.
        writeConfigFileName = config.WritableConfigFile

    if not writeConfigFileName:
        writeConfigFileName = configFileName

    writable = WritableConfig(config, writeConfigFileName)
    writable.read()

    if args:
        for configKey in args:

            if "=" in configKey:
                # This is an assignment
                configKey, stringValue = configKey.split("=")
                value = writable.convertToValue(stringValue)
                writable.set({configKey:value})
            else:
                # This is a read
                c = config
                for subKey in configKey.split("."):
                    c = c.get(subKey, None)
                    if c is None:
                        sys.stderr.write("No such config key: %s\n" % configKey)
                        break
                sys.stdout.write("%s=%s\n" % (configKey, c))

        writable.save(restart=True)

    else:
        # Read plist commands from stdin
        rawInput = sys.stdin.read()
        try:
            plist = readPlistFromString(rawInput)
            # Note: values in plist will already be unicode
        except xml.parsers.expat.ExpatError, e:
            respondWithError(str(e))
            return

        # If the plist is an array, each element of the array is a separate
        # command dictionary.
        if isinstance(plist, list):
            commands = plist
        else:
            commands = [plist]

        runner = Runner(commands)
        runner.run()

class Runner(object):

    """
    A class which carries out commands, which are plist strings containing
    dictionaries with a "command" key, plus command-specific data.
    """

    def __init__(self, commands):
        """
        @param commands: the commands to run
        @type commands: list of plist strings
        """
        self.commands = commands

    def validate(self):
        """
        Validate all the commands by making sure this class implements
        all the command keys.
        @return: True if all commands are valid, False otherwise
        """
        # Make sure commands are valid
        for command in self.commands:
            if 'command' not in command:
                respondWithError("'command' missing from plist")
                return False
            commandName = command['command']
            methodName = "command_%s" % (commandName,)
            if not hasattr(self, methodName):
                respondWithError("Unknown command '%s'" % (commandName,))
                return False
        return True

    def run(self):
        """
        Find the appropriate method for each command and call them.
        """
        try:
            for command in self.commands:
                commandName = command['command']
                methodName = "command_%s" % (commandName,)
                if hasattr(self, methodName):
                    getattr(self, methodName)(command)
                else:
                    respondWithError("Unknown command '%s'" % (commandName,))

        except Exception, e:
            respondWithError("Command failed: '%s'" % (str(e),))
            raise

    def command_readConfig(self, command):
        """
        Return current configuration

        @param command: the dictionary parsed from the plist read from stdin
        @type command: C{dict}
        """
        result = {}
        for keyPath in WRITABLE_CONFIG_KEYS:
            value = getKeyPath(config, keyPath)
            if value is not None:
                # Note: config contains utf-8 encoded strings, but plistlib
                # wants unicode, so decode here:
                if isinstance(value, str):
                    value = value.decode("utf-8")
                setKeyPath(result, keyPath, value)
        respond(command, result)

    def command_writeConfig(self, command):
        """
        Write config to secondary, writable plist

        @param command: the dictionary parsed from the plist read from stdin
        @type command: C{dict}
        """
        writable = WritableConfig(config, config.WritableConfigFile)
        writable.read()
        valuesToWrite = command.get("Values", {})
        # Note: values are unicode if they contain non-ascii
        for keyPath, value in flattenDictionary(valuesToWrite):
            if keyPath in WRITABLE_CONFIG_KEYS:
                writable.set(setKeyPath(ConfigDict(), keyPath, value))
        try:
            writable.save(restart=False)
        except Exception, e:
            respond(command, {"error": str(e)})
        else:
            config.reload()
            self.command_readConfig(command)

def setKeyPath(parent, keyPath, value):
    """
    Allows the setting of arbitrary nested dictionary keys via a single
    dot-separated string.  For example, setKeyPath(parent, "foo.bar.baz",
    "xyzzy") would create any intermediate missing directories (or whatever
    class parent is, such as ConfigDict) so that the following structure
    results:  parent = { "foo" : { "bar" : { "baz" : "xyzzy } } }

    @param parent: the object to modify
    @type parent: any dict-like object
    @param keyPath: a dot-delimited string specifying the path of keys to
        traverse
    @type keyPath: C{str}
    @param value: the value to set
    @type value: c{object}
    @return: parent
    """
    original = parent
    parts = keyPath.split(".")
    for part in parts[:-1]:
        child = parent.get(part, None)
        if child is None:
            parent[part] = child = parent.__class__()
        parent = child
    parent[parts[-1]] = value
    return original

def getKeyPath(parent, keyPath):
    """
    Allows the getting of arbitrary nested dictionary keys via a single
    dot-separated string.  For example, getKeyPath(parent, "foo.bar.baz")
    would fetch parent["foo"]["bar"]["baz"].  If any of the keys don't
    exist, None is returned instead.

    @param parent: the object to traverse
    @type parent: any dict-like object
    @param keyPath: a dot-delimited string specifying the path of keys to
        traverse
    @type keyPath: C{str}
    @return: the value at keyPath
    """
    parts = keyPath.split(".")
    for part in parts[:-1]:
        child = parent.get(part, None)
        if child is None:
            return None
        parent = child
    return parent.get(parts[-1], None)

def flattenDictionary(dictionary, current=""):
    """
    Returns a generator of (keyPath, value) tuples for the given dictionary,
    where each keyPath is a dot-separated string representing the complete
    path to a nested key.

    @param dictionary: the dict object to traverse
    @type dictionary: C{dict}
    @param current: do not use; used internally for recursion
    @type current: C{str}
    @return: generator of (keyPath, value) tuples
    """
    for key, value in dictionary.iteritems():
        if isinstance(value, dict):
            for result in flattenDictionary(value, current + key + "."):
                yield result
        else:
            yield (current + key, value)

def restartService(pidFilename):
    """
    Given the path to a PID file, sends a HUP signal to the contained pid
    in order to cause calendar server to restart.

    @param pidFilename: an absolute path to a PID file
    @type pidFilename: C{str}
    """
    if os.path.exists(pidFilename):
        pidFile = open(pidFilename, "r")
        pid = pidFile.read().strip()
        pidFile.close()
        try:
            pid = int(pid)
        except ValueError:
            return
        try:
            os.kill(pid, signal.SIGHUP)
        except OSError:
            pass

class WritableConfig(object):
    """
    A wrapper around a Config object which allows writing of values.  The idea
    is a deployment could have a master plist which doesn't change, and have
    it include a plist file which does.  This class facilitates writing to that
    included plist.
    """

    def __init__(self, wrappedConfig, fileName):
        """
        @param wrappedConfig: the Config object to read from
        @type wrappedConfig: C{Config}
        @param fileName: the full path to the modifiable plist
        @type fileName: C{str}
        """
        self.config = wrappedConfig
        self.fileName = fileName
        self.changes = None
        self.currentConfigSubset = ConfigDict()
        self.dirty = False

    def set(self, data):
        """
        Merges data into a ConfigDict of changes intended to be saved to disk
        when save( ) is called.

        @param data: a dict containing new values
        @type data: C{dict}
        """
        if not isinstance(data, ConfigDict):
            data = ConfigDict(mapping=data)
        mergeData(self.currentConfigSubset, data)
        self.dirty = True

    def read(self):
        """
        Reads in the data contained in the writable plist file.

        @return: C{ConfigDict}
        """
        if os.path.exists(self.fileName):
            self.currentConfigSubset = ConfigDict(mapping=plistlib.readPlist(self.fileName))
        else:
            self.currentConfigSubset = ConfigDict()

    def toString(self):
        return plistlib.writePlistToString(self.currentConfigSubset)

    def save(self, restart=False):
        """
        Writes any outstanding changes to the writable plist file.  Optionally
        restart calendar server.

        @param restart: whether to restart the calendar server.
        @type restart: C{bool}
        """
        if self.dirty:
            plistlib.writePlist(self.currentConfigSubset, self.fileName)
            self.dirty = False
            if restart:
                restartService(self.config.PIDFile)

    @classmethod
    def convertToValue(cls, string):
        """
        Inspect string and convert the value into an appropriate Python data type
        TODO: change this to look at actual types definied within stdconfig
        """
        if "." in string:
            try:
                value = float(string)
            except ValueError:
                value = string
        else:
            try:
                value = int(string)
            except ValueError:
                if string == "True":
                    value = True
                elif string == "False":
                    value = False
                else:
                    value = string
        return value

def respond(command, result):
    sys.stdout.write(writePlistToString({'command' : command['command'], 'result' : result}))

def respondWithError(msg, status=1):
    sys.stdout.write(writePlistToString({'error' : msg, }))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__version__ = '$Id$'

# The new wikivoyage family that is hosted at wikimedia

from pywikibot import family

class Family(family.WikimediaFamily):
    def __init__(self):
        super(Family, self).__init__()
        self.name = 'wikivoyage'
        self.languages_by_size = [
            'en', 'de', 'pt', 'fr', 'it', 'nl', 'pl', 'ru', 'es', 'vi', 'sv',
            'he', 'ro', 'uk', 'el',
        ]

        self.langs = dict([(lang, '%s.wikivoyage.org' % lang)
                           for lang in self.languages_by_size])
        # Global bot allowed languages on http://meta.wikimedia.org/wiki/Bot_policy/Implementation#Current_implementation
        self.cross_allowed = ['es', 'ru', ]

    def shared_data_repository(self, code, transcluded=False):
        return ('wikidata', 'wikidata')
<commit_msg>Add lang 'zh' to family 'wikivoyage' , update from compat<commit_after># -*- coding: utf-8 -*-

__version__ = '$Id$'

# The new wikivoyage family that is hosted at wikimedia

from pywikibot import family

class Family(family.WikimediaFamily):
    def __init__(self):
        super(Family, self).__init__()
        self.name = 'wikivoyage'
        self.languages_by_size = [
            'en', 'de', 'pt', 'fr', 'it', 'nl', 'pl', 'ru', 'es', 'vi', 'sv',
            'he', 'zh', 'ro', 'uk', 'el',
        ]

        self.langs = dict([(lang, '%s.wikivoyage.org' % lang)
                           for lang in self.languages_by_size])
        # Global bot allowed languages on http://meta.wikimedia.org/wiki/Bot_policy/Implementation#Current_implementation
        self.cross_allowed = ['es', 'ru', ]

    def shared_data_repository(self, code, transcluded=False):
        return ('wikidata', 'wikidata')
<|endoftext|>


<commit_before># flake8: noqa

# Copyright 2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from keystone.middleware.core import *
<commit_msg>Safer noqa handling<commit_after># Copyright 2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from keystone.middleware.core import *  # noqa
<|endoftext|>


<commit_before>import unittest
import sys
import os
import tempfile
import shutil

from alphatwirl.concurrently import HTCondorJobSubmitter

##__________________________________________________________________||
class MockPopen(object):
    def communicate(self, *args, **kwargs):
        return 'submitted to cluster 1012.', ''

##__________________________________________________________________||
class MockPIPE(object):
    pass

##__________________________________________________________________||
class MockSubprocess(object):
    def __init__(self):
        self.PIPE = MockPIPE

    def Popen(self, *args, **kwargs):
        return MockPopen()

##__________________________________________________________________||
class TestHTCondorJobSubmitter(unittest.TestCase):

    def setUp(self):
        self.module = sys.modules['alphatwirl.concurrently.HTCondorJobSubmitter']
        self.org_subprocess = self.module.subprocess
        self.module.subprocess = MockSubprocess()
        self.cwd = os.getcwd()

        self.tmpdir = tempfile.mkdtemp()

    def tearDown(self):
        self.module.subprocess = self.org_subprocess
        os.chdir(self.cwd)

        shutil.rmtree(self.tmpdir)

    def test_run(self):
        obj = HTCondorJobSubmitter()
        obj.run(taskdir = self.tmpdir, package_path = '0.20')

##__________________________________________________________________||

<commit_msg>update TestHTCondorJobSubmitter<commit_after>import unittest
import sys
import os
import tempfile
import shutil

from alphatwirl.concurrently import HTCondorJobSubmitter

##__________________________________________________________________||
class MockWorkingArea(object):
    def open(self):
        self.path = tempfile.mkdtemp()

    def close(self):
        shutil.rmtree(self.path)
        self.path = None

    def package_path(self, package_index):
        return ''

##__________________________________________________________________||
class MockPopen(object):
    def communicate(self, *args, **kwargs):
        return 'submitted to cluster 1012.', ''

##__________________________________________________________________||
class MockPIPE(object):
    pass

##__________________________________________________________________||
class MockSubprocess(object):
    def __init__(self):
        self.PIPE = MockPIPE

    def Popen(self, *args, **kwargs):
        return MockPopen()

##__________________________________________________________________||
class TestHTCondorJobSubmitter(unittest.TestCase):

    def setUp(self):
        self.module = sys.modules['alphatwirl.concurrently.HTCondorJobSubmitter']
        self.org_subprocess = self.module.subprocess
        self.module.subprocess = MockSubprocess()
        self.cwd = os.getcwd()

        self.workingArea = MockWorkingArea()
        self.workingArea.open()

    def tearDown(self):
        self.module.subprocess = self.org_subprocess
        os.chdir(self.cwd)

        self.workingArea.close()

    def test_run(self):
        obj = HTCondorJobSubmitter()
        ## obj.run(workingArea = self.workingArea, package_index = 0)

##__________________________________________________________________||

<|endoftext|>


<commit_before># coding=utf-8

import platform
import time
import json
from datetime import datetime
from threading import Lock

import regex
import requests
from requests.auth import HTTPBasicAuth

from urllib.parse import quote_plus
if 'windows' in str(platform.platform()).lower():
    # noinspection PyPep8Naming
    from classes import Git as git
else:
    from sh import git

from helpers import log
from globalvars import GlobalVars

# noinspection PyRedundantParentheses,PyClassHasNoInit,PyBroadException
class GitManager:

    gitmanager_lock = Lock()

    @classmethod
    def add_to_blacklist(cls, **kwargs):
        blacklist = kwargs.get("blacklist", "")
        item_to_blacklist = kwargs.get("item_to_blacklist", "")
        username = kwargs.get("username", "")
        chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
        code_permissions = False  # kwargs.get("code_permissions", False) - revert this once git is fixed

        # Make sure git credentials are set up
        if git.config("--get", "user.name", _ok_code=[0, 1]) == "":
            return (False, "Tell someone to run `git config user.name \"SmokeDetector\"`")

        if git.config("--get", "user.email", _ok_code=[0, 1]) == "":
            return (False, "Tell someone to run `git config user.email \"smokey@erwaysoftware.com\"`")

        if blacklist == "":
            # If we broke the code, and this isn't assigned, error out before doing anything, but do
            # so gracefully with a nice error message.
            return (False, "Programming Error - Critical information missing for GitManager: blacklist")

        if item_to_blacklist == "":
            # If we broke the code, and this isn't assigned, error out before doing anything, but do
            # so gracefully with a nice error message.
            return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist")

        item_to_blacklist = item_to_blacklist.replace("\s", " ")

        if blacklist == "website":
            blacklist_file_name = "blacklisted_websites.txt"
            ms_search_option = "&body="
        elif blacklist == "keyword":
            blacklist_file_name = "bad_keywords.txt"
            ms_search_option = "&body="
        elif blacklist == "username":
            blacklist_file_name = "blacklisted_usernames.txt"
            ms_search_option = "&username="
        elif blacklist == "watch_keyword":
            blacklist_file_name = "watched_keywords.txt"
            ms_search_option = "&body="
        else:
            # Just checking all bases, but blacklist_file_name *might* have empty value
            # if we don't address it here.
            return (False, "Invalid blacklist type specified, something has broken badly!")

        try:
            cls.gitmanager_lock.acquire()
            git.checkout("master")
            try:
                git.pull()
            except:
                pass

            # Check that we're up-to-date with origin (GitHub)
            git.remote.update()
            if 'windows' in platform.platform().lower():
                if git.rev_parse("refs/remotes/origin/master").strip() != git.rev_parse("master").strip():
                    return (False, "HEAD isn't at tip of origin's master branch")
            else:
                if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
                    return (False, "HEAD isn't at tip of origin's master branch")

            # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
            if blacklist_file_name in git.status():  # Also ugly
                return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name))

            # Set up parameters for watch vs blacklist
            if blacklist_file_name in ['watched_keywords.txt']:
                op = 'watch'
                now = datetime.now().strftime('%s')
                item = item_to_blacklist
                item_to_blacklist = "\t".join([now, username, item])
                item_regex = regex.compile(r'\t\L<item>$', item=[item])
            else:
                op = 'blacklist'
                item = item_to_blacklist
                item_regex = regex.compile(r'^\L<item>$', item=[item])

            # Prevent duplicates
            with open(blacklist_file_name, "r") as blacklist_file:
                for lineno, line in enumerate(blacklist_file, 1):
                    if item_regex.search(line):
                        return (False, '{0} already {1}ed on {2} line {3}'.format(
                            item, op, blacklist_file_name, lineno))

            # Remove from watch if watched
            write_lines = False
            if blacklist_file_name not in ['watched_keywords.txt']:
                watch_lines = []
                watch_regex = regex.compile(r'\t\L<item>$', item=[item])
                with open('watched_keywords.txt', 'r') as watch_file:
                    for lineno, line in enumerate(watch_file, 1):
                        if watch_regex.search(line):
                            write_lines = True
                            continue
                        watch_lines.append(line)
                if write_lines:
                    with open('watched_keywords.txt', 'w') as watch_file:
                        for line in watch_lines:
                            watch_file.write(line)

            # Add item to file
            with open(blacklist_file_name, "a+") as blacklist_file:
                last_character = blacklist_file.read()[-1:]
                if last_character not in ["", "\n"]:
                    blacklist_file.write("\n")
                blacklist_file.write(item_to_blacklist + "\n")

            # Checkout a new branch (for PRs for non-code-privileged people)
            branch = "auto-blacklist-{0}".format(str(time.time()))
            git.checkout("-b", branch)

            # Clear HEAD just in case
            git.reset("HEAD")

            git.add(blacklist_file_name)
            if write_lines:
                git.add('watched_keywords.txt')

            git.commit("--author='SmokeDetector <smokey@erwaysoftware.com>'",
                       "-m", u"Auto {0} of {1} by {2} --autopull".format(op, item, username))

            if code_permissions:
                git.checkout("master")
                git.merge(branch)
                git.push("origin", "master")
                git.branch('-D', branch)  # Delete the branch in the local git tree since we're done with it.
            else:
                git.push("origin", branch)
                git.checkout("master")

                if GlobalVars.github_username is None or GlobalVars.github_password is None:
                    return (False, "Tell someone to set a GH password")

                payload = {"title": u"{0}: {1} {2}".format(username, op.title(), item),
                           "body": u"[{0}]({1}) requests the {2} of the {3} {4}. See the Metasmoke search [here]"
                                   "(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the "
                                   "Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n"
                                   u"<!-- METASMOKE-BLACKLIST-{8} {4} -->".format(
                                       username, chat_profile_link, op, blacklist,
                                       item, ms_search_option,
                                       quote_plus(item),
                                       quote_plus(item.replace("\\W", " ").replace("\\.", ".")),
                                       blacklist.upper()),
                           "head": branch,
                           "base": "master"}
                response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls",
                                         auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password),
                                         data=json.dumps(payload))
                log('debug', response.json())
                try:
                    git.checkout("deploy")  # Return to deploy, pending the accept of the PR in Master.
                    git.branch('-D', branch)  # Delete the branch in the local git tree since we're done with it.
                    url = response.json()["html_url"]
                    return (True,
                            "You don't have code privileges, but I've [created PR#{1} for you]({0}).".format(
                                url, url.split('/')[-1]))
                except KeyError:
                    git.checkout("deploy")  # Return to deploy

                    # Delete the branch in the local git tree, we'll create it again if the
                    # command is run again. This way, we keep things a little more clean in
                    # the local git tree
                    git.branch('-D', branch)

                    # Error capture/checking for any "invalid" GH reply without an 'html_url' item,
                    # which will throw a KeyError.
                    if "bad credentials" in str(response.json()['message']).lower():
                        # Capture the case when GH credentials are bad or invalid
                        return (False, "Something is wrong with the GH credentials, tell someone to check them.")
                    else:
                        # Capture any other invalid response cases.
                        return (False, "A bad or invalid reply was received from GH, the message was: %s" %
                                response.json()['message'])
        except Exception:
            return (False, "Git functions failed for unspecified reasons.")
        finally:
            # Always return to `deploy` branch when done with anything.
            git.checkout("deploy")
            cls.gitmanager_lock.release()

        if op == 'blacklist':
            return (True, "Blacklisted {0}".format(item))
        elif op == 'watch':
            return (True, "Added {0} to watchlist".format(item))

    @staticmethod
    def current_git_status():
        if 'windows' in platform.platform().lower():
            return git.status_stripped()
        else:
            return git("-c", "color.status=false", "status")

    @staticmethod
    def get_remote_diff():
        git.fetch()
        if 'windows' in platform.platform().lower():
            return git.diff_filenames("deploy", "origin/deploy")
        else:
            return git("-c", "color.diff=false", "diff", "--name-only", "deploy", "origin/deploy")

    @staticmethod
    def pull_remote():
        git.pull()
<commit_msg>Revert "Update gitmanager.py"<commit_after># coding=utf-8

import platform
import time
import json
from datetime import datetime
from threading import Lock

import regex
import requests
from requests.auth import HTTPBasicAuth

from urllib.parse import quote_plus
if 'windows' in str(platform.platform()).lower():
    # noinspection PyPep8Naming
    from classes import Git as git
else:
    from sh import git

from helpers import log
from globalvars import GlobalVars

# noinspection PyRedundantParentheses,PyClassHasNoInit,PyBroadException
class GitManager:

    gitmanager_lock = Lock()

    @classmethod
    def add_to_blacklist(cls, **kwargs):
        blacklist = kwargs.get("blacklist", "")
        item_to_blacklist = kwargs.get("item_to_blacklist", "")
        username = kwargs.get("username", "")
        chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
        code_permissions = kwargs.get("code_permissions", False)

        # Make sure git credentials are set up
        if git.config("--get", "user.name", _ok_code=[0, 1]) == "":
            return (False, "Tell someone to run `git config user.name \"SmokeDetector\"`")

        if git.config("--get", "user.email", _ok_code=[0, 1]) == "":
            return (False, "Tell someone to run `git config user.email \"smokey@erwaysoftware.com\"`")

        if blacklist == "":
            # If we broke the code, and this isn't assigned, error out before doing anything, but do
            # so gracefully with a nice error message.
            return (False, "Programming Error - Critical information missing for GitManager: blacklist")

        if item_to_blacklist == "":
            # If we broke the code, and this isn't assigned, error out before doing anything, but do
            # so gracefully with a nice error message.
            return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist")

        item_to_blacklist = item_to_blacklist.replace("\s", " ")

        if blacklist == "website":
            blacklist_file_name = "blacklisted_websites.txt"
            ms_search_option = "&body="
        elif blacklist == "keyword":
            blacklist_file_name = "bad_keywords.txt"
            ms_search_option = "&body="
        elif blacklist == "username":
            blacklist_file_name = "blacklisted_usernames.txt"
            ms_search_option = "&username="
        elif blacklist == "watch_keyword":
            blacklist_file_name = "watched_keywords.txt"
            ms_search_option = "&body="
        else:
            # Just checking all bases, but blacklist_file_name *might* have empty value
            # if we don't address it here.
            return (False, "Invalid blacklist type specified, something has broken badly!")

        try:
            cls.gitmanager_lock.acquire()
            git.checkout("master")
            try:
                git.pull()
            except:
                pass

            # Check that we're up-to-date with origin (GitHub)
            git.remote.update()
            if 'windows' in platform.platform().lower():
                if git.rev_parse("refs/remotes/origin/master").strip() != git.rev_parse("master").strip():
                    return (False, "HEAD isn't at tip of origin's master branch")
            else:
                if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
                    return (False, "HEAD isn't at tip of origin's master branch")

            # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
            if blacklist_file_name in git.status():  # Also ugly
                return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name))

            # Set up parameters for watch vs blacklist
            if blacklist_file_name in ['watched_keywords.txt']:
                op = 'watch'
                now = datetime.now().strftime('%s')
                item = item_to_blacklist
                item_to_blacklist = "\t".join([now, username, item])
                item_regex = regex.compile(r'\t\L<item>$', item=[item])
            else:
                op = 'blacklist'
                item = item_to_blacklist
                item_regex = regex.compile(r'^\L<item>$', item=[item])

            # Prevent duplicates
            with open(blacklist_file_name, "r") as blacklist_file:
                for lineno, line in enumerate(blacklist_file, 1):
                    if item_regex.search(line):
                        return (False, '{0} already {1}ed on {2} line {3}'.format(
                            item, op, blacklist_file_name, lineno))

            # Remove from watch if watched
            write_lines = False
            if blacklist_file_name not in ['watched_keywords.txt']:
                watch_lines = []
                watch_regex = regex.compile(r'\t\L<item>$', item=[item])
                with open('watched_keywords.txt', 'r') as watch_file:
                    for lineno, line in enumerate(watch_file, 1):
                        if watch_regex.search(line):
                            write_lines = True
                            continue
                        watch_lines.append(line)
                if write_lines:
                    with open('watched_keywords.txt', 'w') as watch_file:
                        for line in watch_lines:
                            watch_file.write(line)

            # Add item to file
            with open(blacklist_file_name, "a+") as blacklist_file:
                last_character = blacklist_file.read()[-1:]
                if last_character not in ["", "\n"]:
                    blacklist_file.write("\n")
                blacklist_file.write(item_to_blacklist + "\n")

            # Checkout a new branch (for PRs for non-code-privileged people)
            branch = "auto-blacklist-{0}".format(str(time.time()))
            git.checkout("-b", branch)

            # Clear HEAD just in case
            git.reset("HEAD")

            git.add(blacklist_file_name)
            if write_lines:
                git.add('watched_keywords.txt')

            git.commit("--author='SmokeDetector <smokey@erwaysoftware.com>'",
                       "-m", u"Auto {0} of {1} by {2} --autopull".format(op, item, username))

            if code_permissions:
                git.checkout("master")
                git.merge(branch)
                git.push("origin", "master")
                git.branch('-D', branch)  # Delete the branch in the local git tree since we're done with it.
            else:
                git.push("origin", branch)
                git.checkout("master")

                if GlobalVars.github_username is None or GlobalVars.github_password is None:
                    return (False, "Tell someone to set a GH password")

                payload = {"title": u"{0}: {1} {2}".format(username, op.title(), item),
                           "body": u"[{0}]({1}) requests the {2} of the {3} {4}. See the Metasmoke search [here]"
                                   "(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the "
                                   "Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n"
                                   u"<!-- METASMOKE-BLACKLIST-{8} {4} -->".format(
                                       username, chat_profile_link, op, blacklist,
                                       item, ms_search_option,
                                       quote_plus(item),
                                       quote_plus(item.replace("\\W", " ").replace("\\.", ".")),
                                       blacklist.upper()),
                           "head": branch,
                           "base": "master"}
                response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls",
                                         auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password),
                                         data=json.dumps(payload))
                log('debug', response.json())
                try:
                    git.checkout("deploy")  # Return to deploy, pending the accept of the PR in Master.
                    git.branch('-D', branch)  # Delete the branch in the local git tree since we're done with it.
                    url = response.json()["html_url"]
                    return (True,
                            "You don't have code privileges, but I've [created PR#{1} for you]({0}).".format(
                                url, url.split('/')[-1]))
                except KeyError:
                    git.checkout("deploy")  # Return to deploy

                    # Delete the branch in the local git tree, we'll create it again if the
                    # command is run again. This way, we keep things a little more clean in
                    # the local git tree
                    git.branch('-D', branch)

                    # Error capture/checking for any "invalid" GH reply without an 'html_url' item,
                    # which will throw a KeyError.
                    if "bad credentials" in str(response.json()['message']).lower():
                        # Capture the case when GH credentials are bad or invalid
                        return (False, "Something is wrong with the GH credentials, tell someone to check them.")
                    else:
                        # Capture any other invalid response cases.
                        return (False, "A bad or invalid reply was received from GH, the message was: %s" %
                                response.json()['message'])
        except Exception:
            return (False, "Git functions failed for unspecified reasons.")
        finally:
            # Always return to `deploy` branch when done with anything.
            git.checkout("deploy")
            cls.gitmanager_lock.release()

        if op == 'blacklist':
            return (True, "Blacklisted {0}".format(item))
        elif op == 'watch':
            return (True, "Added {0} to watchlist".format(item))

    @staticmethod
    def current_git_status():
        if 'windows' in platform.platform().lower():
            return git.status_stripped()
        else:
            return git("-c", "color.status=false", "status")

    @staticmethod
    def get_remote_diff():
        git.fetch()
        if 'windows' in platform.platform().lower():
            return git.diff_filenames("deploy", "origin/deploy")
        else:
            return git("-c", "color.diff=false", "diff", "--name-only", "deploy", "origin/deploy")

    @staticmethod
    def pull_remote():
        git.pull()
<|endoftext|>


<commit_before># test_serialization.py

import pytest

from concepts import Context

SERIALIZED = {
    'objects': (
        '1sg', '1pl', '2sg', '2pl', '3sg', '3pl',
    ),
    'properties': (
        '+1', '-1', '+2', '-2', '+3', '-3', '+sg', '+pl', '-sg', '-pl',
    ),
    'context': [
        (0, 3, 5, 6, 9),
        (0, 3, 5, 7, 8),
        (1, 2, 5, 6, 9),
        (1, 2, 5, 7, 8),
        (1, 3, 4, 6, 9),
        (1, 3, 4, 7, 8),
    ],
    'lattice': [
        ((), (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 2, 3, 4, 5, 6), ()),
        ((0,), (0, 3, 5, 6, 9), (7, 8, 9), (0,)),
        ((1,), (0, 3, 5, 7, 8), (7, 10, 11), (0,)),
        ((2,), (1, 2, 5, 6, 9), (8, 12, 13), (0,)),
        ((3,), (1, 2, 5, 7, 8), (10, 12, 14), (0,)),
        ((4,), (1, 3, 4, 6, 9), (9, 13, 15), (0,)),
        ((5,), (1, 3, 4, 7, 8), (11, 14, 15), (0,)),
        ((0, 1), (0, 3, 5), (18, 19), (1, 2)),
        ((0, 2), (5, 6, 9), (16, 18), (1, 3)),
        ((0, 4), (3, 6, 9), (16, 19), (1, 5)),
        ((1, 3), (5, 7, 8), (17, 18), (2, 4)),
        ((1, 5), (3, 7, 8), (17, 19), (2, 6)),
        ((2, 3), (1, 2, 5), (18, 20), (3, 4)),
        ((2, 4), (1, 6, 9), (16, 20), (3, 5)),
        ((3, 5), (1, 7, 8), (17, 20), (4, 6)),
        ((4, 5), (1, 3, 4), (19, 20), (5, 6)),
        ((0, 2, 4), (6, 9), (21,), (8, 9, 13)),
        ((1, 3, 5), (7, 8), (21,), (10, 11, 14)),
        ((0, 1, 2, 3), (5,), (21,), (7, 8, 10, 12)),
        ((0, 1, 4, 5), (3,), (21,), (7, 9, 11, 15)),
        ((2, 3, 4, 5), (1,), (21,), (12, 13, 14, 15)),
        ((0, 1, 2, 3, 4, 5), (), (), (18, 19, 20, 16, 17)),
    ],
}

SERIALIZED_NOLATTICE = {'objects': SERIALIZED['objects'],
                        'properties': SERIALIZED['properties'],
                        'context': list(SERIALIZED['context'])}

@pytest.fixture(params=[SERIALIZED, SERIALIZED_NOLATTICE])
def d(request):
    return request.param

def test_todict(context, d):
    assert 'lattice' not in context.__dict__
    if 'lattice' not in d:
        assert context.todict() == context.todict(include_lattice=False) == d
        assert 'lattice' not in context.__dict__
    else:
        context = Context(context.objects, context.properties, context.bools)
        assert 'lattice' not in context.__dict__
        assert context.todict(include_lattice=True) == context.todict() == d
        assert 'lattice' in context.__dict__

@pytest.fixture
def d_invalid():
    return {'objects': SERIALIZED['objects'],
            'properties': SERIALIZED['properties'],
            'context': list(SERIALIZED['context']),
            'lattice': list(SERIALIZED['lattice'])}

@pytest.mark.parametrize('missing',
                         ['objects', 'properties', 'context', 'lattice'])
def test_fromdict_missing(d_invalid, missing):
    del d_invalid[missing]
    with pytest.raises(ValueError, match=r'missing .*%s' % missing):
        Context.fromdict(d_invalid, require_lattice=(missing == 'lattice'))

@pytest.mark.parametrize('nonstring', ['objects', 'properties'])
def test_fromdict_nonstring(d_invalid, nonstring):
    d_invalid[nonstring] = (42,) + d_invalid[nonstring][1:]
    with pytest.raises(ValueError, match=r'non-string %s' % nonstring):
        Context.fromdict(d_invalid)

@pytest.mark.parametrize('short', ['objects', 'context'])
def test_fromdict_mismatch(d_invalid, short):
    d_invalid[short] = d_invalid[short][1:]
    lens = (5, 6) if short == 'objects' else (6, 5)
    match = r'mismatch: %d objects with %d context' % lens
    with pytest.raises(ValueError, match=match):
        Context.fromdict(d_invalid)

def test_fromdict_context_duplicates(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (first[0], first[0]) + first[2:]
    with pytest.raises(ValueError, match='duplicate'):
        Context.fromdict(d_invalid)

def test_fromdict_context_invalid_index(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (42,) + first[1:]
    with pytest.raises(ValueError, match='invalid index'):
        Context.fromdict(d_invalid)

def test_fromdict_empty_lattice(d_invalid):
    d_invalid['lattice'] = []
    with pytest.raises(ValueError, match='empty lattice'):
        Context.fromdict(d_invalid)

@pytest.fixture(params=[False, True])
def require_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def exclude_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def raw(request):
    return request.param

def test_fromdict(context, d, require_lattice, exclude_lattice, raw):
    if require_lattice and 'lattice' not in d:
        return

    result = Context.fromdict(d,
                              require_lattice=require_lattice,
                              exclude_lattice=exclude_lattice,
                              raw=raw)

    assert result == context

    if exclude_lattice or 'lattice' not in d:
        assert 'lattice' not in result.__dict__
    else:
        assert 'lattice' in result.__dict__
        assert result.lattice

@pytest.mark.parametrize('include_lattice', [False, None, True])
def test_roundtrip(context, include_lattice):
    context = Context(context.objects, context.properties, context.bools)
    assert 'lattice' not in context.__dict__

    d = context.todict(include_lattice=include_lattice)

    assert isinstance(d, dict) and d
    assert all(d[k] for k in ('objects', 'properties', 'context'))
    if include_lattice:
        assert 'lattice' in context.__dict__
        assert d['lattice']
    else:
        assert 'lattice' not in context.__dict__
        assert 'lattice' not in d
<commit_msg>verify reordering-robustness for Context.fromdict(raw=True)<commit_after># test_serialization.py

import random

import pytest

from concepts import Context

SERIALIZED = {
    'objects': (
        '1sg', '1pl', '2sg', '2pl', '3sg', '3pl',
    ),
    'properties': (
        '+1', '-1', '+2', '-2', '+3', '-3', '+sg', '+pl', '-sg', '-pl',
    ),
    'context': [
        (0, 3, 5, 6, 9),
        (0, 3, 5, 7, 8),
        (1, 2, 5, 6, 9),
        (1, 2, 5, 7, 8),
        (1, 3, 4, 6, 9),
        (1, 3, 4, 7, 8),
    ],
    'lattice': [
        ((), (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 2, 3, 4, 5, 6), ()),
        ((0,), (0, 3, 5, 6, 9), (7, 8, 9), (0,)),
        ((1,), (0, 3, 5, 7, 8), (7, 10, 11), (0,)),
        ((2,), (1, 2, 5, 6, 9), (8, 12, 13), (0,)),
        ((3,), (1, 2, 5, 7, 8), (10, 12, 14), (0,)),
        ((4,), (1, 3, 4, 6, 9), (9, 13, 15), (0,)),
        ((5,), (1, 3, 4, 7, 8), (11, 14, 15), (0,)),
        ((0, 1), (0, 3, 5), (18, 19), (1, 2)),
        ((0, 2), (5, 6, 9), (16, 18), (1, 3)),
        ((0, 4), (3, 6, 9), (16, 19), (1, 5)),
        ((1, 3), (5, 7, 8), (17, 18), (2, 4)),
        ((1, 5), (3, 7, 8), (17, 19), (2, 6)),
        ((2, 3), (1, 2, 5), (18, 20), (3, 4)),
        ((2, 4), (1, 6, 9), (16, 20), (3, 5)),
        ((3, 5), (1, 7, 8), (17, 20), (4, 6)),
        ((4, 5), (1, 3, 4), (19, 20), (5, 6)),
        ((0, 2, 4), (6, 9), (21,), (8, 9, 13)),
        ((1, 3, 5), (7, 8), (21,), (10, 11, 14)),
        ((0, 1, 2, 3), (5,), (21,), (7, 8, 10, 12)),
        ((0, 1, 4, 5), (3,), (21,), (7, 9, 11, 15)),
        ((2, 3, 4, 5), (1,), (21,), (12, 13, 14, 15)),
        ((0, 1, 2, 3, 4, 5), (), (), (18, 19, 20, 16, 17)),
    ],
}

SERIALIZED_NOLATTICE = {'objects': SERIALIZED['objects'],
                        'properties': SERIALIZED['properties'],
                        'context': list(SERIALIZED['context'])}

@pytest.fixture(params=[SERIALIZED, SERIALIZED_NOLATTICE])
def d(request):
    return request.param

def test_todict(context, d):
    assert 'lattice' not in context.__dict__
    if 'lattice' not in d:
        assert context.todict() == context.todict(include_lattice=False) == d
        assert 'lattice' not in context.__dict__
    else:
        context = Context(context.objects, context.properties, context.bools)
        assert 'lattice' not in context.__dict__
        assert context.todict(include_lattice=True) == context.todict() == d
        assert 'lattice' in context.__dict__

@pytest.fixture
def d_invalid():
    return {'objects': SERIALIZED['objects'],
            'properties': SERIALIZED['properties'],
            'context': list(SERIALIZED['context']),
            'lattice': list(SERIALIZED['lattice'])}

@pytest.mark.parametrize('missing',
                         ['objects', 'properties', 'context', 'lattice'])
def test_fromdict_missing(d_invalid, missing):
    del d_invalid[missing]
    with pytest.raises(ValueError, match=r'missing .*%s' % missing):
        Context.fromdict(d_invalid, require_lattice=(missing == 'lattice'))

@pytest.mark.parametrize('nonstring', ['objects', 'properties'])
def test_fromdict_nonstring(d_invalid, nonstring):
    d_invalid[nonstring] = (42,) + d_invalid[nonstring][1:]
    with pytest.raises(ValueError, match=r'non-string %s' % nonstring):
        Context.fromdict(d_invalid)

@pytest.mark.parametrize('short', ['objects', 'context'])
def test_fromdict_mismatch(d_invalid, short):
    d_invalid[short] = d_invalid[short][1:]
    lens = (5, 6) if short == 'objects' else (6, 5)
    match = r'mismatch: %d objects with %d context' % lens
    with pytest.raises(ValueError, match=match):
        Context.fromdict(d_invalid)

def test_fromdict_context_duplicates(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (first[0], first[0]) + first[2:]
    with pytest.raises(ValueError, match='duplicate'):
        Context.fromdict(d_invalid)

def test_fromdict_context_invalid_index(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (42,) + first[1:]
    with pytest.raises(ValueError, match='invalid index'):
        Context.fromdict(d_invalid)

def test_fromdict_empty_lattice(d_invalid):
    d_invalid['lattice'] = []
    with pytest.raises(ValueError, match='empty lattice'):
        Context.fromdict(d_invalid)

@pytest.fixture(params=[False, True])
def require_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def exclude_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def raw(request):
    return request.param

def test_fromdict(context, d, require_lattice, exclude_lattice, raw):
    if require_lattice and 'lattice' not in d:
        return

    result = Context.fromdict(d,
                              require_lattice=require_lattice,
                              exclude_lattice=exclude_lattice,
                              raw=raw)

    assert result == context

    if exclude_lattice or 'lattice' not in d:
        assert 'lattice' not in result.__dict__
    else:
        assert 'lattice' in result.__dict__
        assert result.lattice

def test_raw(context, d, raw):
    def shuffled(items):
        result = list(items)
        random.shuffle(result)
        return result

    lattice = d.get('lattice')
    d = {'objects': d['objects'], 'properties': d['properties'],
         'context': [shuffled(intent) for intent in d['context']]}

    if lattice is not None:
        pairs = shuffled(enumerate(lattice))
        index_map = {old: new for new, (old, _) in enumerate(pairs)}
        d['lattice'] = [(shuffled(ex), shuffled(in_),
                         shuffled(index_map[i] for i in up),
                         shuffled(index_map[i] for i in lo))
                        for _, (ex, in_, up, lo) in pairs]

    result = Context.fromdict(d, raw=raw)
    assert result == context
    if lattice is not None:
        assert result.lattice

@pytest.mark.parametrize('include_lattice', [False, None, True])
def test_roundtrip(context, include_lattice):
    context = Context(context.objects, context.properties, context.bools)
    assert 'lattice' not in context.__dict__

    d = context.todict(include_lattice=include_lattice)

    assert isinstance(d, dict) and d
    assert all(d[k] for k in ('objects', 'properties', 'context'))
    if include_lattice:
        assert 'lattice' in context.__dict__
        assert d['lattice']
    else:
        assert 'lattice' not in context.__dict__
        assert 'lattice' not in d
<|endoftext|>


<commit_before>import sys
import os
sys.path[0:0] = [os.path.join(sys.path[0], './pieces')]

p = ["bishop","king","knight","pawn","queen","rook"]
com = ["from {} import {}".format(v,v) for v in p]
for c in com:
    exec(c)

class game():
    def __init__(self):
        self.field = []
        for i in range(8):
            self.field.append([None] * 8)
        self.set_up()
        #sluzi na debug
        self.icons = {"White pawn":"♙","White rook":"♖","White knight":"♘",
                      "White bishop":"♗","White queen":"♕","White king":"♔",
                      "Black pawn":"♟","Black rook":"♜","Black knight":"♞",
                      "Black bishop":"♝","Black queen":"♛","Black king":"♚",
                      "None":" "}

    def set_up(self):
        self.field[0] = [rook("Black"),knight("Black"),bishop("Black"),queen("Black"),
                         king("Black"),bishop("Black"),knight("Black"),rook("Black")]
        self.field[7] = [rook("White"),knight("White"),bishop("White"),queen("White"),
                         king("White"),bishop("White"),knight("White"),rook("White")]
        self.field[1] = [pawn("Black") for i in range(8)]
        self.field[6] = [pawn("White") for i in range(8)]
        for i in range(2,6):
            self.field[i] = [None for i in range(8)]

    def __repr__(self):
        ret = ""
        f = self.field
        for i in range(8):
            ind = 8-i
            ret += str(ind) + ' '
            for j in range(8):
                c = chr(ord("A")+j)
                ret += self.icons[str(f[i][j])]
            ret += "\n"
        ret += '  '
        for c in range(ord("A"),ord("H")+1):
            ret += chr(c) + ' '
        return ret

    def pohni(self,c1 = ("A",1),c2 = ("H",8)):
        c1 = (ord(c1[0])-ord("A"),8-c1[1])
        c2 = (ord(c2[0])-ord("A"),8-c2[1])
        if self.field[c1[1]][c1[0]].pohni(c1,c2,self.field):
            self.field[c2[1]][c2[0]] = self.field[c1[1]][c1[0]]
            self.field[c1[1]][c1[0]] = None
            return True
        else:
            return False

g = game()
print(g)

<commit_msg>Update game.py<commit_after>import sys
import os
sys.path[0:0] = [os.path.join(sys.path[0], './pieces')]

p = ["bishop","king","knight","pawn","queen","rook"]
com = ["from {} import {}".format(v,v) for v in p]
for c in com:
    exec(c)

class game():
    def __init__(self):
        self.field = []
        for i in range(8):
            self.field.append([None] * 8)
        self.set_up()
        #sluzi na debug
        self.icons = {"White pawn":"♙","White rook":"♖","White knight":"♘",
                      "White bishop":"♗","White queen":"♕","White king":"♔",
                      "Black pawn":"♟","Black rook":"♜","Black knight":"♞",
                      "Black bishop":"♝","Black queen":"♛","Black king":"♚",
                      "None":" "}

    def set_up(self):
        self.field[0] = [rook("Black"),knight("Black"),bishop("Black"),queen("Black"),
                         king("Black"),bishop("Black"),knight("Black"),rook("Black")]
        self.field[7] = [rook("White"),knight("White"),bishop("White"),queen("White"),
                         king("White"),bishop("White"),knight("White"),rook("White")]
        self.field[1] = [pawn("Black") for i in range(8)]
        self.field[6] = [pawn("White") for i in range(8)]
        for i in range(2,6):
            self.field[i] = [None for i in range(8)]

    def __repr__(self):
        ret = ""
        f = self.field
        for i in range(8):
            ind = 8-i
            ret += str(ind) + ' '
            for j in range(8):
                c = chr(ord("A")+j)
                ret += self.icons[str(f[i][j])]
            ret += "\n"
        ret += '  '
        for c in range(ord("A"),ord("H")+1):
            ret += chr(c) + ' '
        return ret

    def pohni(self,c1 = ("A",1),c2 = ("H",8)):
        c1 = (ord(c1[0])-ord("A"),8-c1[1])
        c2 = (ord(c2[0])-ord("A"),8-c2[1])
        if self.field[c1[1]][c1[0]] is not None:
            if self.field[c1[1]][c1[0]].pohni(c1,c2,self.field):
                self.field[c2[1]][c2[0]] = self.field[c1[1]][c1[0]]
                self.field[c1[1]][c1[0]] = None
                return True
            else:
                return False
        return False

g = game()
print(g)

<|endoftext|>


<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Interface with the xtas NLP processing system
"""
from amcat.tools import amcates

class ANALYSES:
    postag = [{"module" : "xtas.tasks.single.tokenize"},
              {"module": "xtas.tasks.single.pos_tag",
               "arguments" : {"model" : "nltk"}}]

    corenlp = [{"module" : "xtas.tasks.single.corenlp"}]
    alpino = [{"module" : "xtas.tasks.single.alpino"}]
    sources_nl = [{"module" : "xtas.tasks.single.alpino"},
                  {"module" : "xtas.tasks.single.sources_nl"}]
    tadpole = [{"module" : "xtas.tasks.single.tadpole"}]
    corenlp_lemmatize = [{"module" : "xtas.tasks.single.corenlp_lemmatize"}]

    semafor = [{"module" : "xtas.tasks.single.corenlp"},
               {"module" : "xtas.tasks.single.semafor"}]
    sources_en = [{"module" : "xtas.tasks.single.corenlp"},
                  {"module" : "xtas.tasks.single.semafor"},
                  {"module" : "xtas.tasks.single.sources_en"}]

def get_result(article, analysis, store_intermediate=True, block=True):
    from xtas.tasks.pipeline import pipeline
    if not isinstance(article, int): article = article.id
    if not isinstance(analysis, list):
        if hasattr(ANALYSES, analysis):
            analysis = getattr(ANALYSES, analysis)
        elif "." in analysis:
            analysis = [{"module" : m} for m in analysis.split('__')]
        else:
            raise ValueError("Unknown analysis: {analysis}".format(**locals()))

    es = amcates.ES()
    doc = {'index': es.index, 'type': es.doc_type,
           'id': article, 'field': 'text'}
    r = pipeline(doc, analysis,
                 store_intermediate=store_intermediate)
    return r
<commit_msg>add frog<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Interface with the xtas NLP processing system
"""
from amcat.tools import amcates

class ANALYSES:
    postag = [{"module" : "xtas.tasks.single.tokenize"},
              {"module": "xtas.tasks.single.pos_tag",
               "arguments" : {"model" : "nltk"}}]

    corenlp = [{"module" : "xtas.tasks.single.corenlp"}]
    alpino = [{"module" : "xtas.tasks.single.alpino"}]
    sources_nl = [{"module" : "xtas.tasks.single.alpino"},
                  {"module" : "xtas.tasks.single.sources_nl"}]
    tadpole = [{"module" : "xtas.tasks.single.tadpole"}]
    frog = [{"module" : "xtas.tasks.single.frog"}]
    corenlp_lemmatize = [{"module" : "xtas.tasks.single.corenlp_lemmatize"}]

    semafor = [{"module" : "xtas.tasks.single.corenlp"},
               {"module" : "xtas.tasks.single.semafor"}]
    sources_en = [{"module" : "xtas.tasks.single.corenlp"},
                  {"module" : "xtas.tasks.single.semafor"},
                  {"module" : "xtas.tasks.single.sources_en"}]

def get_result(article, analysis, store_intermediate=True, block=True):
    from xtas.tasks.pipeline import pipeline
    if not isinstance(article, int): article = article.id
    if not isinstance(analysis, list):
        if hasattr(ANALYSES, analysis):
            analysis = getattr(ANALYSES, analysis)
        elif "." in analysis:
            analysis = [{"module" : m} for m in analysis.split('__')]
        else:
            raise ValueError("Unknown analysis: {analysis}".format(**locals()))

    es = amcates.ES()
    doc = {'index': es.index, 'type': es.doc_type,
           'id': article, 'field': 'text'}
    r = pipeline(doc, analysis,
                 store_intermediate=store_intermediate)
    return r
<|endoftext|>


<commit_before># Copyright 2016 the original author or authors.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

__author__ = "Julian Debatin"
__copyright__ = "The authors"
__license__ = "Apache 2"
__email__ = "juliandebatin@gmail.com"
__status__ = "Production"

from Core.CellType import *
from Core.ExecConfig import ExecConfig
from Core.ModelConfig import ModelConfig
from Steppable.InitializerSteppable import InitializerSteppable
from Steppable.DeathSteppable import DeathSteppable
from Steppable.GrowthMitosisSteppable import GrowthMitosisSteppable
from Steppable.GrowthSteppable import GrowthSteppable
from Steppable.CMTransformationSteppable import CMTransformationSteppable
from Steppable.UrinationSteppable import UrinationSteppable
from Logger.VolumeFitnessSteppable import VolumeFitnessSteppable
from Logger.ArrangementFitnessSteppable import ArrangementFitnessSteppable
from Logger.DummyFitnessSteppable import DummyFitnessSteppable
from Steppable.ColonySteppable import ColonySteppable
from Steppable.MutationSteppable import MutationSteppable

class SpaBpaPcdiInUa(ModelConfig):
    def __init__(self, sim, simthread):
        ModelConfig.__init__(self, sim, simthread)

    def _initModel(self):
        self.name = "SpaBpaPcdiInUa"
        self.cellTypes = self._createCellTypes()
        self.energyMatrix = self._createEnergyMatrix()
        self._run() # Must be the last statement.

    def _createCellTypes(self):
        cellTypes = []
        stem = Stemcell
        stem.setGrowthVolumePerDayRelVolume(0.13)
        self.stemNecrosisProb = stem.necrosisProb = 0

        basal = Basalcell
        basal.setGrowthVolumePerDayRelVolume(0.12)
        basal.apoptosisTimeInDays = 80000000000.0
        self.basalNecrosisProb = basal.necrosisProb = 0.000015

        intermediate = Intermediatecell
        intermediate.setGrowthVolumePerDayRelVolume(0.07)
        intermediate.apoptosisTimeInDays = 200000000.0
        self.intermediateNecrosisProb = intermediate.necrosisProb = 0.00018

        umbrella = Umbrellacell
        umbrella.setGrowthVolumePerDayRelVolume(0.06)
        umbrella.apoptosisTimeInDays = 100000000.0
        self.umbrellaNecrosisProb = umbrella.necrosisProb = 0.00020

        stem.setDescendants(0.90, [stem.id, basal.id])
        stem.setDescendants(0.05, [stem.id, stem.id])
        stem.setDescendants(0.05, [basal.id, basal.id])
        basal.setDescendants(0.90, [basal.id, intermediate.id])
        basal.setDescendants(0.05, [basal.id, basal.id])
        basal.setDescendants(0.05, [intermediate.id, intermediate.id])
        intermediate.setDescendants(1.0, [intermediate.id, intermediate.id])

        cellTypes.extend((Medium, Basalmembrane, stem, basal, intermediate, umbrella))

        return cellTypes

    def _getSteppables(self):
        steppableList = []
        steppableList.append(ColonySteppable(self.sim, self))
        steppableList.append(InitializerSteppable(self.sim, self))
        steppableList.append(GrowthSteppable(self.sim, self))
        steppableList.append(GrowthMitosisSteppable(self.sim, self))
        steppableList.append(CMTransformationSteppable(self.sim, self))
        steppableList.append(UrinationSteppable(self.sim, self, prop=0.02))
        steppableList.append(DeathSteppable(self.sim, self))
        #steppableList.append(OptimumSearchSteppable(self.sim, self))
        steppableList.append(VolumeFitnessSteppable(self.sim, self))
        steppableList.append(ArrangementFitnessSteppable(self.sim, self))
        steppableList.append(DummyFitnessSteppable(self.sim, self))
        steppableList.append(MutationSteppable(self.sim, self, self.stemNecrosisProb, self.basalNecrosisProb,
                                               self.intermediateNecrosisProb, self.umbrellaNecrosisProb))

        return steppableList

    def _createExecConfig(self):
        return ExecConfig(MCSperDay=500,# SEED=10,
                          xLength=500, yLength=150, zLength=0, voxelDensity=.8)
<commit_msg>no message<commit_after># Copyright 2016 the original author or authors.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

__author__ = "Julian Debatin"
__copyright__ = "The authors"
__license__ = "Apache 2"
__email__ = "juliandebatin@gmail.com"
__status__ = "Production"

from Core.CellType import *
from Core.ExecConfig import ExecConfig
from Core.ModelConfig import ModelConfig
from Steppable.InitializerSteppable import InitializerSteppable
from Steppable.DeathSteppable import DeathSteppable
from Steppable.GrowthMitosisSteppable import GrowthMitosisSteppable
from Steppable.GrowthSteppable import GrowthSteppable
from Steppable.CMTransformationSteppable import CMTransformationSteppable
from Steppable.UrinationSteppable import UrinationSteppable
from Logger.VolumeFitnessSteppable import VolumeFitnessSteppable
from Logger.ArrangementFitnessSteppable import ArrangementFitnessSteppable
from Logger.DummyFitnessSteppable import DummyFitnessSteppable
from Steppable.ColonySteppable import ColonySteppable
from Steppable.MutationSteppable import MutationSteppable

class SpaBpaPcdiInUa(ModelConfig):
    def __init__(self, sim, simthread):
        ModelConfig.__init__(self, sim, simthread)

    def _initModel(self):
        self.name = "SpaBpaPcdiInUa"
        self.cellTypes = self._createCellTypes()
        self.energyMatrix = self._createEnergyMatrix()
        self._run() # Must be the last statement.

    def _createCellTypes(self):
        cellTypes = []
        stem = Stemcell
        stem.setGrowthVolumePerDayRelVolume(0.13)
        self.stemNecrosisProb = stem.necrosisProb = 0

        basal = Basalcell
        basal.setGrowthVolumePerDayRelVolume(0.12)
        basal.apoptosisTimeInDays = 80000000000.0
        self.basalNecrosisProb = basal.necrosisProb = 0.000015

        intermediate = Intermediatecell
        intermediate.setGrowthVolumePerDayRelVolume(0.07)
        intermediate.apoptosisTimeInDays = 200000000.0
        self.intermediateNecrosisProb = intermediate.necrosisProb = 0.00016

        umbrella = Umbrellacell
        umbrella.setGrowthVolumePerDayRelVolume(0.06)
        umbrella.apoptosisTimeInDays = 100000000.0
        self.umbrellaNecrosisProb = umbrella.necrosisProb = 0.00020

        stem.setDescendants(0.90, [stem.id, basal.id])
        stem.setDescendants(0.05, [stem.id, stem.id])
        stem.setDescendants(0.05, [basal.id, basal.id])
        basal.setDescendants(0.90, [basal.id, intermediate.id])
        basal.setDescendants(0.05, [basal.id, basal.id])
        basal.setDescendants(0.05, [intermediate.id, intermediate.id])
        intermediate.setDescendants(1.0, [intermediate.id, intermediate.id])

        cellTypes.extend((Medium, Basalmembrane, stem, basal, intermediate, umbrella))

        return cellTypes

    def _getSteppables(self):
        steppableList = []
        steppableList.append(ColonySteppable(self.sim, self))
        steppableList.append(InitializerSteppable(self.sim, self))
        steppableList.append(GrowthSteppable(self.sim, self))
        steppableList.append(GrowthMitosisSteppable(self.sim, self))
        steppableList.append(CMTransformationSteppable(self.sim, self))
        steppableList.append(UrinationSteppable(self.sim, self, prop=0.02))
        steppableList.append(DeathSteppable(self.sim, self))
        #steppableList.append(OptimumSearchSteppable(self.sim, self))
        steppableList.append(VolumeFitnessSteppable(self.sim, self))
        steppableList.append(ArrangementFitnessSteppable(self.sim, self))
        steppableList.append(DummyFitnessSteppable(self.sim, self))
        steppableList.append(MutationSteppable(self.sim, self, self.stemNecrosisProb, self.basalNecrosisProb,
                                               self.intermediateNecrosisProb, self.umbrellaNecrosisProb))

        return steppableList

    def _createExecConfig(self):
        return ExecConfig(MCSperDay=500,# SEED=10,
                          xLength=500, yLength=150, zLength=0, voxelDensity=.8)
<|endoftext|>


<commit_before>from twisted.internet import reactor
from twisted.internet.protocol import Factory, DatagramProtocol
from twisted.internet.error import ConnectionDone
from twisted.protocols.basic import LineOnlyReceiver, Int32StringReceiver
from carbon.cache import MetricCache
from carbon.relay import relay
from carbon.instrumentation import increment
from carbon.events import metricReceived
from carbon import log

try:
  import cPickle as pickle
except ImportError:
  import pickle

class LoggingMixin:
  def connectionMade(self):
    self.peer = self.transport.getPeer()
    self.peerAddr = "%s:%d" % (self.peer.host, self.peer.port)
    log.listener("%s connection with %s established" % (self.__class__.__name__, self.peerAddr))

  def connectionLost(self, reason):
    if reason.check(ConnectionDone):
      log.listener("%s connection with %s closed cleanly" % (self.__class__.__name__, self.peerAddr))
    else:
      log.listener("%s connection with %s lost: %s" % (self.__class__.__name__, self.peerAddr, reason.value))

class MetricLineReceiver(LoggingMixin, LineOnlyReceiver):
  delimiter = '\n'

  def lineReceived(self, line):
    try:
      metric, value, timestamp = line.strip().split()
      datapoint = ( float(timestamp), float(value) )
    except:
      log.listener('invalid line received from client %s, ignoring' % self.peerAddr)
      return

    increment('metricsReceived')
    metricReceived(metric, datapoint)

class MetricDatagramReceiver(LoggingMixin, DatagramProtocol):
  def datagramReceived(self, data, (host, port)):
    try:
      metric, value, timestamp = data.strip().split()
      datapoint = ( float(timestamp), float(value) )
    except:
      log.listener('invalid line received from client %s, ignoring' % host)
      return

    increment('metricsReceived')
    metricReceived(metric, datapoint)

class MetricPickleReceiver(LoggingMixin, Int32StringReceiver):
  MAX_LENGTH = 2 ** 20

  def stringReceived(self, data):
    try:
      datapoints = pickle.loads(data)
    except:
      log.listener('invalid pickle received from client %s, ignoring' % self.peerAddr)
      return

    for (metric, datapoint) in datapoints:
      try:
        datapoint = ( float(datapoint[0]), float(datapoint[1]) ) #force proper types
      except:
        continue

      if datapoint[1] == datapoint[1]: # filter out NaN values
        metricReceived(metric, datapoint)

    increment('metricsReceived', len(datapoints))

class CacheQueryHandler(LoggingMixin, Int32StringReceiver):
  def stringReceived(self, metric):
    values = MetricCache.get(metric, [])
    log.query('cache query for %s returned %d values' % (metric, len(values)))
    response = pickle.dumps(values, protocol=-1)
    self.sendString(response)
    increment('cacheQueries')

def startListener(interface, port, protocol):
  factory = Factory()
  factory.protocol = protocol
  return reactor.listenTCP( int(port), factory, interface=interface )
<commit_msg>UDP Reciever can accept multiple lines per datagram<commit_after>from twisted.internet import reactor
from twisted.internet.protocol import Factory, DatagramProtocol
from twisted.internet.error import ConnectionDone
from twisted.protocols.basic import LineOnlyReceiver, Int32StringReceiver
from carbon.cache import MetricCache
from carbon.relay import relay
from carbon.instrumentation import increment
from carbon.events import metricReceived
from carbon import log

try:
  import cPickle as pickle
except ImportError:
  import pickle

class LoggingMixin:
  def connectionMade(self):
    self.peer = self.transport.getPeer()
    self.peerAddr = "%s:%d" % (self.peer.host, self.peer.port)
    log.listener("%s connection with %s established" % (self.__class__.__name__, self.peerAddr))

  def connectionLost(self, reason):
    if reason.check(ConnectionDone):
      log.listener("%s connection with %s closed cleanly" % (self.__class__.__name__, self.peerAddr))
    else:
      log.listener("%s connection with %s lost: %s" % (self.__class__.__name__, self.peerAddr, reason.value))

class MetricLineReceiver(LoggingMixin, LineOnlyReceiver):
  delimiter = '\n'

  def lineReceived(self, line):
    try:
      metric, value, timestamp = line.strip().split()
      datapoint = ( float(timestamp), float(value) )
    except:
      log.listener('invalid line received from client %s, ignoring' % self.peerAddr)
      return

    increment('metricsReceived')
    metricReceived(metric, datapoint)

class MetricDatagramReceiver(LoggingMixin, DatagramProtocol):
  def datagramReceived(self, data, (host, port)):
    for line in data.splitlines():
      try:
        metric, value, timestamp = line.strip().split()
        datapoint = ( float(timestamp), float(value) )

        increment('metricsReceived')
        metricReceived(metric, datapoint)
      except:
        log.listener('invalid line received from client %s, ignoring' % host)

class MetricPickleReceiver(LoggingMixin, Int32StringReceiver):
  MAX_LENGTH = 2 ** 20

  def stringReceived(self, data):
    try:
      datapoints = pickle.loads(data)
    except:
      log.listener('invalid pickle received from client %s, ignoring' % self.peerAddr)
      return

    for (metric, datapoint) in datapoints:
      try:
        datapoint = ( float(datapoint[0]), float(datapoint[1]) ) #force proper types
      except:
        continue

      if datapoint[1] == datapoint[1]: # filter out NaN values
        metricReceived(metric, datapoint)

    increment('metricsReceived', len(datapoints))

class CacheQueryHandler(LoggingMixin, Int32StringReceiver):
  def stringReceived(self, metric):
    values = MetricCache.get(metric, [])
    log.query('cache query for %s returned %d values' % (metric, len(values)))
    response = pickle.dumps(values, protocol=-1)
    self.sendString(response)
    increment('cacheQueries')

def startListener(interface, port, protocol):
  factory = Factory()
  factory.protocol = protocol
  return reactor.listenTCP( int(port), factory, interface=interface )
<|endoftext|>