<commit_before># -*- coding: utf-8 -*-

'''
Some ad-hoc test - not very easy to reproduce, I'm afraid

Created on 1 Dec 2016

@author: meunier
'''
import os

import libxml2

import crf.Graph_MultiPageXml as Graph_MultiPageXml
from crf.NodeType_PageXml   import NodeType_PageXml

import crf.Edge as Edge
import xml_formats.PageXml as PageXml

def test_RectangleFitting():

    nt = NodeType_PageXml("TR"                   #some short prefix because labels below are prefixed with it
                              , ['catch-word', 'header', 'heading', 'marginalia', 'page-number']   #EXACTLY as in GT data!!!!
                              , []      #no ignored label/ One of those above or nothing, otherwise Exception!!
                              , True    #no label means OTHER
                              )
    nt.setXpathExpr( (".//pc:TextRegion"        #how to find the nodes
                      , "./pc:TextEquiv")       #how to get their text
                       )
    Graph_MultiPageXml.Graph_MultiPageXml.addNodeType(nt)

    print "- classes: ", Graph_MultiPageXml.Graph_MultiPageXml.getLabelNameList()

    obj = Graph_MultiPageXml.Graph_MultiPageXml()

    filename = os.path.join(os.path.dirname(__file__), "7749.mpxml")
    doc = libxml2.parseFile(filename)

    #load the block of each page, keeping the list of blocks of previous page
    lPrevPageNode = None

    for pnum, page, domNdPage in obj._iter_Page_DomNode(doc):
        #now that we have the page, let's create the node for each type!
        lPageNode = list()
        setPageNdDomId = set() #the set of DOM id
        # because the node types are supposed to have an empty intersection

        lPageNode = [nd for nodeType in obj.getNodeTypeList() for nd in nodeType._iter_GraphNode(doc, domNdPage, page) ]

        obj.lNode.extend(lPageNode)

        lPageEdge = Edge.Edge.computeEdges(lPrevPageNode, lPageNode)

        obj.lEdge.extend(lPageEdge)
        print "\tPage %5d    %6d nodes    %7d edges" %(pnum, len(lPageNode), len(lPageEdge))

        nc = 20
        fmt = "%%s p%%s %%%ss  --->  p%%s %%%ss"%(nc, nc)
        for edge in lPageEdge:
            sa, sb = edge.A.text, edge.B.text
            sa, sb = sa[:min(len(sa), nc)], sb[:min(len(sb), nc)]
            assert edge.A.node.prop("id") != edge.B.node.prop("id")
            print fmt%(edge.__class__.__name__[0], edge.A.pnum, sa, edge.B.pnum, sb)
        lPrevPageNode = lPageNode

    sOut = "TEST_getPageXmlBlock.mpxml"

    assert os.path.exists(sOut)

if __name__ == "__main__":
    test_RectangleFitting()

    <commit_msg>fixed test<commit_after># -*- coding: utf-8 -*-

'''
Some ad-hoc test - not very easy to reproduce, I'm afraid

Created on 1 Dec 2016

@author: meunier
'''
import os

import libxml2

import crf.Graph_MultiPageXml as Graph_MultiPageXml
from crf.NodeType_PageXml   import NodeType_PageXml

import crf.Edge as Edge
import xml_formats.PageXml as PageXml

def test_RectangleFitting():

    nt = NodeType_PageXml("TR"                   #some short prefix because labels below are prefixed with it
                              , ['catch-word', 'header', 'heading', 'marginalia', 'page-number']   #EXACTLY as in GT data!!!!
                              , []      #no ignored label/ One of those above or nothing, otherwise Exception!!
                              , True    #no label means OTHER
                              )
    nt.setXpathExpr( (".//pc:TextRegion"        #how to find the nodes
                      , "./pc:TextEquiv")       #how to get their text
                       )
    Graph_MultiPageXml.Graph_MultiPageXml.addNodeType(nt)

    print "- classes: ", Graph_MultiPageXml.Graph_MultiPageXml.getLabelNameList()

    obj = Graph_MultiPageXml.Graph_MultiPageXml()

    filename = os.path.join(os.path.dirname(__file__), "7749.mpxml")
    doc = libxml2.parseFile(filename)

    #load the block of each page, keeping the list of blocks of previous page
    lPrevPageNode = None

    for pnum, page, domNdPage in obj._iter_Page_DomNode(doc):
        #now that we have the page, let's create the node for each type!
        lPageNode = list()
        setPageNdDomId = set() #the set of DOM id
        # because the node types are supposed to have an empty intersection

        lPageNode = [nd for nodeType in obj.getNodeTypeList() for nd in nodeType._iter_GraphNode(doc, domNdPage, page) ]

        obj.lNode.extend(lPageNode)

        lPageEdge = Edge.Edge.computeEdges(lPrevPageNode, lPageNode)

        obj.lEdge.extend(lPageEdge)
        print "\tPage %5d    %6d nodes    %7d edges" %(pnum, len(lPageNode), len(lPageEdge))

        nc = 20
        fmt = "%%s p%%s %%%ss  --->  p%%s %%%ss"%(nc, nc)
        for edge in lPageEdge:
            sa, sb = edge.A.text, edge.B.text
            sa, sb = sa[:min(len(sa), nc)], sb[:min(len(sb), nc)]
            assert edge.A.node.prop("id") != edge.B.node.prop("id")
            print fmt%(edge.__class__.__name__[0], edge.A.pnum, sa, edge.B.pnum, sb)
        lPrevPageNode = lPageNode

#     sOut = "TEST_getPageXmlBlock.mpxml"
#
#     assert os.path.exists(sOut)

if __name__ == "__main__":
    test_RectangleFitting()

    <|endoftext|>


<commit_before>#############################################################################
##
## Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
## Contact: http://www.qt-project.org/legal
##
## This file is part of Qt Creator.
##
## Commercial License Usage
## Licensees holding valid commercial Qt licenses may use this file in
## accordance with the commercial license agreement provided with the
## Software or, alternatively, in accordance with the terms contained in
## a written agreement between you and Digia.  For licensing terms and
## conditions see http://qt.digia.com/licensing.  For further information
## use the contact form at http://qt.digia.com/contact-us.
##
## GNU Lesser General Public License Usage
## Alternatively, this file may be used under the terms of the GNU Lesser
## General Public License version 2.1 as published by the Free Software
## Foundation and appearing in the file LICENSE.LGPL included in the
## packaging of this file.  Please review the following information to
## ensure the GNU Lesser General Public License version 2.1 requirements
## will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
##
## In addition, as a special exception, Digia gives you certain additional
## rights.  These rights are described in the Digia Qt LGPL Exception
## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
##
#############################################################################

source("../../shared/qtcreator.py")

def main():
    projects = prepareTestExamples()
    if not projects:
        return
    sessionName = "SampleSession"
    startApplication("qtcreator" + SettingsPath)
    if not startedWithoutPluginError():
        return
    createAndSwitchToSession(sessionName)
    mainWindow = waitForObject(":Qt Creator_Core::Internal::MainWindow")
    test.verify(waitFor("sessionName in str(mainWindow.windowTitle)", 2000),
                "Verifying window title contains created session name.")
    for project in projects:
        openQmakeProject(project)
    progressBarWait(20000)
    checkNavigator(49, "Verifying whether all projects have been opened.")
    openDocument("propertyanimation.QML.qml.color-animation\\.qml")
    openDocument("declarative-music-browser.Headers.utility\\.h")
    checkOpenDocuments(2, "Verifying whether 2 files are open.")
    originalText = str(waitForObject(":Qt Creator_CppEditor::Internal::CPPEditorWidget").plainText)
    switchSession("default")
    test.verify(waitFor("'Qt Creator' == str(mainWindow.windowTitle)", 2000),
                "Verifying window title is set to default.")
    checkNavigator(1, "Verifying that no more project is opened.")
    checkOpenDocuments(0, "Verifying whether all files have been closed.")
    switchSession(sessionName)
    test.verify(waitFor("sessionName in str(mainWindow.windowTitle)", 2000),
                "Verifying window title contains created session name.")
    checkNavigator(49, "Verifying whether all projects have been re-opened.")
    checkOpenDocuments(2, "Verifying whether 2 files have been re-opened.")
    if test.verify("utility.h" in str(mainWindow.windowTitle),
                   "Verifying whether utility.h has been opened."):
        current = str(waitForObject(":Qt Creator_CppEditor::Internal::CPPEditorWidget").plainText)
        test.verify(originalText == current, "Verifying that same file has been opened.")
    checkForSessionFile(sessionName, projects)
    invokeMenuItem("File", "Exit")

def prepareTestExamples():
    examples = [os.path.join(sdkPath, "Examples", "4.7", "declarative", "animation", "basics",
                             "property-animation", "propertyanimation.pro"),
                os.path.join(sdkPath, "Examples", "QtMobility", "declarative-music-browser",
                             "declarative-music-browser.pro")
                ]
    projects = []
    for sourceExample in examples:
        if not neededFilePresent(sourceExample):
            return None
    # copy example projects to temp directory
    for sourceExample in examples:
        templateDir = prepareTemplate(os.path.dirname(sourceExample))
        projects.append(os.path.join(templateDir, os.path.basename(sourceExample)))
    return projects

def switchSession(toSession):
    test.log("Switching to session '%s'" % toSession)
    invokeMenuItem("File", "Session Manager...")
    clickItem(waitForObject("{name='sessionList' type='QListWidget' visible='1' "
                            "window=':Session Manager_ProjectExplorer::Internal::SessionDialog'}"),
                            toSession, 5, 5, 0, Qt.LeftButton)
    clickButton(waitForObject("{name='btSwitch' text='Switch to' type='QPushButton' visible='1' "
                              "window=':Session Manager_ProjectExplorer::Internal::SessionDialog'}"))

def createAndSwitchToSession(toSession):
    sessionInputDialog = ("{type='ProjectExplorer::Internal::SessionNameInputDialog' unnamed='1' "
                          "visible='1' windowTitle='New session name'}")
    test.log("Switching to session '%s' after creating it." % toSession)
    invokeMenuItem("File", "Session Manager...")
    clickButton(waitForObject("{name='btCreateNew' text='New' type='QPushButton' visible='1' "
                              "window=':Session Manager_ProjectExplorer::Internal::SessionDialog'}"))
    lineEdit = waitForObject("{type='QLineEdit' unnamed='1' visible='1' window=%s}"
                             % sessionInputDialog)
    replaceEditorContent(lineEdit, toSession)
    clickButton(waitForObject("{text='Switch to' type='QPushButton' unnamed='1' visible='1' "
                              "window=%s}" % sessionInputDialog))

def checkWelcomePage(sessionName, isCurrent=False):
    switchViewTo(ViewConstants.WELCOME)
    mouseClick(waitForObject("{clip='false' container=':Qt Creator_QDeclarativeView' enabled='true'"
                             " text='Develop' type='LinkedText' unnamed='1' visible='true'}"),
                             5, 5, 0, Qt.LeftButton)
    waitForObject("{clip='false' container=':Qt Creator_QDeclarativeView' enabled='true' "
                  "text='Sessions' type='Text' unnamed='1' visible='true'}")
    if isCurrent:
        sessions = ["default", "%s (current session)" % sessionName]
    else:
        sessions = ["default (current session)", sessionName]
    for sessionName in sessions:
        test.verify(object.exists("{clip='false' container=':Qt Creator_QDeclarativeView' "
                                  "enabled='true' type='LinkedText' "
                                  "unnamed='1' visible='true' text='%s'}" % sessionName),
                                  "Verifying session '%s' exists." % sessionName)

def checkNavigator(expectedRows, message):
    navigatorModel = waitForObject(":Qt Creator_Utils::NavigationTreeView").model()
    test.compare(expectedRows, len(__iterateChildren__(navigatorModel, QModelIndex())), message)

def checkOpenDocuments(expectedRows, message):
    selectFromCombo(":Qt Creator_Core::Internal::NavComboBox", "Open Documents")
    openDocsWidget = waitForObject(":OpenDocuments_Widget")
    test.compare(openDocsWidget.model().rowCount(), expectedRows, message)

def checkForSessionFile(sessionName, proFiles):
    global tmpSettingsDir
    sessionFile = os.path.join(tmpSettingsDir, "QtProject", "qtcreator", "%s.qws" % sessionName)
    if test.verify(os.path.exists(sessionFile),
                   "Verifying whether session file '%s' has been created." % sessionFile):
        content = readFile(sessionFile)
        for proFile in proFiles:
            if platform.system() in ('Microsoft', 'Windows'):
                proFile = proFile.replace('\\', '/')
            test.verify(proFile in content, "Verifying whether expected .pro file (%s) is listed "
                        "inside session file." % proFile)

def init():
    removeQmlDebugFolderIfExists()
<commit_msg>Squish: Fix tst_session_handling<commit_after>#############################################################################
##
## Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies).
## Contact: http://www.qt-project.org/legal
##
## This file is part of Qt Creator.
##
## Commercial License Usage
## Licensees holding valid commercial Qt licenses may use this file in
## accordance with the commercial license agreement provided with the
## Software or, alternatively, in accordance with the terms contained in
## a written agreement between you and Digia.  For licensing terms and
## conditions see http://qt.digia.com/licensing.  For further information
## use the contact form at http://qt.digia.com/contact-us.
##
## GNU Lesser General Public License Usage
## Alternatively, this file may be used under the terms of the GNU Lesser
## General Public License version 2.1 as published by the Free Software
## Foundation and appearing in the file LICENSE.LGPL included in the
## packaging of this file.  Please review the following information to
## ensure the GNU Lesser General Public License version 2.1 requirements
## will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
##
## In addition, as a special exception, Digia gives you certain additional
## rights.  These rights are described in the Digia Qt LGPL Exception
## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
##
#############################################################################

source("../../shared/qtcreator.py")

def main():
    projects = prepareTestExamples()
    if not projects:
        return
    sessionName = "SampleSession"
    startApplication("qtcreator" + SettingsPath)
    if not startedWithoutPluginError():
        return
    createAndSwitchToSession(sessionName)
    mainWindow = waitForObject(":Qt Creator_Core::Internal::MainWindow")
    test.verify(waitFor("sessionName in str(mainWindow.windowTitle)", 2000),
                "Verifying window title contains created session name.")
    for project in projects:
        openQmakeProject(project)
    progressBarWait(20000)
    checkNavigator(68, "Verifying whether all projects have been opened.")
    openDocument("propertyanimation.QML.qml.color-animation\\.qml")
    openDocument("declarative-music-browser.Headers.utility\\.h")
    checkOpenDocuments(2, "Verifying whether 2 files are open.")
    originalText = str(waitForObject(":Qt Creator_CppEditor::Internal::CPPEditorWidget").plainText)
    switchSession("default")
    test.verify(waitFor("'Qt Creator' == str(mainWindow.windowTitle)", 2000),
                "Verifying window title is set to default.")
    checkNavigator(1, "Verifying that no more project is opened.")
    checkOpenDocuments(0, "Verifying whether all files have been closed.")
    switchSession(sessionName)
    test.verify(waitFor("sessionName in str(mainWindow.windowTitle)", 2000),
                "Verifying window title contains created session name.")
    checkNavigator(68, "Verifying whether all projects have been re-opened.")
    checkOpenDocuments(2, "Verifying whether 2 files have been re-opened.")
    if test.verify("utility.h" in str(mainWindow.windowTitle),
                   "Verifying whether utility.h has been opened."):
        current = str(waitForObject(":Qt Creator_CppEditor::Internal::CPPEditorWidget").plainText)
        test.verify(originalText == current, "Verifying that same file has been opened.")
    checkForSessionFile(sessionName, projects)
    invokeMenuItem("File", "Exit")

def prepareTestExamples():
    examples = [os.path.join(sdkPath, "Examples", "4.7", "declarative", "animation", "basics",
                             "property-animation", "propertyanimation.pro"),
                os.path.join(sdkPath, "Examples", "QtMobility", "declarative-music-browser",
                             "declarative-music-browser.pro")
                ]
    projects = []
    for sourceExample in examples:
        if not neededFilePresent(sourceExample):
            return None
    # copy example projects to temp directory
    for sourceExample in examples:
        templateDir = prepareTemplate(os.path.dirname(sourceExample))
        projects.append(os.path.join(templateDir, os.path.basename(sourceExample)))
    return projects

def switchSession(toSession):
    test.log("Switching to session '%s'" % toSession)
    invokeMenuItem("File", "Session Manager...")
    clickItem(waitForObject("{name='sessionList' type='QListWidget' visible='1' "
                            "window=':Session Manager_ProjectExplorer::Internal::SessionDialog'}"),
                            toSession, 5, 5, 0, Qt.LeftButton)
    clickButton(waitForObject("{name='btSwitch' text='Switch to' type='QPushButton' visible='1' "
                              "window=':Session Manager_ProjectExplorer::Internal::SessionDialog'}"))

def createAndSwitchToSession(toSession):
    sessionInputDialog = ("{type='ProjectExplorer::Internal::SessionNameInputDialog' unnamed='1' "
                          "visible='1' windowTitle='New session name'}")
    test.log("Switching to session '%s' after creating it." % toSession)
    invokeMenuItem("File", "Session Manager...")
    clickButton(waitForObject("{name='btCreateNew' text='New' type='QPushButton' visible='1' "
                              "window=':Session Manager_ProjectExplorer::Internal::SessionDialog'}"))
    lineEdit = waitForObject("{type='QLineEdit' unnamed='1' visible='1' window=%s}"
                             % sessionInputDialog)
    replaceEditorContent(lineEdit, toSession)
    clickButton(waitForObject("{text='Switch to' type='QPushButton' unnamed='1' visible='1' "
                              "window=%s}" % sessionInputDialog))

def checkWelcomePage(sessionName, isCurrent=False):
    switchViewTo(ViewConstants.WELCOME)
    mouseClick(waitForObject("{clip='false' container=':Qt Creator_QDeclarativeView' enabled='true'"
                             " text='Develop' type='LinkedText' unnamed='1' visible='true'}"),
                             5, 5, 0, Qt.LeftButton)
    waitForObject("{clip='false' container=':Qt Creator_QDeclarativeView' enabled='true' "
                  "text='Sessions' type='Text' unnamed='1' visible='true'}")
    if isCurrent:
        sessions = ["default", "%s (current session)" % sessionName]
    else:
        sessions = ["default (current session)", sessionName]
    for sessionName in sessions:
        test.verify(object.exists("{clip='false' container=':Qt Creator_QDeclarativeView' "
                                  "enabled='true' type='LinkedText' "
                                  "unnamed='1' visible='true' text='%s'}" % sessionName),
                                  "Verifying session '%s' exists." % sessionName)

def checkNavigator(expectedRows, message):
    navigatorModel = waitForObject(":Qt Creator_Utils::NavigationTreeView").model()
    test.compare(expectedRows, len(__iterateChildren__(navigatorModel, QModelIndex())), message)

def checkOpenDocuments(expectedRows, message):
    selectFromCombo(":Qt Creator_Core::Internal::NavComboBox", "Open Documents")
    openDocsWidget = waitForObject(":OpenDocuments_Widget")
    test.compare(openDocsWidget.model().rowCount(), expectedRows, message)

def checkForSessionFile(sessionName, proFiles):
    global tmpSettingsDir
    sessionFile = os.path.join(tmpSettingsDir, "QtProject", "qtcreator", "%s.qws" % sessionName)
    if test.verify(os.path.exists(sessionFile),
                   "Verifying whether session file '%s' has been created." % sessionFile):
        content = readFile(sessionFile)
        for proFile in proFiles:
            if platform.system() in ('Microsoft', 'Windows'):
                proFile = proFile.replace('\\', '/')
            test.verify(proFile in content, "Verifying whether expected .pro file (%s) is listed "
                        "inside session file." % proFile)

def init():
    removeQmlDebugFolderIfExists()
<|endoftext|>


<commit_before>import asyncio
import logging
from asyncio import CancelledError

import pytest

import lightbus
import lightbus.path
from lightbus import EventMessage
from lightbus.utilities.async_tools import cancel
from tests.conftest import Worker

pytestmark = pytest.mark.reliability

logger = logging.getLogger(__name__)

@pytest.mark.asyncio
async def test_random_failures(
    bus: lightbus.path.BusPath, new_bus, caplog, redis_client, dummy_api, worker: Worker
):
    """Keep killing bus clients and check that we don't loose any events regardless"""

    caplog.set_level(logging.ERROR)
    event_ok_ids = dict()
    history = []

    async def listener(event_message: EventMessage, field, **kwargs):
        call_id = int(field)
        event_ok_ids.setdefault(call_id, 0)
        event_ok_ids[call_id] += 1
        await asyncio.sleep(0.1)

    # Put a lot of events onto the bus (we'll pull them off shortly)
    bus.client.register_api(dummy_api)
    for n in range(0, 50):
        await bus.my.dummy.my_event.fire_async(field=str(n))

    # Now pull the events off, and sometimes kill a worker early.
    # We kill 20% of listeners, so run 20% extra workers (we don't kill
    # any listeners in that extra 20% because these are just mop-up)
    for n in range(0, int(50 * 1.2)):
        cursed_bus = new_bus()
        cursed_bus.my.dummy.my_event.listen(
            listener, listener_name="test", bus_options={"since": "0"}
        )

        async with worker(cursed_bus):
            logger.debug(f"Worker {n}")
            await asyncio.sleep(0.05)
            if n % 5 == 0 and n < 50:
                # Cancel 1 in every 5 attempts at handling the event
                tasks = cursed_bus.client.event_client._event_listener_tasks
                await cancel(list(tasks)[0])
            await asyncio.sleep(0.15)

    await asyncio.sleep(0.1)

    info = await redis_client.xinfo_groups(stream="my.dummy.my_event:stream")

    duplicate_calls = [n for n, v in event_ok_ids.items() if v > 1]
    assert len(event_ok_ids) == 50, event_ok_ids
    assert len(duplicate_calls) > 0

    assert (
        len(info) == 1
    ), "There should only be one consumer group which was reused by every listener above"
    assert info[0][b"pending"] == 0
<commit_msg>Renaming test_random_failures to test_listener_failures<commit_after>import asyncio
import logging
from asyncio import CancelledError

import pytest

import lightbus
import lightbus.path
from lightbus import EventMessage
from lightbus.utilities.async_tools import cancel
from tests.conftest import Worker

pytestmark = pytest.mark.reliability

logger = logging.getLogger(__name__)

@pytest.mark.asyncio
async def test_listener_failures(
    bus: lightbus.path.BusPath, new_bus, caplog, redis_client, dummy_api, worker: Worker
):
    """Keep killing bus clients and check that we don't loose any events regardless"""

    caplog.set_level(logging.ERROR)
    event_ok_ids = dict()
    history = []

    async def listener(event_message: EventMessage, field, **kwargs):
        call_id = int(field)
        event_ok_ids.setdefault(call_id, 0)
        event_ok_ids[call_id] += 1
        await asyncio.sleep(0.1)

    # Put a lot of events onto the bus (we'll pull them off shortly)
    bus.client.register_api(dummy_api)
    for n in range(0, 50):
        await bus.my.dummy.my_event.fire_async(field=str(n))

    # Now pull the events off, and sometimes kill a worker early.
    # We kill 20% of listeners, so run 20% extra workers (we don't kill
    # any listeners in that extra 20% because these are just mop-up)
    for n in range(0, int(50 * 1.2)):
        cursed_bus = new_bus()
        cursed_bus.my.dummy.my_event.listen(
            listener, listener_name="test", bus_options={"since": "0"}
        )

        async with worker(cursed_bus):
            logger.debug(f"Worker {n}")
            await asyncio.sleep(0.05)
            if n % 5 == 0 and n < 50:
                # Cancel 1 in every 5 attempts at handling the event
                tasks = cursed_bus.client.event_client._event_listener_tasks
                await cancel(list(tasks)[0])
            await asyncio.sleep(0.15)

    await asyncio.sleep(0.1)

    info = await redis_client.xinfo_groups(stream="my.dummy.my_event:stream")

    duplicate_calls = [n for n, v in event_ok_ids.items() if v > 1]
    assert len(event_ok_ids) == 50, event_ok_ids
    assert len(duplicate_calls) > 0

    assert (
        len(info) == 1
    ), "There should only be one consumer group which was reused by every listener above"
    assert info[0][b"pending"] == 0
<|endoftext|>


<commit_before>import itertools
import random
import spacy
import numpy

from interfaces.SentenceOperation import SentenceOperation
from tasks.TaskTypes import TaskType
from SoundsLike.SoundsLike import Search
from spacy.attrs import LOWER, POS, ENT_TYPE, IS_ALPHA
from spacy.tokens import Doc

"""
Base Class for implementing the different input transformations a generation should be robust against.
"""

def close_homophones_swap(text, corrupt_prob, seed=0, max_output=1):
    random.seed(seed)
    nlp = spacy.load("en_core_web_sm")
    doc = nlp(text)
    perturbed_texts = []
    spaces = [True if tok.whitespace_ else False for tok in doc]
    for _ in range(max_output):
        perturbed_text = []
        for index, token in enumerate(doc):
            if random.uniform(0, 1) < corrupt_prob:
                try:
                    replacement = random.choice(Search.closeHomophones(token.text))
                    perturbed_text.append(replacement)
                except:
                    perturbed_text.append(token.text)
            else:
                perturbed_text.append(token.text)

        textbf = []
        for index, token in enumerate(perturbed_text):
            textbf.append(token)
            if spaces[index]:
                textbf.append(' ')
        perturbed_texts.append(''.join(textbf))
    return perturbed_texts

class CloseHomophonesSwap(SentenceOperation):
    tasks = [
        TaskType.TEXT_CLASSIFICATION,
        TaskType.TEXT_TO_TEXT_GENERATION,
        TaskType.TEXT_TAGGING,
    ]
    languages = ["en"]

    def __init__(self, seed=0, max_output=1):
        super().__init__(seed)
        self.max_output = max_output

    def generate(self, sentence: str):
        perturbed_texts = close_homophones_swap(
            text=sentence, corrupt_prob=0.5, seed=self.seed, max_output=self.max_output
        )
        return perturbed_texts

# Sample code to demonstrate usage. Can also assist in adding test cases.
# You don't need to keep this code in your transformation.
if __name__ == '__main__':
    import json
    from TestRunner import convert_to_snake_case

    tf = CloseHomophonesSwap(max_output=1)
    sentence = "Andrew finally returned the French book to Chris that I bought last week"
    test_cases = []
    for sentence in ["Andrew finally returned the French book to Chris that I bought last week",
                     "Sentences with gapping, such as Paul likes coffee and Mary tea, lack an overt predicate to indicate the relation between two or more arguments.",
                     "Alice in Wonderland is a 2010 American live-action/animated dark fantasy adventure film",
                     "Ujjal Dev Dosanjh served as 33rd Premier of British Columbia from 2000 to 2001",
                     "Neuroplasticity is a continuous processing allowing short-term, medium-term, and long-term remodeling of the neuronosynaptic organization."]:
        test_cases.append({
            "class": tf.name(),
            "inputs": {"sentence": sentence}, "outputs": [{"sentence": o} for o in tf.generate(sentence)]}
        )
    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
    print(json.dumps(json_file))
<commit_msg>remove extra code<commit_after>import itertools
import random
import spacy
import numpy

from interfaces.SentenceOperation import SentenceOperation
from tasks.TaskTypes import TaskType
from SoundsLike.SoundsLike import Search
from spacy.attrs import LOWER, POS, ENT_TYPE, IS_ALPHA
from spacy.tokens import Doc

"""
Base Class for implementing the different input transformations a generation should be robust against.
"""

def close_homophones_swap(text, corrupt_prob, seed=0, max_output=1):
    random.seed(seed)
    nlp = spacy.load("en_core_web_sm")
    doc = nlp(text)
    perturbed_texts = []
    spaces = [True if tok.whitespace_ else False for tok in doc]
    for _ in range(max_output):
        perturbed_text = []
        for index, token in enumerate(doc):
            if random.uniform(0, 1) < corrupt_prob:
                try:
                    replacement = random.choice(Search.closeHomophones(token.text))
                    perturbed_text.append(replacement)
                except:
                    perturbed_text.append(token.text)
            else:
                perturbed_text.append(token.text)

        textbf = []
        for index, token in enumerate(perturbed_text):
            textbf.append(token)
            if spaces[index]:
                textbf.append(' ')
        perturbed_texts.append(''.join(textbf))
    return perturbed_texts

class CloseHomophonesSwap(SentenceOperation):
    tasks = [
        TaskType.TEXT_CLASSIFICATION,
        TaskType.TEXT_TO_TEXT_GENERATION,
        TaskType.TEXT_TAGGING,
    ]
    languages = ["en"]

    def __init__(self, seed=0, max_output=1):
        super().__init__(seed)
        self.max_output = max_output

    def generate(self, sentence: str):
        perturbed_texts = close_homophones_swap(
            text=sentence, corrupt_prob=0.5, seed=self.seed, max_output=self.max_output
        )
        return perturbed_texts
<|endoftext|>


<commit_before>"""
.. _stats_cluster_sensors_2samp_spatial:

=====================================================
Spatiotemporal permutation F-test on full sensor data
=====================================================

Tests for differential evoked responses in at least
one condition using a permutation clustering test.
The FieldTrip neighbor templates will be used to determine
the adjacency between sensors. This serves as a spatial prior
to the clustering. Significant spatiotemporal clusters will then
be visualized using custom matplotlib code.
"""
# Authors: Denis Engemann <denis.engemann@gmail.com>
#
# License: BSD (3-clause)

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
from mne.viz import plot_topomap

import mne
from mne.stats import spatio_temporal_cluster_test
from mne.datasets import sample
from mne.channels import find_ch_connectivity

print(__doc__)

###############################################################################
# Set parameters
# --------------
data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw.fif'
event_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw-eve.fif'
event_id = {'Aud_L': 1, 'Aud_R': 2, 'Vis_L': 3, 'Vis_R': 4}
tmin = -0.2
tmax = 0.5

# Setup for reading the raw data
raw = mne.io.read_raw_fif(raw_fname, preload=True)
raw.filter(1, 30, fir_design='firwin')
events = mne.read_events(event_fname)

###############################################################################
# Read epochs for the channel of interest
# ---------------------------------------

picks = mne.pick_types(raw.info, meg='mag', eog=True)

reject = dict(mag=4e-12, eog=150e-6)
epochs = mne.Epochs(raw, events, event_id, tmin, tmax, picks=picks,
                    baseline=None, reject=reject, preload=True)

epochs.drop_channels(['EOG 061'])
epochs.equalize_event_counts(event_id)

condition_names = 'Aud_L', 'Aud_R', 'Vis_L', 'Vis_R'
X = [epochs[k].get_data() for k in condition_names]  # as 3D matrix
X = [np.transpose(x, (0, 2, 1)) for x in X]  # transpose for clustering

###############################################################################
# Find the FieldTrip neighbor definition to setup sensor connectivity
# -------------------------------------------------------------------
connectivity, ch_names = find_ch_connectivity(epochs.info, ch_type='mag')

print(type(connectivity))  # it's a sparse matrix!

plt.imshow(connectivity.toarray(), cmap='gray', origin='lower',
           interpolation='nearest')
plt.xlabel('{} Magnetometers'.format(len(ch_names)))
plt.ylabel('{} Magnetometers'.format(len(ch_names)))
plt.title('Between-sensor adjacency')

###############################################################################
# Compute permutation statistic
# -----------------------------
#
# How does it work? We use clustering to `bind` together features which are
# similar. Our features are the magnetic fields measured over our sensor
# array at different times. This reduces the multiple comparison problem.
# To compute the actual test-statistic, we first sum all F-values in all
# clusters. We end up with one statistic for each cluster.
# Then we generate a distribution from the data by shuffling our conditions
# between our samples and recomputing our clusters and the test statistics.
# We test for the significance of a given cluster by computing the probability
# of observing a cluster of that size. For more background read:
# Maris/Oostenveld (2007), "Nonparametric statistical testing of EEG- and
# MEG-data" Journal of Neuroscience Methods, Vol. 164, No. 1., pp. 177-190.
# doi:10.1016/j.jneumeth.2007.03.024

# set cluster threshold
threshold = 50.0  # very high, but the test is quite sensitive on this data
# set family-wise p-value
p_accept = 0.001

cluster_stats = spatio_temporal_cluster_test(X, n_permutations=1000,
                                             threshold=threshold, tail=1,
                                             n_jobs=1,
                                             connectivity=connectivity)

T_obs, clusters, p_values, _ = cluster_stats
good_cluster_inds = np.where(p_values < p_accept)[0]

###############################################################################
# Note. The same functions work with source estimate. The only differences
# are the origin of the data, the size, and the connectivity definition.
# It can be used for single trials or for groups of subjects.
#
# Visualize clusters
# ------------------

# configure variables for visualization
times = epochs.times * 1e3
colors = 'r', 'r', 'steelblue', 'steelblue'
linestyles = '-', '--', '-', '--'

# grand average as numpy arrray
grand_ave = np.array(X).mean(axis=1)

# get sensor positions via layout
pos = mne.find_layout(epochs.info).pos

# loop over significant clusters
for i_clu, clu_idx in enumerate(good_cluster_inds):
    # unpack cluster information, get unique indices
    time_inds, space_inds = np.squeeze(clusters[clu_idx])
    ch_inds = np.unique(space_inds)
    time_inds = np.unique(time_inds)

    # get topography for F stat
    f_map = T_obs[time_inds, ...].mean(axis=0)

    # get signals at significant sensors
    signals = grand_ave[..., ch_inds].mean(axis=-1)
    sig_times = times[time_inds]

    # create spatial mask
    mask = np.zeros((f_map.shape[0], 1), dtype=bool)
    mask[ch_inds, :] = True

    # initialize figure
    fig, ax_topo = plt.subplots(1, 1, figsize=(10, 3))
    title = 'Cluster #{0}'.format(i_clu + 1)
    fig.suptitle(title, fontsize=14)

    # plot average test statistic and mark significant sensors
    image, _ = plot_topomap(f_map, pos, mask=mask, axes=ax_topo,
                            cmap='Reds', vmin=np.min, vmax=np.max)

    # advanced matplotlib for showing image with figure and colorbar
    # in one plot
    divider = make_axes_locatable(ax_topo)

    # add axes for colorbar
    ax_colorbar = divider.append_axes('right', size='5%', pad=0.05)
    plt.colorbar(image, cax=ax_colorbar)
    ax_topo.set_xlabel('Averaged F-map ({:0.1f} - {:0.1f} ms)'.format(
        *sig_times[[0, -1]]
    ))

    # add new axis for time courses and plot time courses
    ax_signals = divider.append_axes('right', size='300%', pad=1.2)
    for signal, name, col, ls in zip(signals, condition_names, colors,
                                     linestyles):
        ax_signals.plot(times, signal, color=col, linestyle=ls, label=name)

    # add information
    ax_signals.axvline(0, color='k', linestyle=':', label='stimulus onset')
    ax_signals.set_xlim([times[0], times[-1]])
    ax_signals.set_xlabel('time [ms]')
    ax_signals.set_ylabel('evoked magnetic fields [fT]')

    # plot significant time range
    ymin, ymax = ax_signals.get_ylim()
    ax_signals.fill_betweenx((ymin, ymax), sig_times[0], sig_times[-1],
                             color='orange', alpha=0.3)
    ax_signals.legend(loc='lower right')
    ax_signals.set_ylim(ymin, ymax)

    # clean up viz
    mne.viz.tight_layout(fig=fig)
    fig.subplots_adjust(bottom=.05)
    plt.show()

###############################################################################
# Exercises
# ----------
#
# - What is the smallest p-value you can obtain, given the finite number of
#    permutations?
# - use an F distribution to compute the threshold by traditional significance
#    levels. Hint: take a look at ``scipy.stats.distributions.f``
<commit_msg>changed p_value to find significant clusters to plot (#4929)<commit_after>"""
.. _stats_cluster_sensors_2samp_spatial:

=====================================================
Spatiotemporal permutation F-test on full sensor data
=====================================================

Tests for differential evoked responses in at least
one condition using a permutation clustering test.
The FieldTrip neighbor templates will be used to determine
the adjacency between sensors. This serves as a spatial prior
to the clustering. Significant spatiotemporal clusters will then
be visualized using custom matplotlib code.
"""
# Authors: Denis Engemann <denis.engemann@gmail.com>
#
# License: BSD (3-clause)

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
from mne.viz import plot_topomap

import mne
from mne.stats import spatio_temporal_cluster_test
from mne.datasets import sample
from mne.channels import find_ch_connectivity

print(__doc__)

###############################################################################
# Set parameters
# --------------
data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw.fif'
event_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw-eve.fif'
event_id = {'Aud_L': 1, 'Aud_R': 2, 'Vis_L': 3, 'Vis_R': 4}
tmin = -0.2
tmax = 0.5

# Setup for reading the raw data
raw = mne.io.read_raw_fif(raw_fname, preload=True)
raw.filter(1, 30, fir_design='firwin')
events = mne.read_events(event_fname)

###############################################################################
# Read epochs for the channel of interest
# ---------------------------------------

picks = mne.pick_types(raw.info, meg='mag', eog=True)

reject = dict(mag=4e-12, eog=150e-6)
epochs = mne.Epochs(raw, events, event_id, tmin, tmax, picks=picks,
                    baseline=None, reject=reject, preload=True)

epochs.drop_channels(['EOG 061'])
epochs.equalize_event_counts(event_id)

condition_names = 'Aud_L', 'Aud_R', 'Vis_L', 'Vis_R'
X = [epochs[k].get_data() for k in condition_names]  # as 3D matrix
X = [np.transpose(x, (0, 2, 1)) for x in X]  # transpose for clustering

###############################################################################
# Find the FieldTrip neighbor definition to setup sensor connectivity
# -------------------------------------------------------------------
connectivity, ch_names = find_ch_connectivity(epochs.info, ch_type='mag')

print(type(connectivity))  # it's a sparse matrix!

plt.imshow(connectivity.toarray(), cmap='gray', origin='lower',
           interpolation='nearest')
plt.xlabel('{} Magnetometers'.format(len(ch_names)))
plt.ylabel('{} Magnetometers'.format(len(ch_names)))
plt.title('Between-sensor adjacency')

###############################################################################
# Compute permutation statistic
# -----------------------------
#
# How does it work? We use clustering to `bind` together features which are
# similar. Our features are the magnetic fields measured over our sensor
# array at different times. This reduces the multiple comparison problem.
# To compute the actual test-statistic, we first sum all F-values in all
# clusters. We end up with one statistic for each cluster.
# Then we generate a distribution from the data by shuffling our conditions
# between our samples and recomputing our clusters and the test statistics.
# We test for the significance of a given cluster by computing the probability
# of observing a cluster of that size. For more background read:
# Maris/Oostenveld (2007), "Nonparametric statistical testing of EEG- and
# MEG-data" Journal of Neuroscience Methods, Vol. 164, No. 1., pp. 177-190.
# doi:10.1016/j.jneumeth.2007.03.024

# set cluster threshold
threshold = 50.0  # very high, but the test is quite sensitive on this data
# set family-wise p-value
p_accept = 0.01

cluster_stats = spatio_temporal_cluster_test(X, n_permutations=1000,
                                             threshold=threshold, tail=1,
                                             n_jobs=1,
                                             connectivity=connectivity)

T_obs, clusters, p_values, _ = cluster_stats
good_cluster_inds = np.where(p_values < p_accept)[0]

###############################################################################
# Note. The same functions work with source estimate. The only differences
# are the origin of the data, the size, and the connectivity definition.
# It can be used for single trials or for groups of subjects.
#
# Visualize clusters
# ------------------

# configure variables for visualization
times = epochs.times * 1e3
colors = 'r', 'r', 'steelblue', 'steelblue'
linestyles = '-', '--', '-', '--'

# grand average as numpy arrray
grand_ave = np.array(X).mean(axis=1)

# get sensor positions via layout
pos = mne.find_layout(epochs.info).pos

# loop over significant clusters
for i_clu, clu_idx in enumerate(good_cluster_inds):
    # unpack cluster information, get unique indices
    time_inds, space_inds = np.squeeze(clusters[clu_idx])
    ch_inds = np.unique(space_inds)
    time_inds = np.unique(time_inds)

    # get topography for F stat
    f_map = T_obs[time_inds, ...].mean(axis=0)

    # get signals at significant sensors
    signals = grand_ave[..., ch_inds].mean(axis=-1)
    sig_times = times[time_inds]

    # create spatial mask
    mask = np.zeros((f_map.shape[0], 1), dtype=bool)
    mask[ch_inds, :] = True

    # initialize figure
    fig, ax_topo = plt.subplots(1, 1, figsize=(10, 3))
    title = 'Cluster #{0}'.format(i_clu + 1)
    fig.suptitle(title, fontsize=14)

    # plot average test statistic and mark significant sensors
    image, _ = plot_topomap(f_map, pos, mask=mask, axes=ax_topo,
                            cmap='Reds', vmin=np.min, vmax=np.max)

    # advanced matplotlib for showing image with figure and colorbar
    # in one plot
    divider = make_axes_locatable(ax_topo)

    # add axes for colorbar
    ax_colorbar = divider.append_axes('right', size='5%', pad=0.05)
    plt.colorbar(image, cax=ax_colorbar)
    ax_topo.set_xlabel('Averaged F-map ({:0.1f} - {:0.1f} ms)'.format(
        *sig_times[[0, -1]]
    ))

    # add new axis for time courses and plot time courses
    ax_signals = divider.append_axes('right', size='300%', pad=1.2)
    for signal, name, col, ls in zip(signals, condition_names, colors,
                                     linestyles):
        ax_signals.plot(times, signal, color=col, linestyle=ls, label=name)

    # add information
    ax_signals.axvline(0, color='k', linestyle=':', label='stimulus onset')
    ax_signals.set_xlim([times[0], times[-1]])
    ax_signals.set_xlabel('time [ms]')
    ax_signals.set_ylabel('evoked magnetic fields [fT]')

    # plot significant time range
    ymin, ymax = ax_signals.get_ylim()
    ax_signals.fill_betweenx((ymin, ymax), sig_times[0], sig_times[-1],
                             color='orange', alpha=0.3)
    ax_signals.legend(loc='lower right')
    ax_signals.set_ylim(ymin, ymax)

    # clean up viz
    mne.viz.tight_layout(fig=fig)
    fig.subplots_adjust(bottom=.05)
    plt.show()

###############################################################################
# Exercises
# ----------
#
# - What is the smallest p-value you can obtain, given the finite number of
#    permutations?
# - use an F distribution to compute the threshold by traditional significance
#    levels. Hint: take a look at ``scipy.stats.distributions.f``
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
# Copyright © 2011 Clément Schaff, Mahdi Ben Jelloul

"""
openFisca, Logiciel libre de simulation du système socio-fiscal français
Copyright © 2011 Clément Schaff, Mahdi Ben Jelloul

This file is part of openFisca.

    openFisca is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    openFisca is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with openFisca.  If not, see <http://www.gnu.org/licenses/>.
"""

from __future__ import division
import numpy as np
from Config import CONF
from datetime import datetime

class Column(object):
    """
    DataSet data item

    `label` : (str, unicode)
    `default` : any type, optional
    """
    count = 0
    def __init__(self, label = None, default=None):
        self._order = Column.count
        Column.count += 1
        self._name = None
        self._nrows = None
        self._value = None
        self._label = label
        self._default = default
        self._dtype = None

    def set_name(self, name):
        self._name = name

    def get_name(self):
        return self._name

    def set_nrows(self, nrows):
        self._nrows = nrows

    def _init_value(self, nrows):
        self._nrows = nrows
        self._value = np.ones(nrows, dtype = self._dtype)*self._default

    def get_value(self, index = None, opt = None, dflt = 0, sum_ = False):
        '''
        method to read the value in an array
        index is a dict with the coordinates of each person in the array
            - if index is none, returns the whole column (every person)
            - if index is not none, return an array of length len(unit)
        opt is a dict with the id of the person for which you want the value
            - if opt is None, returns the value for the person 0 (i.e. 'vous' for 'foy', 'chef' for 'fam', 'pref' for 'men')
            - if opt is not None, return a dict with key 'person' and values for this person
        '''
        var = self._value
        if index is None:
            return var
        nb = index['nb']
        if self._dtype == np.bool: dflt= (dflt != 0)
        if opt is None:
            temp = np.ones(nb, dtype = self._dtype)*dflt
            idx = index[0]
            temp[idx['idxUnit']] = var[idx['idxIndi']]
            return temp
        else:
            out = {}
            for person in opt:
                temp = np.ones(nb, dtype = self._dtype)*dflt
                idx = index[person]
                temp[idx['idxUnit']] = var[idx['idxIndi']]
                out[person] = temp
            if sum_ is False:
                return out
            else:
                sumout = 0
                for val in out.itervalues():
                    sumout += val
                return sumout

    def set_value(self, value, index, opt = None):
        if value.dtype == np.bool:
            self._dtype = np.bool

        if opt is None:
        # TODO: check if it's realy right?
            idx = index[0]
        else:
            idx = index[opt]
        var = np.array(self._value, dtype = self._dtype)
        val = np.array(value, dtype = self._dtype)
        var[idx['idxIndi']] = val[idx['idxUnit']]
        self._value = var

    def __str__(self):
        return '%s' % self._name

class DataTableMeta(type):
    """
    DataTable metaclass

    Create class attribute `_columns`: list of the DataTable class attributes,
    created in the same order as these attributes were written
    """
    def __new__(cls, name, bases, dct):
        columns = {}
        for base in bases:
            if getattr(base, "__metaclass__", None) is DataTableMeta:
                for column in base._columns:
                    columns[column._name] = column

        for attrname, value in dct.items():
            if isinstance( value, Column ):
                value.set_name(attrname)
                if attrname in columns:
                    value._order = columns[attrname]._order
                columns[attrname] = value
        columns_list = columns.values()
        columns_list.sort(key=lambda x:x._order)
        dct["_columns"] = columns_list
        return type.__new__(cls, name, bases, dct)

class DataTable(object):
    """
    Construct a SystemSf object is a set of Prestation objects
        * title [string]
        * comment [string]: text shown on the top of the first data item
    """
    __metaclass__ = DataTableMeta
    def __init__(self, nmen, title=None, comment=None):
        self._nmen = nmen
        self._nrows = nmen
        self.__title = title
        self.__comment = comment
        self.currentline = {}
        self.count = 0

        comp_title, comp_comment = self._compute_title_and_comment()
        if title is None:   self.__title = comp_title
        if comment is None: self.__comment = comp_comment

        self.col_names = set()
        self._isPopulated = False
        self.datesim = datetime.strptime(CONF.get('simulation', 'datesim') ,"%Y-%m-%d").date()
        self.NMEN = CONF.get('simulation', 'nmen')
        self.MAXREV = CONF.get('simulation', 'maxrev')
        self.XAXIS = CONF.get('simulation', 'xaxis')

    def _init_columns(self, nrows):
        for column in self._columns:
            self.col_names.add(column.get_name())
            column._init_value(nrows)
            column._isCalculated = False

    def _compute_title_and_comment(self):
        """
        Private method to compute title and comment of the data set
        """
        comp_title = self.__class__.__name__
        comp_comment = None
        if self.__doc__:
            doc_lines = self.__doc__.splitlines()
            # Remove empty lines at the begining of comment
            while doc_lines and not doc_lines[0].strip():
                del doc_lines[0]
            if doc_lines:
                comp_title = doc_lines.pop(0).strip()
            if doc_lines:
                comp_comment = "\n".join([x.strip() for x in doc_lines])
        return comp_title, comp_comment

    def get_title(self):
        """
        Return data set title
        """
        return self.__title

    def get_comment(self):
        """
        Return data set comment
        """
        return self.__comment

    def gen_index(self, units):
        if not self._isPopulated:
            raise Exception('Table should be populated from a scenario or external data')

        self.index = {'ind': {0: {'idxIndi':np.arange(self._nrows),
                                  'idxUnit':np.arange(self._nrows)},
                              'nb': self._nrows},
                      'noi': {}}
        dct = self.index['noi']
        nois = self.noi.get_value()
        listnoi = np.unique(nois)
        for noi in listnoi:
            idxIndi = np.sort(np.squeeze((np.argwhere(nois == noi))))
            idxUnit = np.searchsorted(listnoi, nois[idxIndi])
            temp = {'idxIndi':idxIndi, 'idxUnit':idxUnit}
            dct.update({noi: temp})

        for unit in units:
            try:
                idx = getattr(self, 'id'+unit).get_value()
                quicol = getattr(self, 'qui'+unit)
                qui = quicol.get_value()
                enum = quicol.enum
            except:
                raise Exception('DataTable needs columns %s and %s to build index with unit %s' %
                          ('id' + unit, 'qui' + unit, unit))

            self.index[unit] = {}
            dct = self.index[unit]
            idxlist = np.unique(idx)
            dct['nb'] = len(idxlist)

            for person in enum.itervalues():
                idxIndi = np.sort(np.squeeze((np.argwhere(qui == person))))
                idxUnit = np.searchsorted(idxlist, idx[idxIndi])
                temp = {'idxIndi':idxIndi, 'idxUnit':idxUnit}
                dct.update({person: temp})

    def populate_from_scenario(self, scenario):
        self._nrows = self._nmen*len(scenario.indiv)
        self._init_columns(self._nrows)
        XAXIS = self.XAXIS
        MAXREV = self.MAXREV
        datesim = self.datesim
        # pour l'instant, un seul menage répliqué n fois
        for noi, dct in scenario.indiv.iteritems():
            birth = dct['birth']
            age = datesim.year- birth.year
            agem = 12*(datesim.year- birth.year) + datesim.month - birth.month
            noipref = dct['noipref']
            noidec = dct['noidec']
            noichef = dct['noichef']
            quifoy = self.quifoy.enum[dct['quifoy']]
            quifam = self.quifam.enum[dct['quifam']]
            quimen = self.quimen.enum[dct['quimen']]
            self.addPerson(noi, age, agem, quifoy, quifam, quimen, noidec, noichef, noipref)
        self._isPopulated = True

        self.gen_index(['foy', 'fam', 'men'])

        index = self.index['noi']
        for noi, dct in scenario.indiv.iteritems():
            for var, val in dct.iteritems():
                if var in ('birth', 'noipref', 'noidec', 'noichef', 'quifoy', 'quimen', 'quifam'): continue
                col = getattr(self, var)
                if not index[noi] is None:
                    col.set_value(np.ones(self._nmen)*val, index, noi)

        # set xaxis
        # TODO: how to set xaxis vals properly
        if self._nmen>1:
            col = getattr(self, XAXIS)
            vls = np.linspace(0, MAXREV, self._nmen)
            col.set_value(vls, {0:{'idxIndi': index[0]['idxIndi'], 'idxUnit': index[0]['idxIndi']}})

    def addPerson(self, noi, age, agem, quifoy, quifam, quimen, noidec, noichef, noipref):
        for i in xrange(self._nmen):
            indiv = self.row()
            indiv['noi']   = noi
            indiv['age']   = age
            indiv['agem']   = agem
            indiv['quifoy'] = quifoy
            indiv['quifam'] = quifam
            indiv['quimen'] = quimen
            indiv['idmen'] = 60000 + i + 1
            indiv['idfoy'] = indiv['idmen']*100 + noidec
            indiv['idfam'] = indiv['idmen']*100 + noichef
            self.append()

    def append(self):
        if self.count >= self._nrows:
            raise Exception('La table est pleine')
        for key, val in self.currentline.iteritems():
            a = getattr(self, key).get_value()

            a[self.count] = val
        self.currentline = {}
        self.count += 1

    def row(self):
        return self.currentline

    def __str__(self):
        return self.to_string(debug=True)

    def to_string(self, debug=False, indent=None, align=False):
        """
        Return readable string representation of the data set
        If debug is True, add more details on data items
        """
        if indent is None:
            indent = "\n    "
        txt = self.__title+":"
        def _get_label(column):
            if debug:
                return column._name
            else:
                return column.get_prop_value("display", self, "label")
        length = 0
        if align:
            for column in self._columns:
                column_length = len(_get_label(column))
                if column_length > length:
                    length = column_length
        for column in self._columns:
            if debug:
                label = column._name
            else:
                label = column.get_prop_value("display", self, "label")
            if length:
                label = label.ljust(length)

            txt += indent+label+": "
            if debug:
                txt += column.__class__.__name__
        return txt

class IntCol(Column):
    '''
    A column of integer
    '''
    def __init__(self, label = None, default = 0):
        super(IntCol, self).__init__(label, default)
        self._dtype = np.int32

class EnumCol(IntCol):
    '''
    A column of integer
    '''
    def __init__(self, enum, label = None, default = 0):
        super(EnumCol, self).__init__(label, default)
        self.enum = enum

class BoolCol(Column):
    '''
    A column of boolean
    '''
    def __init__(self, label = None, default = False):
        super(BoolCol, self).__init__(label, default)
        self._dtype = np.bool

    def _init_value(self, nrows):
        self._nrows = nrows
        self._value = np.zeros(nrows, dtype = np.bool) | self._default

class FloatCol(Column):
    '''
    A column of float 32
    '''
    def __init__(self, label = None, default = 0):
        super(FloatCol, self).__init__(label, default)
        self._dtype = np.float32

class AgesCol(IntCol):
    '''
    A column of Int to store ages of people
    '''
    def __init__(self, label = None, default = 0):
        super(AgesCol, self).__init__(label, default)

    def get_value(self, index = None, opt = None, dflt = 0):
        return super(AgesCol, self).get_value(index, opt, dflt = -9999)

class DateCol(Column):
    '''
    A column of Int to store ages of people
    '''
    def __init__(self, label = None, default = 0):
        super(DateCol, self).__init__(label, default)
        self._dtype = np.datetime64

    def _init_value(self, nrows):
        self._nrows = nrows
        self._value = np.ones(nrows, dtype = self._dtype)
<commit_msg>added menage vars<commit_after># -*- coding:utf-8 -*-
# Copyright © 2011 Clément Schaff, Mahdi Ben Jelloul

"""
openFisca, Logiciel libre de simulation du système socio-fiscal français
Copyright © 2011 Clément Schaff, Mahdi Ben Jelloul

This file is part of openFisca.

    openFisca is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    openFisca is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with openFisca.  If not, see <http://www.gnu.org/licenses/>.
"""

from __future__ import division
import numpy as np
from Config import CONF
from datetime import datetime

class Column(object):
    """
    DataSet data item

    `label` : (str, unicode)
    `default` : any type, optional
    """
    count = 0
    def __init__(self, label = None, default=None):
        self._order = Column.count
        Column.count += 1
        self._name = None
        self._nrows = None
        self._value = None
        self._label = label
        self._default = default
        self._dtype = None

    def set_name(self, name):
        self._name = name

    def get_name(self):
        return self._name

    def set_nrows(self, nrows):
        self._nrows = nrows

    def _init_value(self, nrows):
        self._nrows = nrows
        self._value = np.ones(nrows, dtype = self._dtype)*self._default

    def get_value(self, index = None, opt = None, dflt = 0, sum_ = False):
        '''
        method to read the value in an array
        index is a dict with the coordinates of each person in the array
            - if index is none, returns the whole column (every person)
            - if index is not none, return an array of length len(unit)
        opt is a dict with the id of the person for which you want the value
            - if opt is None, returns the value for the person 0 (i.e. 'vous' for 'foy', 'chef' for 'fam', 'pref' for 'men')
            - if opt is not None, return a dict with key 'person' and values for this person
        '''
        var = self._value
        if index is None:
            return var
        nb = index['nb']
        if self._dtype == np.bool: dflt= (dflt != 0)
        if opt is None:
            temp = np.ones(nb, dtype = self._dtype)*dflt
            idx = index[0]
            temp[idx['idxUnit']] = var[idx['idxIndi']]
            return temp
        else:
            out = {}
            for person in opt:
                temp = np.ones(nb, dtype = self._dtype)*dflt
                idx = index[person]
                temp[idx['idxUnit']] = var[idx['idxIndi']]
                out[person] = temp
            if sum_ is False:
                return out
            else:
                sumout = 0
                for val in out.itervalues():
                    sumout += val
                return sumout

    def set_value(self, value, index, opt = None):
        if value.dtype == np.bool:
            self._dtype = np.bool

        if opt is None:
        # TODO: check if it's realy right?
            idx = index[0]
        else:
            idx = index[opt]
        var = np.array(self._value, dtype = self._dtype)
        val = np.array(value, dtype = self._dtype)
        var[idx['idxIndi']] = val[idx['idxUnit']]
        self._value = var

    def __str__(self):
        return '%s' % self._name

class DataTableMeta(type):
    """
    DataTable metaclass

    Create class attribute `_columns`: list of the DataTable class attributes,
    created in the same order as these attributes were written
    """
    def __new__(cls, name, bases, dct):
        columns = {}
        for base in bases:
            if getattr(base, "__metaclass__", None) is DataTableMeta:
                for column in base._columns:
                    columns[column._name] = column

        for attrname, value in dct.items():
            if isinstance( value, Column ):
                value.set_name(attrname)
                if attrname in columns:
                    value._order = columns[attrname]._order
                columns[attrname] = value
        columns_list = columns.values()
        columns_list.sort(key=lambda x:x._order)
        dct["_columns"] = columns_list
        return type.__new__(cls, name, bases, dct)

class DataTable(object):
    """
    Construct a SystemSf object is a set of Prestation objects
        * title [string]
        * comment [string]: text shown on the top of the first data item
    """
    __metaclass__ = DataTableMeta
    def __init__(self, nmen, title=None, comment=None):
        self._nmen = nmen
        self._nrows = nmen
        self.__title = title
        self.__comment = comment
        self.currentline = {}
        self.count = 0

        comp_title, comp_comment = self._compute_title_and_comment()
        if title is None:   self.__title = comp_title
        if comment is None: self.__comment = comp_comment

        self.col_names = set()
        self._isPopulated = False
        self.datesim = datetime.strptime(CONF.get('simulation', 'datesim') ,"%Y-%m-%d").date()
        self.NMEN = CONF.get('simulation', 'nmen')
        self.MAXREV = CONF.get('simulation', 'maxrev')
        self.XAXIS = CONF.get('simulation', 'xaxis')

    def _init_columns(self, nrows):
        for column in self._columns:
            self.col_names.add(column.get_name())
            column._init_value(nrows)
            column._isCalculated = False

    def _compute_title_and_comment(self):
        """
        Private method to compute title and comment of the data set
        """
        comp_title = self.__class__.__name__
        comp_comment = None
        if self.__doc__:
            doc_lines = self.__doc__.splitlines()
            # Remove empty lines at the begining of comment
            while doc_lines and not doc_lines[0].strip():
                del doc_lines[0]
            if doc_lines:
                comp_title = doc_lines.pop(0).strip()
            if doc_lines:
                comp_comment = "\n".join([x.strip() for x in doc_lines])
        return comp_title, comp_comment

    def get_title(self):
        """
        Return data set title
        """
        return self.__title

    def get_comment(self):
        """
        Return data set comment
        """
        return self.__comment

    def gen_index(self, units):
        if not self._isPopulated:
            raise Exception('Table should be populated from a scenario or external data')

        self.index = {'ind': {0: {'idxIndi':np.arange(self._nrows),
                                  'idxUnit':np.arange(self._nrows)},
                              'nb': self._nrows},
                      'noi': {}}
        dct = self.index['noi']
        nois = self.noi.get_value()
        listnoi = np.unique(nois)
        for noi in listnoi:
            idxIndi = np.sort(np.squeeze((np.argwhere(nois == noi))))
            idxUnit = np.searchsorted(listnoi, nois[idxIndi])
            temp = {'idxIndi':idxIndi, 'idxUnit':idxUnit}
            dct.update({noi: temp})

        for unit in units:
            try:
                idx = getattr(self, 'id'+unit).get_value()
                quicol = getattr(self, 'qui'+unit)
                qui = quicol.get_value()
                enum = quicol.enum
            except:
                raise Exception('DataTable needs columns %s and %s to build index with unit %s' %
                          ('id' + unit, 'qui' + unit, unit))

            self.index[unit] = {}
            dct = self.index[unit]
            idxlist = np.unique(idx)
            dct['nb'] = len(idxlist)

            for person in enum.itervalues():
                idxIndi = np.sort(np.squeeze((np.argwhere(qui == person))))
                idxUnit = np.searchsorted(idxlist, idx[idxIndi])
                temp = {'idxIndi':idxIndi, 'idxUnit':idxUnit}
                dct.update({person: temp})

    def populate_from_scenario(self, scenario):
        self._nrows = self._nmen*len(scenario.indiv)
        self._init_columns(self._nrows)
        XAXIS = self.XAXIS
        MAXREV = self.MAXREV
        datesim = self.datesim
        # pour l'instant, un seul menage répliqué n fois
        for noi, dct in scenario.indiv.iteritems():
            birth = dct['birth']
            age = datesim.year- birth.year
            agem = 12*(datesim.year- birth.year) + datesim.month - birth.month
            noipref = dct['noipref']
            noidec = dct['noidec']
            noichef = dct['noichef']
            quifoy = self.quifoy.enum[dct['quifoy']]
            quifam = self.quifam.enum[dct['quifam']]
            quimen = self.quimen.enum[dct['quimen']]
            self.addPerson(noi, age, agem, quifoy, quifam, quimen, noidec, noichef, noipref)
        self._isPopulated = True

        self.gen_index(['foy', 'fam', 'men'])

        index = self.index['noi']
        for noi, dct in scenario.indiv.iteritems():
            for var, val in dct.iteritems():
                if var in ('birth', 'noipref', 'noidec', 'noichef', 'quifoy', 'quimen', 'quifam'): continue
                col = getattr(self, var)
                if not index[noi] is None:
                    col.set_value(np.ones(self._nmen)*val, index, noi)

        for noi, dct in scenario.menage.iteritems():
            for var, val in dct.iteritems():
                col = getattr(self, var)
                if not index[noi] is None:
                    col.set_value(np.ones(self._nmen)*val, index, noi)

        # set xaxis
        # TODO: how to set xaxis vals properly
        if self._nmen>1:
            col = getattr(self, XAXIS)
            vls = np.linspace(0, MAXREV, self._nmen)
            col.set_value(vls, {0:{'idxIndi': index[0]['idxIndi'], 'idxUnit': index[0]['idxIndi']}})

    def addPerson(self, noi, age, agem, quifoy, quifam, quimen, noidec, noichef, noipref):
        for i in xrange(self._nmen):
            indiv = self.row()
            indiv['noi']   = noi
            indiv['age']   = age
            indiv['agem']   = agem
            indiv['quifoy'] = quifoy
            indiv['quifam'] = quifam
            indiv['quimen'] = quimen
            indiv['idmen'] = 60000 + i + 1
            indiv['idfoy'] = indiv['idmen']*100 + noidec
            indiv['idfam'] = indiv['idmen']*100 + noichef
            self.append()

    def append(self):
        if self.count >= self._nrows:
            raise Exception('La table est pleine')
        for key, val in self.currentline.iteritems():
            a = getattr(self, key).get_value()

            a[self.count] = val
        self.currentline = {}
        self.count += 1

    def row(self):
        return self.currentline

    def __str__(self):
        return self.to_string(debug=True)

    def to_string(self, debug=False, indent=None, align=False):
        """
        Return readable string representation of the data set
        If debug is True, add more details on data items
        """
        if indent is None:
            indent = "\n    "
        txt = self.__title+":"
        def _get_label(column):
            if debug:
                return column._name
            else:
                return column.get_prop_value("display", self, "label")
        length = 0
        if align:
            for column in self._columns:
                column_length = len(_get_label(column))
                if column_length > length:
                    length = column_length
        for column in self._columns:
            if debug:
                label = column._name
            else:
                label = column.get_prop_value("display", self, "label")
            if length:
                label = label.ljust(length)

            txt += indent+label+": "
            if debug:
                txt += column.__class__.__name__
        return txt

class IntCol(Column):
    '''
    A column of integer
    '''
    def __init__(self, label = None, default = 0):
        super(IntCol, self).__init__(label, default)
        self._dtype = np.int32

class EnumCol(IntCol):
    '''
    A column of integer
    '''
    def __init__(self, enum, label = None, default = 0):
        super(EnumCol, self).__init__(label, default)
        self.enum = enum

class BoolCol(Column):
    '''
    A column of boolean
    '''
    def __init__(self, label = None, default = False):
        super(BoolCol, self).__init__(label, default)
        self._dtype = np.bool

    def _init_value(self, nrows):
        self._nrows = nrows
        self._value = np.zeros(nrows, dtype = np.bool) | self._default

class FloatCol(Column):
    '''
    A column of float 32
    '''
    def __init__(self, label = None, default = 0):
        super(FloatCol, self).__init__(label, default)
        self._dtype = np.float32

class AgesCol(IntCol):
    '''
    A column of Int to store ages of people
    '''
    def __init__(self, label = None, default = 0):
        super(AgesCol, self).__init__(label, default)

    def get_value(self, index = None, opt = None, dflt = 0):
        return super(AgesCol, self).get_value(index, opt, dflt = -9999)

class DateCol(Column):
    '''
    A column of Int to store ages of people
    '''
    def __init__(self, label = None, default = 0):
        super(DateCol, self).__init__(label, default)
        self._dtype = np.datetime64

    def _init_value(self, nrows):
        self._nrows = nrows
        self._value = np.ones(nrows, dtype = self._dtype)
<|endoftext|>


<commit_before>import re
import string

from . import base
from . import mixins

from datetime import date

APOSTROPHE_LETTER_REGEX = re.compile("([a-z])'([A-Z])")
DIGIT_LETTER_REGEX = re.compile(r'\d([A-Z])')

def better_title(value):
    """Convert a string into titlecase."""
    m = ' '.join(word.strip(string.punctuation) for word in value.split())
    t = APOSTROPHE_LETTER_REGEX.sub(
        lambda m: m.group(0).lower(), m.title())
    t = DIGIT_LETTER_REGEX.sub(lambda m: m.group(0).lower(), t)

    return ' '.join(t.split())

class TransformedRecord(
        mixins.GenericCompensationMixin,
        mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
        mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': 'Last Name',
        'first_name': 'First Name',
        'department': 'Department Title',
        'job_title': 'Contract Title',
        'hire_date': 'Hire Date',
        'compensation': 'Salary/Hourly or Daily Rate',
        'gender': 'Sex',
        'nationality': 'Race',
        'employee_type': 'Part Time',
    }

    # The order of the name fields to build a full name.
    # If `full_name` is in MAP, you don't need this at all.
    NAME_FIELDS = ('first_name', 'last_name', )

    # The name of the organization this WILL SHOW UP ON THE SITE, so double check it!
    ORGANIZATION_NAME = 'Cypress-Fairbanks ISD'

    # What type of organization is this? This MUST match what we use on the site, double check against salaries.texastribune.org
    ORGANIZATION_CLASSIFICATION = 'School District'

    # How would you describe the compensation field? We try to respect how they use their system.
    # description = 'Annual salary'

    # When did you receive the data? NOT when we added it to the site.
    DATE_PROVIDED = date(2017, 5, 9)

    # The URL to find the raw data in our S3 bucket.
    URL = ('https://s3.amazonaws.com/raw.texastribune.org/'
           'cypress_fairbanks_isd/salaries/2017-05/cypress-fairbanks-isd.xlsx')

    # How do they track gender? We need to map what they use to `F` and `M`.
    gender_map = {'F': 'F', 'M': 'M'}

    race_map = {
        'I': 'American Indian or Alaskan',
        'W': 'White',
        'H': 'Hispanic',
        'A': 'Asian or Pacific Islander',
        'B': 'Black or African American',
        'O': 'Other',
    }

    # This is how the loader checks for valid people.
    # Defaults to checking to see if `last_name` is empty.
    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.compensation.strip() != '-' and self.last_name.strip() != ''

    @property
    def person(self):
        name = self.get_name()
        r = {
            'family_name': name.last,
            'given_name': name.first,
            # 'additional_name': name.middle,
            'name': unicode(name),
            'gender': self.gender_map[self.gender.strip()]
        }

        return r

    @property
    def description(self):
        status = self.get_mapped_value('employee_type').strip()
        department = self.get_mapped_value('department').strip()
        salary = float(self.get_mapped_value('compensation'))
        # If the employee isn't a sub, their rate is more than $100 and
        # they're a full-time employee, their pay is their Annual salary
        if department != 'SUBSTITUTE' and salary > 100.00 and status == 'F':
            return 'Annual salary'
        # If the employee isn't a sub, their salary is more than $100, and
        # then their pay is their Part-time salary
        elif department != 'SUBSTITUTE' and salary > 100.00:
            return 'Part-time salary'
        # If the employee isn't a sub and their salary is less than $100,
        # then their pay is an hourly rate
        elif department != 'SUBSTITUTE' and salary < 100.00:
            return 'Hourly rate'
        # If the employee is a sub, their salary is a daily rate
        elif department == 'SUBSTITUTE':
            return 'Daily rate'

    @property
    def job_title(self):
        jobTitle = self.get_mapped_value('job_title').strip()
        department = self.get_mapped_value('department').strip()
        substitute = 'Substitute'
        print better_title(jobTitle)
        if department == 'SUBSTITUTE' and jobTitle == '':
            return substitute
        else:
            return better_title(jobTitle)

    @property
    def race(self):
        return {
            'name': self.race_map[self.nationality.strip()]
        }

    @property
    def compensation_type(self):
        status = self.get_mapped_value('employee_type').strip()
        department = self.get_mapped_value('department').strip()
        salary = float(self.get_mapped_value('compensation'))
        # If the employee isn't a sub, their rate is more than $100 and
        # they're a full-time employee, their pay is their Annual salary
        if department != 'SUBSTITUTE' and salary > 100.00 and status == 'F':
            return 'FT'
        # If the employee isn't a sub, their salary is more than $100, and
        # then their pay is their Part-time salary
        elif department != 'SUBSTITUTE' and salary > 100.00:
            return 'PT'
        # If the employee isn't a sub and their salary is less than $100,
        # then their pay is an hourly rate
        elif department != 'SUBSTITUTE' and salary < 100.00:
            return 'PT'
        # If the employee is a sub, their salary is a daily rate
        elif department == 'SUBSTITUTE':
            return 'PT'

    def calculate_tenure(self):
        hire_date_data = map(int, self.hire_date.split('-'))
        hire_date = date(hire_date_data[0], hire_date_data[1],
                         hire_date_data[2])
        tenure = float((self.DATE_PROVIDED - hire_date).days) / float(360)
        if tenure < 0:
            tenure = 0
        return tenure

transform = base.transform_factory(TransformedRecord)
<commit_msg>Remove leftover print statement<commit_after>import re
import string

from . import base
from . import mixins

from datetime import date

APOSTROPHE_LETTER_REGEX = re.compile("([a-z])'([A-Z])")
DIGIT_LETTER_REGEX = re.compile(r'\d([A-Z])')

def better_title(value):
    """Convert a string into titlecase."""
    m = ' '.join(word.strip(string.punctuation) for word in value.split())
    t = APOSTROPHE_LETTER_REGEX.sub(
        lambda m: m.group(0).lower(), m.title())
    t = DIGIT_LETTER_REGEX.sub(lambda m: m.group(0).lower(), t)

    return ' '.join(t.split())

class TransformedRecord(
        mixins.GenericCompensationMixin,
        mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
        mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': 'Last Name',
        'first_name': 'First Name',
        'department': 'Department Title',
        'job_title': 'Contract Title',
        'hire_date': 'Hire Date',
        'compensation': 'Salary/Hourly or Daily Rate',
        'gender': 'Sex',
        'nationality': 'Race',
        'employee_type': 'Part Time',
    }

    # The order of the name fields to build a full name.
    # If `full_name` is in MAP, you don't need this at all.
    NAME_FIELDS = ('first_name', 'last_name', )

    # The name of the organization this WILL SHOW UP ON THE SITE, so double check it!
    ORGANIZATION_NAME = 'Cypress-Fairbanks ISD'

    # What type of organization is this? This MUST match what we use on the site, double check against salaries.texastribune.org
    ORGANIZATION_CLASSIFICATION = 'School District'

    # How would you describe the compensation field? We try to respect how they use their system.
    # description = 'Annual salary'

    # When did you receive the data? NOT when we added it to the site.
    DATE_PROVIDED = date(2017, 5, 9)

    # The URL to find the raw data in our S3 bucket.
    URL = ('https://s3.amazonaws.com/raw.texastribune.org/'
           'cypress_fairbanks_isd/salaries/2017-05/cypress-fairbanks-isd.xlsx')

    # How do they track gender? We need to map what they use to `F` and `M`.
    gender_map = {'F': 'F', 'M': 'M'}

    race_map = {
        'I': 'American Indian or Alaskan',
        'W': 'White',
        'H': 'Hispanic',
        'A': 'Asian or Pacific Islander',
        'B': 'Black or African American',
        'O': 'Other',
    }

    # This is how the loader checks for valid people.
    # Defaults to checking to see if `last_name` is empty.
    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.compensation.strip() != '-' and self.last_name.strip() != ''

    @property
    def person(self):
        name = self.get_name()
        r = {
            'family_name': name.last,
            'given_name': name.first,
            # 'additional_name': name.middle,
            'name': unicode(name),
            'gender': self.gender_map[self.gender.strip()]
        }

        return r

    @property
    def description(self):
        status = self.get_mapped_value('employee_type').strip()
        department = self.get_mapped_value('department').strip()
        salary = float(self.get_mapped_value('compensation'))
        # If the employee isn't a sub, their rate is more than $100 and
        # they're a full-time employee, their pay is their Annual salary
        if department != 'SUBSTITUTE' and salary > 100.00 and status == 'F':
            return 'Annual salary'
        # If the employee isn't a sub, their salary is more than $100, and
        # then their pay is their Part-time salary
        elif department != 'SUBSTITUTE' and salary > 100.00:
            return 'Part-time salary'
        # If the employee isn't a sub and their salary is less than $100,
        # then their pay is an hourly rate
        elif department != 'SUBSTITUTE' and salary < 100.00:
            return 'Hourly rate'
        # If the employee is a sub, their salary is a daily rate
        elif department == 'SUBSTITUTE':
            return 'Daily rate'

    @property
    def job_title(self):
        jobTitle = self.get_mapped_value('job_title').strip()
        department = self.get_mapped_value('department').strip()
        substitute = 'Substitute'
        if department == 'SUBSTITUTE' and jobTitle == '':
            return substitute
        else:
            return better_title(jobTitle)

    @property
    def race(self):
        return {
            'name': self.race_map[self.nationality.strip()]
        }

    @property
    def compensation_type(self):
        status = self.get_mapped_value('employee_type').strip()
        department = self.get_mapped_value('department').strip()
        salary = float(self.get_mapped_value('compensation'))
        # If the employee isn't a sub, their rate is more than $100 and
        # they're a full-time employee, their pay is their Annual salary
        if department != 'SUBSTITUTE' and salary > 100.00 and status == 'F':
            return 'FT'
        # If the employee isn't a sub, their salary is more than $100, and
        # then their pay is their Part-time salary
        elif department != 'SUBSTITUTE' and salary > 100.00:
            return 'PT'
        # If the employee isn't a sub and their salary is less than $100,
        # then their pay is an hourly rate
        elif department != 'SUBSTITUTE' and salary < 100.00:
            return 'PT'
        # If the employee is a sub, their salary is a daily rate
        elif department == 'SUBSTITUTE':
            return 'PT'

    def calculate_tenure(self):
        hire_date_data = map(int, self.hire_date.split('-'))
        hire_date = date(hire_date_data[0], hire_date_data[1],
                         hire_date_data[2])
        tenure = float((self.DATE_PROVIDED - hire_date).days) / float(360)
        if tenure < 0:
            tenure = 0
        return tenure

transform = base.transform_factory(TransformedRecord)
<|endoftext|>