<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon Nov  6 14:02:13 2017

@author: a002028
"""

import os
import sys
current_path = os.path.dirname(os.path.realpath(__file__))[:-4]
sys.path.append(current_path)

import pandas as pd
import numpy as np
import time

import utils
import core

"""
#==============================================================================
#==============================================================================
"""
class BooleanFilter(object):
    """
    Kan detta vara en idee....
    """
    def __init__(self):
        self.current_level = 0
        self.current_subset = 0

    def add_filter(self):
        pass

"""
#==============================================================================
#==============================================================================
"""
class IndexHandler(object):
    """
    - Ska kunna ta emot filterobject
    - Summera index arrayer
    - Utgår från föregående boolean för det specifika subsetet..
    - Pratar med DataHandler och dess DataFrame för att plocka fram boolean
    - select by columns

    Dictionary structure
    - step_0
    - Subset (optional name)
    - step_1
    - step_2
    - Water Body (optional name)
    - Indicator (optional name)
    """
    def __init__(self, workspace_object=None, data_handler_object=None):
        self.workspace_object = workspace_object
        self.data_handler_object = data_handler_object
        self._initiate_attributes()

    #==========================================================================
    def _add_boolean_to_dict(self, *args, filter_object=None, df=None): #  **kwargs ?
        """
        *args: step_0, subset, step_1, step_2, water_body, indicator
        Uses bool_dict as a dynamic reference to a specific part of self.booleans
        """
        bool_dict = self.booleans
        for key in reversed(args):
            if key:
                use_key=key
                break

        for key in args:
            if key == use_key:
                if bool_dict.get('boolean') != None:
                    # Merge boolean from parent with new boolean from filter_object
                    bool_dict[key]['boolean'] = bool_dict.get('boolean') & filter_object.get_filter_boolean_for_df(df)
                else:
                    # No boolean from parent, use new boolean from filter_object
                    bool_dict[key]['boolean'] = filter_object.get_filter_boolean_for_df(df)

                break
            else:
                # "dynamic reference" to a specific part of self.booleans
                bool_dict = bool_dict[key]

    #==========================================================================
    def _get_boolean(self, *args):
        """
        *args: step_0, subset, step_1, step_2, water_body, indicator
        - Loop through *args, save a reference for the specific dictionary
          within self.booleans
        - Return boolean for the specific argument input
        """
        bool_dict = self.booleans.copy()
        for key in args:
            if key and key in bool_dict:
                bool_dict = bool_dict[key]
            else:
                break

        return bool_dict.get('boolean')

    #==========================================================================
    def _get_steps(self, step=''):
        if step=='step_0':
            return step, None, None
        elif step=='step_1':
            return 'step_0', step, None
        elif step=='step_2':
            return 'step_0', 'step_1', step
        else:
            raise UserWarning('Step definition is incorrect. Acceptable step is step_x, where x equals 0-2')

    #==========================================================================
    def _initiate_attributes(self):
        """
        - The intended structure of self.booleans is as follows:
        {'step_0':{'boolean':bool,
                   'subset_A':{'boolean':bool,
                               'step_1':{'boolean':bool,
                                         'step_2':{'boolean':bool,
                                                   'water_body_X':{'boolean':bool,
                                                                   'indicator_X':{'boolean':bool}
                                                                   }
                                                   'water_body_Y':{'boolean':bool,
                                                                   'indicator_X':{'boolean':bool}
                                                                   }
                                                   }
                                         }
                               }
                   'subset_B':{'boolean':bool,
                               'step_1':{'boolean':bool,
                                         'step_2':{'boolean':bool,
                                                   'water_body_X':{'boolean':bool,
                                                                   'indicator_X':{'boolean':bool}
                                                                   }
                                                   }
                                         }
                               }
                   }
        }
        """
        self.booleans = {}

    #==========================================================================
    def _reset_boolean(self):
        """
        - Sets up a dictionary including key 'boolean' containing None
        """
        return {'boolean':None}

    #==========================================================================
    def _set_dict(self, *args):
        """
        *args: step_0, subset, step_1, step_2, water_body, indicator
        - Set reference of master booleans dictionary, bool_dict.
          bool_dict will change dynamically depending on key..
        - Sets up a dictionary including key 'boolean' containing None
        """
        bool_dict = self.booleans
        for key in args:
            if key and key not in bool_dict:
                bool_dict[key] = self._reset_boolean()
                break
            elif key:
                bool_dict = bool_dict[key]
            else:
                break

    #==========================================================================
    def add_filter(self, filter_object=None, subset=None, step=None, water_body=None, indicator=None):
        """
        - Get reference of master DataFrame, df
        - Set up step sequence: step_0, step_1, step_2
        - Set up a dictionary including key 'boolean' containing None
        - Add boolean to the given subset, step, water_body or indicator

        If water_body is given: subset and step must also be given
        """
        df = self.data_handler_object.get_all_column_data_df()

        step_0, step_1, step_2 = self._get_steps(step=step)

        self._set_dict(step_0, subset, step_1, step_2, water_body, indicator)

        self._add_boolean_to_dict(step_0, subset, step_1, step_2, water_body, indicator,
                                  filter_object=filter_object,
                                  df=df)

    #==========================================================================
    def get_filtered_data(self, subset=None, step=None, water_body=None, indicator=None):
        """
        Returns filtered data for the given boolean specification
        """
        step_0, step_1, step_2 = self._get_steps(step=step)

        boolean = self._get_boolean(step_0, subset, step_1, step_2, water_body, indicator)

        return self.data_handler_object.get_all_column_data_df(boolean_filter=boolean)

    #==========================================================================
    def reset_booleans(self, subset=None, step=None, water_body=None, indicator=None):
        """
        All keys that shall be kept should be specified in *args.
        When a key doesnt exists we reset the dict for the key before and break
        """
        step_0, step_1, step_2 = self._get_steps(step=step)
        args = [step_0, subset, step_1, step_2, water_body, indicator]

        bool_dict = self.booleans
        for key in args:
            if key:
                bool_dict = bool_dict[key]
            else:
                bool_dict = self._reset_boolean()
                break

    #==========================================================================

if __name__ == '__main__':
    print('='*50)
    print('Running module "index_handler.py"')
    print('-'*50)
    print('')

    raw_data_file_path = 'D:/Utveckling/g_EKOSTAT_tool/test_data/raw_data/data_BAS_2000-2009.txt'
    first_filter_directory = 'D:/Utveckling/g_EKOSTAT_tool/test_data/filtered_data'

    # Handler
#    raw_data = core.DataHandler('raw')
#    raw_data.add_txt_file(raw_data_file_path, data_type='column')

    print('-'*50)
    print('done')
    print('-'*50)<commit_msg>change method name in index_handler<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon Nov  6 14:02:13 2017

@author: a002028
"""

import os
import sys
current_path = os.path.dirname(os.path.realpath(__file__))[:-4]
sys.path.append(current_path)

import pandas as pd
import numpy as np
import time

import utils
import core

"""
#==============================================================================
#==============================================================================
"""
class BooleanFilter(object):
    """
    Kan detta vara en idee....
    """
    def __init__(self):
        self.current_level = 0
        self.current_subset = 0

    def add_filter(self):
        pass

"""
#==============================================================================
#==============================================================================
"""
class IndexHandler(object):
    """
    - Ska kunna ta emot filterobject
    - Summera index arrayer
    - Utgår från föregående boolean för det specifika subsetet..
    - Pratar med DataHandler och dess DataFrame för att plocka fram boolean
    - select by columns

    Dictionary structure
    - step_0
    - Subset (optional name)
    - step_1
    - step_2
    - Water Body (optional name)
    - Indicator (optional name)
    """
    def __init__(self, workspace_object=None, data_handler_object=None):
        self.workspace_object = workspace_object
        self.data_handler_object = data_handler_object
        self._initiate_attributes()

    #==========================================================================
    def _add_boolean_to_dict(self, *args, filter_object=None, df=None): #  **kwargs ?
        """
        *args: step_0, subset, step_1, step_2, water_body, indicator
        Uses bool_dict as a dynamic reference to a specific part of self.booleans
        """
        bool_dict = self.booleans
        for key in reversed(args):
            if key:
                use_key=key
                break

        for key in args:
            if key == use_key:
                if bool_dict.get('boolean') != None:
                    # Merge boolean from parent with new boolean from filter_object
                    bool_dict[key]['boolean'] = bool_dict.get('boolean') & filter_object.get_filter_boolean_for_df(df)
                else:
                    # No boolean from parent, use new boolean from filter_object
                    bool_dict[key]['boolean'] = filter_object.get_filter_boolean_for_df(df)

                break
            else:
                # "dynamic reference" to a specific part of self.booleans
                bool_dict = bool_dict[key]

    #==========================================================================
    def _get_boolean(self, *args):
        """
        *args: step_0, subset, step_1, step_2, water_body, indicator
        - Loop through *args, save a reference for the specific dictionary
          within self.booleans
        - Return boolean for the specific argument input
        """
        bool_dict = self.booleans.copy()
        for key in args:
            if key and key in bool_dict:
                bool_dict = bool_dict[key]
            else:
                break

        return bool_dict.get('boolean')

    #==========================================================================
    def _get_steps(self, step=''):
        if step=='step_0':
            return step, None, None
        elif step=='step_1':
            return 'step_0', step, None
        elif step=='step_2':
            return 'step_0', 'step_1', step
        else:
            raise UserWarning('Step definition is incorrect. Acceptable step is step_x, where x equals 0-2')

    #==========================================================================
    def _initiate_attributes(self):
        """
        - The intended structure of self.booleans is as follows:
        {'step_0':{'boolean':bool,
                   'subset_A':{'boolean':bool,
                               'step_1':{'boolean':bool,
                                         'step_2':{'boolean':bool,
                                                   'water_body_X':{'boolean':bool,
                                                                   'indicator_X':{'boolean':bool}
                                                                   }
                                                   'water_body_Y':{'boolean':bool,
                                                                   'indicator_X':{'boolean':bool}
                                                                   }
                                                   }
                                         }
                               }
                   'subset_B':{'boolean':bool,
                               'step_1':{'boolean':bool,
                                         'step_2':{'boolean':bool,
                                                   'water_body_X':{'boolean':bool,
                                                                   'indicator_X':{'boolean':bool}
                                                                   }
                                                   }
                                         }
                               }
                   }
        }
        """
        self.booleans = {}

    #==========================================================================
    def _get_default_boolean(self):
        """
        - Sets up a dictionary including key 'boolean' containing None
        """
        return {'boolean':None}

    #==========================================================================
    def _set_dict(self, *args):
        """
        *args: step_0, subset, step_1, step_2, water_body, indicator
        - Set reference of master booleans dictionary, bool_dict.
          bool_dict will change dynamically depending on key..
        - Sets up a dictionary including key 'boolean' containing None
        """
        bool_dict = self.booleans
        for key in args:
            if key and key not in bool_dict:
                bool_dict[key] = self._get_default_boolean()
                break
            elif key:
                bool_dict = bool_dict[key]
            else:
                break

    #==========================================================================
    def add_filter(self, filter_object=None, subset=None, step=None, water_body=None, indicator=None):
        """
        - Get reference of master DataFrame, df
        - Set up step sequence: step_0, step_1, step_2
        - Set up a dictionary including key 'boolean' containing None
        - Add boolean to the given subset, step, water_body or indicator

        If water_body is given: subset and step must also be given
        """
        df = self.data_handler_object.get_all_column_data_df()

        step_0, step_1, step_2 = self._get_steps(step=step)

        self._set_dict(step_0, subset, step_1, step_2, water_body, indicator)

        self._add_boolean_to_dict(step_0, subset, step_1, step_2, water_body, indicator,
                                  filter_object=filter_object,
                                  df=df)

    #==========================================================================
    def get_filtered_data(self, subset=None, step=None, water_body=None, indicator=None):
        """
        Returns filtered data for the given boolean specification
        """
        step_0, step_1, step_2 = self._get_steps(step=step)

        boolean = self._get_boolean(step_0, subset, step_1, step_2, water_body, indicator)

        return self.data_handler_object.get_all_column_data_df(boolean_filter=boolean)

    #==========================================================================
    def reset_booleans(self, subset=None, step=None, water_body=None, indicator=None):
        """
        All keys that shall be kept should be specified in *args.
        When a key doesnt exists we reset the dict for the key before and break
        """
        step_0, step_1, step_2 = self._get_steps(step=step)
        args = [step_0, subset, step_1, step_2, water_body, indicator]

        bool_dict = self.booleans
        for key in args:
            if key:
                bool_dict = bool_dict[key]
            else:
                bool_dict = self._get_default_boolean()
                break

    #==========================================================================

if __name__ == '__main__':
    print('='*50)
    print('Running module "index_handler.py"')
    print('-'*50)
    print('')

    raw_data_file_path = 'D:/Utveckling/g_EKOSTAT_tool/test_data/raw_data/data_BAS_2000-2009.txt'
    first_filter_directory = 'D:/Utveckling/g_EKOSTAT_tool/test_data/filtered_data'

    # Handler
#    raw_data = core.DataHandler('raw')
#    raw_data.add_txt_file(raw_data_file_path, data_type='column')

    print('-'*50)
    print('done')
    print('-'*50)<|endoftext|>


<commit_before>"""Test the System Bridge config flow."""
from unittest.mock import patch

from aiohttp.client_exceptions import ClientConnectionError
from systembridge.exceptions import BridgeAuthenticationException

from homeassistant import config_entries, data_entry_flow
from homeassistant.components import zeroconf
from homeassistant.components.system_bridge.const import DOMAIN
from homeassistant.const import CONF_API_KEY, CONF_HOST, CONF_PORT

from tests.common import MockConfigEntry

FIXTURE_MAC_ADDRESS = "aa:bb:cc:dd:ee:ff"
FIXTURE_UUID = "e91bf575-56f3-4c83-8f42-70ac17adcd33"

FIXTURE_AUTH_INPUT = {CONF_API_KEY: "abc-123-def-456-ghi"}

FIXTURE_USER_INPUT = {
    CONF_API_KEY: "abc-123-def-456-ghi",
    CONF_HOST: "test-bridge",
    CONF_PORT: "9170",
}

FIXTURE_ZEROCONF_INPUT = {
    CONF_API_KEY: "abc-123-def-456-ghi",
    CONF_HOST: "1.1.1.1",
    CONF_PORT: "9170",
}

FIXTURE_ZEROCONF = zeroconf.ZeroconfServiceInfo(
    host="1.1.1.1",
    port=9170,
    hostname="test-bridge.local.",
    type="_system-bridge._udp.local.",
    name="System Bridge - test-bridge._system-bridge._udp.local.",
    properties={
        "address": "http://test-bridge:9170",
        "fqdn": "test-bridge",
        "host": "test-bridge",
        "ip": "1.1.1.1",
        "mac": FIXTURE_MAC_ADDRESS,
        "port": "9170",
        "uuid": FIXTURE_UUID,
    },
)

FIXTURE_ZEROCONF_BAD = zeroconf.ZeroconfServiceInfo(
    host="1.1.1.1",
    port=9170,
    hostname="test-bridge.local.",
    type="_system-bridge._udp.local.",
    name="System Bridge - test-bridge._system-bridge._udp.local.",
    properties={
        "something": "bad",
    },
)

FIXTURE_INFORMATION = {
    "address": "http://test-bridge:9170",
    "apiPort": 9170,
    "fqdn": "test-bridge",
    "host": "test-bridge",
    "ip": "1.1.1.1",
    "mac": FIXTURE_MAC_ADDRESS,
    "updates": {
        "available": False,
        "newer": False,
        "url": "https://github.com/timmo001/system-bridge/releases/tag/v2.3.2",
        "version": {"current": "2.3.2", "new": "2.3.2"},
    },
    "uuid": FIXTURE_UUID,
    "version": "2.3.2",
    "websocketAddress": "ws://test-bridge:9172",
    "websocketPort": 9172,
}

FIXTURE_BASE_URL = (
    f"http://{FIXTURE_USER_INPUT[CONF_HOST]}:{FIXTURE_USER_INPUT[CONF_PORT]}"
)

FIXTURE_ZEROCONF_BASE_URL = (
    f"http://{FIXTURE_ZEROCONF[CONF_HOST]}:{FIXTURE_ZEROCONF[CONF_PORT]}"
)

async def test_user_flow(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test full user flow."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": config_entries.SOURCE_USER}
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["errors"] is None

    aioclient_mock.get(
        f"{FIXTURE_BASE_URL}/information",
        headers={"Content-Type": "application/json"},
        json=FIXTURE_INFORMATION,
    )

    with patch(
        "homeassistant.components.system_bridge.async_setup_entry",
        return_value=True,
    ) as mock_setup_entry:
        result2 = await hass.config_entries.flow.async_configure(
            result["flow_id"], FIXTURE_USER_INPUT
        )
        await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY
    assert result2["title"] == "test-bridge"
    assert result2["data"] == FIXTURE_USER_INPUT
    assert len(mock_setup_entry.mock_calls) == 1

async def test_form_invalid_auth(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we handle invalid auth."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": config_entries.SOURCE_USER}
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["errors"] is None

    aioclient_mock.get(
        f"{FIXTURE_BASE_URL}/information", exc=BridgeAuthenticationException
    )

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_USER_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "user"
    assert result2["errors"] == {"base": "invalid_auth"}

async def test_form_cannot_connect(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we handle cannot connect error."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": config_entries.SOURCE_USER}
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["errors"] is None

    aioclient_mock.get(f"{FIXTURE_BASE_URL}/information", exc=ClientConnectionError)

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_USER_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "user"
    assert result2["errors"] == {"base": "cannot_connect"}

async def test_form_unknown_error(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we handle unknown error."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": config_entries.SOURCE_USER}
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["errors"] is None

    with patch(
        "homeassistant.components.system_bridge.config_flow.Bridge.async_get_information",
        side_effect=Exception("Boom"),
    ):
        result2 = await hass.config_entries.flow.async_configure(
            result["flow_id"], FIXTURE_USER_INPUT
        )
        await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "user"
    assert result2["errors"] == {"base": "unknown"}

async def test_reauth_authorization_error(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we show user form on authorization error."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": "reauth"}, data=FIXTURE_USER_INPUT
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["step_id"] == "authenticate"

    aioclient_mock.get(
        f"{FIXTURE_BASE_URL}/information", exc=BridgeAuthenticationException
    )

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_AUTH_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "authenticate"
    assert result2["errors"] == {"base": "invalid_auth"}

async def test_reauth_connection_error(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we show user form on connection error."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": "reauth"}, data=FIXTURE_USER_INPUT
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["step_id"] == "authenticate"

    aioclient_mock.get(f"{FIXTURE_BASE_URL}/information", exc=ClientConnectionError)

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_AUTH_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "authenticate"
    assert result2["errors"] == {"base": "cannot_connect"}

async def test_reauth_flow(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test reauth flow."""
    mock_config = MockConfigEntry(
        domain=DOMAIN, unique_id=FIXTURE_UUID, data=FIXTURE_USER_INPUT
    )
    mock_config.add_to_hass(hass)

    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": "reauth"}, data=FIXTURE_USER_INPUT
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["step_id"] == "authenticate"

    aioclient_mock.get(
        f"{FIXTURE_BASE_URL}/information",
        headers={"Content-Type": "application/json"},
        json=FIXTURE_INFORMATION,
    )

    with patch(
        "homeassistant.components.system_bridge.async_setup_entry",
        return_value=True,
    ) as mock_setup_entry:
        result2 = await hass.config_entries.flow.async_configure(
            result["flow_id"], FIXTURE_AUTH_INPUT
        )
        await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_ABORT
    assert result2["reason"] == "reauth_successful"

    assert len(mock_setup_entry.mock_calls) == 1

async def test_zeroconf_flow(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test zeroconf flow."""

    result = await hass.config_entries.flow.async_init(
        DOMAIN,
        context={"source": config_entries.SOURCE_ZEROCONF},
        data=FIXTURE_ZEROCONF,
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert not result["errors"]

    aioclient_mock.get(
        f"{FIXTURE_ZEROCONF_BASE_URL}/information",
        headers={"Content-Type": "application/json"},
        json=FIXTURE_INFORMATION,
    )

    with patch(
        "homeassistant.components.system_bridge.async_setup_entry",
        return_value=True,
    ) as mock_setup_entry:
        result2 = await hass.config_entries.flow.async_configure(
            result["flow_id"], FIXTURE_AUTH_INPUT
        )
        await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY
    assert result2["title"] == "test-bridge"
    assert result2["data"] == FIXTURE_ZEROCONF_INPUT
    assert len(mock_setup_entry.mock_calls) == 1

async def test_zeroconf_cannot_connect(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test zeroconf cannot connect flow."""

    result = await hass.config_entries.flow.async_init(
        DOMAIN,
        context={"source": config_entries.SOURCE_ZEROCONF},
        data=FIXTURE_ZEROCONF,
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert not result["errors"]

    aioclient_mock.get(
        f"{FIXTURE_ZEROCONF_BASE_URL}/information", exc=ClientConnectionError
    )

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_AUTH_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "authenticate"
    assert result2["errors"] == {"base": "cannot_connect"}

async def test_zeroconf_bad_zeroconf_info(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test zeroconf cannot connect flow."""

    result = await hass.config_entries.flow.async_init(
        DOMAIN,
        context={"source": config_entries.SOURCE_ZEROCONF},
        data=FIXTURE_ZEROCONF_BAD,
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_ABORT
    assert result["reason"] == "unknown"
<commit_msg>Use dataclass properties in system_bridge tests (#60888)<commit_after>"""Test the System Bridge config flow."""
from unittest.mock import patch

from aiohttp.client_exceptions import ClientConnectionError
from systembridge.exceptions import BridgeAuthenticationException

from homeassistant import config_entries, data_entry_flow
from homeassistant.components import zeroconf
from homeassistant.components.system_bridge.const import DOMAIN
from homeassistant.const import CONF_API_KEY, CONF_HOST, CONF_PORT

from tests.common import MockConfigEntry

FIXTURE_MAC_ADDRESS = "aa:bb:cc:dd:ee:ff"
FIXTURE_UUID = "e91bf575-56f3-4c83-8f42-70ac17adcd33"

FIXTURE_AUTH_INPUT = {CONF_API_KEY: "abc-123-def-456-ghi"}

FIXTURE_USER_INPUT = {
    CONF_API_KEY: "abc-123-def-456-ghi",
    CONF_HOST: "test-bridge",
    CONF_PORT: "9170",
}

FIXTURE_ZEROCONF_INPUT = {
    CONF_API_KEY: "abc-123-def-456-ghi",
    CONF_HOST: "1.1.1.1",
    CONF_PORT: "9170",
}

FIXTURE_ZEROCONF = zeroconf.ZeroconfServiceInfo(
    host="1.1.1.1",
    port=9170,
    hostname="test-bridge.local.",
    type="_system-bridge._udp.local.",
    name="System Bridge - test-bridge._system-bridge._udp.local.",
    properties={
        "address": "http://test-bridge:9170",
        "fqdn": "test-bridge",
        "host": "test-bridge",
        "ip": "1.1.1.1",
        "mac": FIXTURE_MAC_ADDRESS,
        "port": "9170",
        "uuid": FIXTURE_UUID,
    },
)

FIXTURE_ZEROCONF_BAD = zeroconf.ZeroconfServiceInfo(
    host="1.1.1.1",
    port=9170,
    hostname="test-bridge.local.",
    type="_system-bridge._udp.local.",
    name="System Bridge - test-bridge._system-bridge._udp.local.",
    properties={
        "something": "bad",
    },
)

FIXTURE_INFORMATION = {
    "address": "http://test-bridge:9170",
    "apiPort": 9170,
    "fqdn": "test-bridge",
    "host": "test-bridge",
    "ip": "1.1.1.1",
    "mac": FIXTURE_MAC_ADDRESS,
    "updates": {
        "available": False,
        "newer": False,
        "url": "https://github.com/timmo001/system-bridge/releases/tag/v2.3.2",
        "version": {"current": "2.3.2", "new": "2.3.2"},
    },
    "uuid": FIXTURE_UUID,
    "version": "2.3.2",
    "websocketAddress": "ws://test-bridge:9172",
    "websocketPort": 9172,
}

FIXTURE_BASE_URL = (
    f"http://{FIXTURE_USER_INPUT[CONF_HOST]}:{FIXTURE_USER_INPUT[CONF_PORT]}"
)

FIXTURE_ZEROCONF_BASE_URL = f"http://{FIXTURE_ZEROCONF.host}:{FIXTURE_ZEROCONF.port}"

async def test_user_flow(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test full user flow."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": config_entries.SOURCE_USER}
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["errors"] is None

    aioclient_mock.get(
        f"{FIXTURE_BASE_URL}/information",
        headers={"Content-Type": "application/json"},
        json=FIXTURE_INFORMATION,
    )

    with patch(
        "homeassistant.components.system_bridge.async_setup_entry",
        return_value=True,
    ) as mock_setup_entry:
        result2 = await hass.config_entries.flow.async_configure(
            result["flow_id"], FIXTURE_USER_INPUT
        )
        await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY
    assert result2["title"] == "test-bridge"
    assert result2["data"] == FIXTURE_USER_INPUT
    assert len(mock_setup_entry.mock_calls) == 1

async def test_form_invalid_auth(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we handle invalid auth."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": config_entries.SOURCE_USER}
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["errors"] is None

    aioclient_mock.get(
        f"{FIXTURE_BASE_URL}/information", exc=BridgeAuthenticationException
    )

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_USER_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "user"
    assert result2["errors"] == {"base": "invalid_auth"}

async def test_form_cannot_connect(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we handle cannot connect error."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": config_entries.SOURCE_USER}
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["errors"] is None

    aioclient_mock.get(f"{FIXTURE_BASE_URL}/information", exc=ClientConnectionError)

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_USER_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "user"
    assert result2["errors"] == {"base": "cannot_connect"}

async def test_form_unknown_error(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we handle unknown error."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": config_entries.SOURCE_USER}
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["errors"] is None

    with patch(
        "homeassistant.components.system_bridge.config_flow.Bridge.async_get_information",
        side_effect=Exception("Boom"),
    ):
        result2 = await hass.config_entries.flow.async_configure(
            result["flow_id"], FIXTURE_USER_INPUT
        )
        await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "user"
    assert result2["errors"] == {"base": "unknown"}

async def test_reauth_authorization_error(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we show user form on authorization error."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": "reauth"}, data=FIXTURE_USER_INPUT
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["step_id"] == "authenticate"

    aioclient_mock.get(
        f"{FIXTURE_BASE_URL}/information", exc=BridgeAuthenticationException
    )

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_AUTH_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "authenticate"
    assert result2["errors"] == {"base": "invalid_auth"}

async def test_reauth_connection_error(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test we show user form on connection error."""
    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": "reauth"}, data=FIXTURE_USER_INPUT
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["step_id"] == "authenticate"

    aioclient_mock.get(f"{FIXTURE_BASE_URL}/information", exc=ClientConnectionError)

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_AUTH_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "authenticate"
    assert result2["errors"] == {"base": "cannot_connect"}

async def test_reauth_flow(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test reauth flow."""
    mock_config = MockConfigEntry(
        domain=DOMAIN, unique_id=FIXTURE_UUID, data=FIXTURE_USER_INPUT
    )
    mock_config.add_to_hass(hass)

    result = await hass.config_entries.flow.async_init(
        DOMAIN, context={"source": "reauth"}, data=FIXTURE_USER_INPUT
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result["step_id"] == "authenticate"

    aioclient_mock.get(
        f"{FIXTURE_BASE_URL}/information",
        headers={"Content-Type": "application/json"},
        json=FIXTURE_INFORMATION,
    )

    with patch(
        "homeassistant.components.system_bridge.async_setup_entry",
        return_value=True,
    ) as mock_setup_entry:
        result2 = await hass.config_entries.flow.async_configure(
            result["flow_id"], FIXTURE_AUTH_INPUT
        )
        await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_ABORT
    assert result2["reason"] == "reauth_successful"

    assert len(mock_setup_entry.mock_calls) == 1

async def test_zeroconf_flow(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test zeroconf flow."""

    result = await hass.config_entries.flow.async_init(
        DOMAIN,
        context={"source": config_entries.SOURCE_ZEROCONF},
        data=FIXTURE_ZEROCONF,
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert not result["errors"]

    aioclient_mock.get(
        f"{FIXTURE_ZEROCONF_BASE_URL}/information",
        headers={"Content-Type": "application/json"},
        json=FIXTURE_INFORMATION,
    )

    with patch(
        "homeassistant.components.system_bridge.async_setup_entry",
        return_value=True,
    ) as mock_setup_entry:
        result2 = await hass.config_entries.flow.async_configure(
            result["flow_id"], FIXTURE_AUTH_INPUT
        )
        await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY
    assert result2["title"] == "test-bridge"
    assert result2["data"] == FIXTURE_ZEROCONF_INPUT
    assert len(mock_setup_entry.mock_calls) == 1

async def test_zeroconf_cannot_connect(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test zeroconf cannot connect flow."""

    result = await hass.config_entries.flow.async_init(
        DOMAIN,
        context={"source": config_entries.SOURCE_ZEROCONF},
        data=FIXTURE_ZEROCONF,
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert not result["errors"]

    aioclient_mock.get(
        f"{FIXTURE_ZEROCONF_BASE_URL}/information", exc=ClientConnectionError
    )

    result2 = await hass.config_entries.flow.async_configure(
        result["flow_id"], FIXTURE_AUTH_INPUT
    )
    await hass.async_block_till_done()

    assert result2["type"] == data_entry_flow.RESULT_TYPE_FORM
    assert result2["step_id"] == "authenticate"
    assert result2["errors"] == {"base": "cannot_connect"}

async def test_zeroconf_bad_zeroconf_info(
    hass, aiohttp_client, aioclient_mock, current_request_with_host
) -> None:
    """Test zeroconf cannot connect flow."""

    result = await hass.config_entries.flow.async_init(
        DOMAIN,
        context={"source": config_entries.SOURCE_ZEROCONF},
        data=FIXTURE_ZEROCONF_BAD,
    )

    assert result["type"] == data_entry_flow.RESULT_TYPE_ABORT
    assert result["reason"] == "unknown"
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import requests
from ipaddr import IPv4Address
from maas_common import (status_ok, status_err, metric, get_keystone_client,
                         get_auth_ref)
from requests import exceptions as exc

# NOTE(mancdaz): until https://review.openstack.org/#/c/111051/
# lands, there is no way to pass a custom (local) endpoint to
# cinderclient. Only way to test local is direct http. :sadface:

def check(auth_ref, args):

    keystone = get_keystone_client(auth_ref)
    auth_token = keystone.auth_token
    VOLUME_ENDPOINT = 'http://{ip}:8776/v1/{tenant}' \
                      .format(ip=args.ip, tenant=keystone.tenant_id)

    s = requests.Session()

    s.headers.update(
        {'Content-type': 'application/json',
         'x-auth-token': auth_token})

    try:
        r = s.get('%s/volumes' % VOLUME_ENDPOINT,
                  verify=False,
                  timeout=10)
        is_up = r.ok
    except (exc.ConnectionError,
            exc.HTTPError,
            exc.Timeout) as e:
        status_err(str(e))
    finally:
        status_ok()
        metric('cinder_api_local_status', 'uint32', is_up)
        # only want to send other metrics if api is up
        if is_up:
            milliseconds = r.elapsed.total_seconds() * 1000
            metric('cinder_api_response_time', 'uint32', milliseconds)

def main(args):
    auth_ref = get_auth_ref()
    check(auth_ref, args)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Check neutron agents')
    parser.add_argument('ip',
                        type=IPv4Address,
                        help='neutron service IP address.')
    args = parser.parse_args()
    main(args)
<commit_msg>Use metric_bool<commit_after>#!/usr/bin/env python

import argparse
import requests
from ipaddr import IPv4Address
from maas_common import (status_ok, status_err, metric, metric_bool,
                         get_keystone_client, get_auth_ref)
from requests import exceptions as exc

# NOTE(mancdaz): until https://review.openstack.org/#/c/111051/
# lands, there is no way to pass a custom (local) endpoint to
# cinderclient. Only way to test local is direct http. :sadface:

def check(auth_ref, args):

    keystone = get_keystone_client(auth_ref)
    auth_token = keystone.auth_token
    VOLUME_ENDPOINT = 'http://{ip}:8776/v1/{tenant}' \
                      .format(ip=args.ip, tenant=keystone.tenant_id)

    s = requests.Session()

    s.headers.update(
        {'Content-type': 'application/json',
         'x-auth-token': auth_token})

    try:
        r = s.get('%s/volumes' % VOLUME_ENDPOINT,
                  verify=False,
                  timeout=10)
        is_up = r.ok
    except (exc.ConnectionError,
            exc.HTTPError,
            exc.Timeout) as e:
        status_err(str(e))
    finally:
        status_ok()
        metric_bool('cinder_api_local_status', is_up)
        # only want to send other metrics if api is up
        if is_up:
            milliseconds = r.elapsed.total_seconds() * 1000
            metric('cinder_api_response_time', 'uint32', milliseconds)

def main(args):
    auth_ref = get_auth_ref()
    check(auth_ref, args)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Check neutron agents')
    parser.add_argument('ip',
                        type=IPv4Address,
                        help='neutron service IP address.')
    args = parser.parse_args()
    main(args)
<|endoftext|>


<commit_before>from __future__ import division
from collections import namedtuple
from datetime import datetime
import importlib

import sys

import simplejson
from django.conf import settings

from dimagi.utils.chunked import chunked
from dimagi.utils.parsing import string_to_utc_datetime

from pillowtop.exceptions import PillowNotFoundError

def get_pillow_instance(full_class_str):
    pillow_class = _import_class_or_function(full_class_str)
    return pillow_class()

def _import_class_or_function(full_class_str):
    mod_path, pillow_class_name = full_class_str.rsplit('.', 1)
    try:
        mod = importlib.import_module(mod_path)
        return getattr(mod, pillow_class_name)
    except (AttributeError, ImportError):
        if settings.DEBUG:
            raise
        raise ValueError("Could not find class or function: '%s'" % full_class_str)

def get_all_pillow_classes():
    return [config.get_class() for config in get_all_pillow_configs()]

def get_all_pillow_instances():
    return [config.get_instance() for config in get_all_pillow_configs()]

def get_all_pillow_configs():
    return get_pillow_configs_from_settings_dict(getattr(settings, 'PILLOWTOPS', {}))

def get_pillow_configs_from_settings_dict(pillow_settings_dict):
    """
    The pillow_settings_dict is expected to be a dict mapping groups to list of pillow configs
    """
    for section, list_of_pillows in pillow_settings_dict.items():
        for pillow_config in list_of_pillows:
            yield get_pillow_config_from_setting(section, pillow_config)

class PillowConfig(namedtuple('PillowConfig', ['section', 'name', 'class_name', 'instance_generator'])):
    """
    Helper object for getting pillow classes/instances from settings
    """
    def get_class(self):
        return _import_class_or_function(self.class_name)

    def get_instance(self):
        if self.instance_generator:
            instance_generator_fn = _import_class_or_function(self.instance_generator)
            return instance_generator_fn(self.name)
        else:
            return get_pillow_instance(self.class_name)

def get_pillow_config_from_setting(section, pillow_config_string_or_dict):
    if isinstance(pillow_config_string_or_dict, basestring):
        return PillowConfig(
            section,
            pillow_config_string_or_dict.rsplit('.', 1)[1],
            pillow_config_string_or_dict,
            None,
        )
    else:
        assert 'class' in pillow_config_string_or_dict
        class_name = pillow_config_string_or_dict['class']
        return PillowConfig(
            section,
            pillow_config_string_or_dict.get('name', class_name),
            class_name,
            pillow_config_string_or_dict.get('instance', None),
        )

def get_pillow_by_name(pillow_class_name, instantiate=True):
    config = get_pillow_config_by_name(pillow_class_name)
    return config.get_instance() if instantiate else config.get_class()

def get_pillow_config_by_name(pillow_name):
    all_configs = get_all_pillow_configs()
    for config in all_configs:
        if config.name == pillow_name:
            return config
    raise PillowNotFoundError(u'No pillow found with name {}'.format(pillow_name))

def force_seq_int(seq):
    if seq is None or seq == '':
        return None
    elif isinstance(seq, dict):
        # multi-topic checkpoints don't support a single sequence id
        return None
    elif isinstance(seq, basestring):
        return int(seq.split('-')[0])
    else:
        assert isinstance(seq, int)
        return seq

def get_all_pillows_json():
    pillow_configs = get_all_pillow_configs()
    return [get_pillow_json(pillow_config) for pillow_config in pillow_configs]

def get_pillow_json(pillow_config):
    assert isinstance(pillow_config, PillowConfig)
    from pillowtop.listener import AliasedElasticPillow

    pillow_class = pillow_config.get_class()
    pillow = (pillow_class(online=False) if issubclass(pillow_class, AliasedElasticPillow)
              else pillow_config.get_instance())

    checkpoint = pillow.get_checkpoint()
    timestamp = checkpoint.timestamp
    if timestamp:
        time_since_last = datetime.utcnow() - timestamp
        hours_since_last = time_since_last.total_seconds() // 3600

        try:
            # remove microsecond portion
            time_since_last = str(time_since_last)
            time_since_last = time_since_last[0:time_since_last.index('.')]
        except ValueError:
            pass
    else:
        time_since_last = ''
        hours_since_last = None
    try:
        db_seq = pillow.get_change_feed().get_latest_change_id()
    except ValueError:
        db_seq = None
    return {
        'name': pillow_config.name,
        'seq': force_seq_int(checkpoint.wrapped_sequence),
        'old_seq': force_seq_int(checkpoint.old_sequence) or 0,
        'db_seq': force_seq_int(db_seq),
        'time_since_last': time_since_last,
        'hours_since_last': hours_since_last
    }

def prepare_bulk_payloads(bulk_changes, max_size, chunk_size=100):
    payloads = ['']
    for bulk_chunk in chunked(bulk_changes, chunk_size):
        current_payload = payloads[-1]
        payload_chunk = '\n'.join(map(simplejson.dumps, bulk_chunk)) + '\n'
        appended_payload = current_payload + payload_chunk
        new_payload_size = sys.getsizeof(appended_payload)
        if new_payload_size > max_size:
            payloads.append(payload_chunk)
        else:
            payloads[-1] = appended_payload

    return filter(None, payloads)
<commit_msg>use to_function<commit_after>from __future__ import division
from collections import namedtuple
from datetime import datetime
import importlib

import sys

import simplejson
from django.conf import settings

from dimagi.utils.chunked import chunked
from dimagi.utils.modules import to_function
from dimagi.utils.parsing import string_to_utc_datetime

from pillowtop.exceptions import PillowNotFoundError

def get_pillow_instance(full_class_str):
    pillow_class = _import_class_or_function(full_class_str)
    return pillow_class()

def _import_class_or_function(full_class_str):
    return to_function(full_class_str, failhard=settings.DEBUG)

def get_all_pillow_classes():
    return [config.get_class() for config in get_all_pillow_configs()]

def get_all_pillow_instances():
    return [config.get_instance() for config in get_all_pillow_configs()]

def get_all_pillow_configs():
    return get_pillow_configs_from_settings_dict(getattr(settings, 'PILLOWTOPS', {}))

def get_pillow_configs_from_settings_dict(pillow_settings_dict):
    """
    The pillow_settings_dict is expected to be a dict mapping groups to list of pillow configs
    """
    for section, list_of_pillows in pillow_settings_dict.items():
        for pillow_config in list_of_pillows:
            yield get_pillow_config_from_setting(section, pillow_config)

class PillowConfig(namedtuple('PillowConfig', ['section', 'name', 'class_name', 'instance_generator'])):
    """
    Helper object for getting pillow classes/instances from settings
    """
    def get_class(self):
        return _import_class_or_function(self.class_name)

    def get_instance(self):
        if self.instance_generator:
            instance_generator_fn = _import_class_or_function(self.instance_generator)
            return instance_generator_fn(self.name)
        else:
            return get_pillow_instance(self.class_name)

def get_pillow_config_from_setting(section, pillow_config_string_or_dict):
    if isinstance(pillow_config_string_or_dict, basestring):
        return PillowConfig(
            section,
            pillow_config_string_or_dict.rsplit('.', 1)[1],
            pillow_config_string_or_dict,
            None,
        )
    else:
        assert 'class' in pillow_config_string_or_dict
        class_name = pillow_config_string_or_dict['class']
        return PillowConfig(
            section,
            pillow_config_string_or_dict.get('name', class_name),
            class_name,
            pillow_config_string_or_dict.get('instance', None),
        )

def get_pillow_by_name(pillow_class_name, instantiate=True):
    config = get_pillow_config_by_name(pillow_class_name)
    return config.get_instance() if instantiate else config.get_class()

def get_pillow_config_by_name(pillow_name):
    all_configs = get_all_pillow_configs()
    for config in all_configs:
        if config.name == pillow_name:
            return config
    raise PillowNotFoundError(u'No pillow found with name {}'.format(pillow_name))

def force_seq_int(seq):
    if seq is None or seq == '':
        return None
    elif isinstance(seq, dict):
        # multi-topic checkpoints don't support a single sequence id
        return None
    elif isinstance(seq, basestring):
        return int(seq.split('-')[0])
    else:
        assert isinstance(seq, int)
        return seq

def get_all_pillows_json():
    pillow_configs = get_all_pillow_configs()
    return [get_pillow_json(pillow_config) for pillow_config in pillow_configs]

def get_pillow_json(pillow_config):
    assert isinstance(pillow_config, PillowConfig)
    from pillowtop.listener import AliasedElasticPillow

    pillow_class = pillow_config.get_class()
    pillow = (pillow_class(online=False) if issubclass(pillow_class, AliasedElasticPillow)
              else pillow_config.get_instance())

    checkpoint = pillow.get_checkpoint()
    timestamp = checkpoint.timestamp
    if timestamp:
        time_since_last = datetime.utcnow() - timestamp
        hours_since_last = time_since_last.total_seconds() // 3600

        try:
            # remove microsecond portion
            time_since_last = str(time_since_last)
            time_since_last = time_since_last[0:time_since_last.index('.')]
        except ValueError:
            pass
    else:
        time_since_last = ''
        hours_since_last = None
    try:
        db_seq = pillow.get_change_feed().get_latest_change_id()
    except ValueError:
        db_seq = None
    return {
        'name': pillow_config.name,
        'seq': force_seq_int(checkpoint.wrapped_sequence),
        'old_seq': force_seq_int(checkpoint.old_sequence) or 0,
        'db_seq': force_seq_int(db_seq),
        'time_since_last': time_since_last,
        'hours_since_last': hours_since_last
    }

def prepare_bulk_payloads(bulk_changes, max_size, chunk_size=100):
    payloads = ['']
    for bulk_chunk in chunked(bulk_changes, chunk_size):
        current_payload = payloads[-1]
        payload_chunk = '\n'.join(map(simplejson.dumps, bulk_chunk)) + '\n'
        appended_payload = current_payload + payload_chunk
        new_payload_size = sys.getsizeof(appended_payload)
        if new_payload_size > max_size:
            payloads.append(payload_chunk)
        else:
            payloads[-1] = appended_payload

    return filter(None, payloads)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import numpy as np
import os

from gscripts.qtools import Submitter

"""Original script
from gscripts.qtools import Submitter
import numpy as np
import math

downsample_command = 'java -jar /projects/ps-yeolab/software/picard-tools-1.93/DownsampleSam.jar'

iter_per_downsample = 10

#initial_command_list = [#"DIR=/home/obotvinnik/scratch/mn_diff_singlecell/fastq/downsample",
#                "BAM=/home/obotvinnik/scratch/mn_diff_singlecell/fastq/P9_1.bam"]
input_bam = "/home/obotvinnik/scratch/mn_diff_singlecell/fastq/P9_1.Aligned.out.sam.bam.sorted.bam"

#downsample_dir = '~/projects/singlecell/data/round1/downsampling'
command_list = []
for downsample_prob in np.arange(0.01, 1, 0.01):
    for i in range(iter_per_downsample):
        out_bam = 'P9_1_prob{}_iter{}.bam'.format(downsample_prob, i)
        random_seed = 2014 + i
        command_list.append(
            '{} INPUT={} OUTPUT={} RANDOM_SEED={} PROBABILITY={} CREATE_INDEX=true'.format(downsample_command,
                                                                           input_bam,
                                                                           out_bam,
                                                                           random_seed,
                                                                           downsample_prob))
job_name = 'downsample'
sub = Submitter(sh_file='downsample.sh', queue_type='PBS',
                command_list=command_list, job_name=job_name, walltime='4:00:00')
sub.write_sh(submit=True, nodes=1, ppn=1, queue='home', array=True, max_running=20)

"""

class CommandLine(object):
    def __init__(self, inOpts=None):
        self.parser = parser = argparse.ArgumentParser(
            description='Downsample reads from a bam file')
        bam = parser.add_mutually_exclusive_group(required=True)
        bam.add_argument('--bam',
                         type=str, action='store',
                         help='Bam file to downsample')
        bam.add_argument('--bam-list',
                         type=str, action='store',
                         help='A file containing names of bam files to '
                              'downsample on separate lines, '
                              'and tab-separated with sample ids (no '
                              'header), e.g.:\n'
                              'sample1.bam\tcontrol\n'
                              'sample2.bam\ttreatment\n')
        parser.add_argument('--sample-id', type=str, action='store',
                            help='Sample ID to prefix the output bam files'
                                 '. Required if providing a single bam '
                                 'file.')
        parser.add_argument('-j', '--jar',
                            default="/projects/ps-yeolab/software/picard-tools-1.93/DownsampleSam.jar",
                            type=str, action='store',
                            help='The Java Archive from PicardTools to use')
        parser.add_argument('-t', '--iter-per-percentage',
                            type=str, action='store', default=3,
                            help='For each fraction downsampling, how many '
                                 'iterations to perform')
        parser.add_argument('-s', '--step-size', type=float, action='store',
                            default=0.05,
                            help='Step size of downsampling, default is 0.05 '
                                 'for downsampling every 0.05 fraction of the '
                                 'reads in '
                                 'the bam file, resulting in 0.05, 0.10, '
                                 '0.15, '
                                 'etc downsampled files.')
        parser.add_argument('-r', '--random-seed-base', type=int,
                            action='store', default=2014,
                            help='To make sure this exact downsampling is '
                                 'reproducible but still random for different '
                                 'iterations, use this value + iteration '
                                 'number as the random seed.')
        parser.add_argument('-o', '--out-dir', type=str,
                            action='store', default='./',
                            help='Where you want to save the downsampled bams'
                                 '. Default is the current directory.')
        parser.add_argument('-n', '--name', default='downsample',
                            action='store', type=str,
                            help='The name of the submitted job in the queue')
        parser.add_argument('--out-sh', action='store', type=str,
                            required=False,
                            help='The sh file written and submitted to the '
                                 'cluster. Default is the name of the job + "'
                                 '.sh"')
        parser.add_argument('--do-not-submit', required=False,
                            action='store_true', default=False,
                            help='Flag to not actually submit the job but '
                                 'just write the sh file (for testing)')
        parser.add_argument('--queue-type', required=False, type=str,
                            action='store', default='PBS',
                            help='Type of the queue to submit to. For testing '
                                 'purposes on non-server devices, e.g. laptops')
        if inOpts is None:
            self.args = vars(self.parser.parse_args())
        else:
            self.args = vars(self.parser.parse_args(inOpts))

    def do_usage_and_die(self, str):
        '''
        If a critical error is encountered, where it is suspected that the
        program is not being called with consistent parameters or data, this
        method will write out an error string (str), then terminate execution
        of the program.
        '''
        import sys

        print >> sys.stderr, str
        self.parser.print_usage()
        return 2

# Class: Usage
class Usage(Exception):
    '''
    Used to signal a Usage error, evoking a usage statement and eventual
    exit when raised
    '''

    def __init__(self, msg):
        self.msg = msg

# Class: Downsample
class Downsample(object):
    def __init__(self, bams, sample_ids, jar, iter_per_percentage, step_size,
                 random_seed_base, out_dir, name, out_sh=None, submit=True,
                 queue_type='PBS'):
        """Any CamelCase here is directly copied from the STAR inputs for
        complete compatibility
        """
        # Make the directory if it's not there already
        try:
            os.mkdir(out_dir)
        except OSError:
            pass

        downsample_command = 'java -jar {}'.format(jar)

        commands = []
        for bam, sample_id in zip(bams, sample_ids):
            for downsample_prob in np.arange(step_size, 1, step_size):
                for i in range(iter_per_percentage):
                    out_bam = '{}_prob{}_iter{}.bam'.format(sample_id,
                                                            downsample_prob, i)
                    random_seed = random_seed_base + i
                    commands.append(
                        '{} INPUT={} OUTPUT={} RANDOM_SEED={} PROBABILITY={} CREATE_INDEX=true'.format(
                            downsample_command,
                            bam,
                            out_bam,
                            random_seed,
                            downsample_prob))

        sub = Submitter(sh_filename=out_sh, queue_type=queue_type,
                        commands=commands, job_name=name,
                        walltime='1:00:00', nodes=1, ppn=1, queue='home',
                        array=True,
                        max_running=20)
        sub.write_sh(submit=submit)

if __name__ == '__main__':
    try:
        cl = CommandLine()

        job_name = cl.args['name']
        out_dir = cl.args['out_dir']
        out_sh = name = job_name + '.sh' if cl.args['out_sh'] is None \
            else cl.args['out_sh']
        submit = not cl.args['do_not_submit']
        queue_type = cl.args['queue_type']

        bam = cl.args['bam']
        sample_id = cl.args['sample_id']
        if bam is not None and sample_id is None:
            raise ValueError('If "--bam" is provided, "--sample-id" is also '
                             'required.')

        if cl.args['bams'] is not None:
            bams = []
            sample_ids = []
            with open(cl.args['bams']) as f:
                for line in f:
                    line = line.strip().split()
                    bams.append(line[0])
                    sample_ids.append(line[1])
        else:
            bams = [bam]
            sample_ids = [sample_id]

        jar = cl.args['jar']
        iter_per_percentage = cl.args['iter_per_percentage']
        step_size = cl.args['step_size']
        random_seed_base = cl.args['random_seed_base']

        Downsample(bams, sample_ids, jar, iter_per_percentage, step_size,
                   random_seed_base, out_dir, name, out_sh, submit, queue_type)

    except Usage, err:
        cl.do_usage_and_die()<commit_msg>escape newline and tab characters so they appear (since argparser takes away any formatting that you did)<commit_after>#!/usr/bin/env python

import argparse
import numpy as np
import os

from gscripts.qtools import Submitter

"""Original script
from gscripts.qtools import Submitter
import numpy as np
import math

downsample_command = 'java -jar /projects/ps-yeolab/software/picard-tools-1.93/DownsampleSam.jar'

iter_per_downsample = 10

#initial_command_list = [#"DIR=/home/obotvinnik/scratch/mn_diff_singlecell/fastq/downsample",
#                "BAM=/home/obotvinnik/scratch/mn_diff_singlecell/fastq/P9_1.bam"]
input_bam = "/home/obotvinnik/scratch/mn_diff_singlecell/fastq/P9_1.Aligned.out.sam.bam.sorted.bam"

#downsample_dir = '~/projects/singlecell/data/round1/downsampling'
command_list = []
for downsample_prob in np.arange(0.01, 1, 0.01):
    for i in range(iter_per_downsample):
        out_bam = 'P9_1_prob{}_iter{}.bam'.format(downsample_prob, i)
        random_seed = 2014 + i
        command_list.append(
            '{} INPUT={} OUTPUT={} RANDOM_SEED={} PROBABILITY={} CREATE_INDEX=true'.format(downsample_command,
                                                                           input_bam,
                                                                           out_bam,
                                                                           random_seed,
                                                                           downsample_prob))
job_name = 'downsample'
sub = Submitter(sh_file='downsample.sh', queue_type='PBS',
                command_list=command_list, job_name=job_name, walltime='4:00:00')
sub.write_sh(submit=True, nodes=1, ppn=1, queue='home', array=True, max_running=20)

"""

class CommandLine(object):
    def __init__(self, inOpts=None):
        self.parser = parser = argparse.ArgumentParser(
            description='Downsample reads from a bam file')
        bam = parser.add_mutually_exclusive_group(required=True)
        bam.add_argument('--bam',
                         type=str, action='store',
                         help='Bam file to downsample')
        bam.add_argument('--bam-list',
                         type=str, action='store',
                         help='A file containing names of bam files to '
                              'downsample on separate lines, '
                              'and tab-separated with sample ids (no '
                              'header), e.g.:'
                              'sample1.bam\\tcontrol\\n'
                              'sample2.bam\\ttreatment\\n')
        parser.add_argument('--sample-id', type=str, action='store',
                            help='Sample ID to prefix the output bam files'
                                 '. Required if providing a single bam '
                                 'file.')
        parser.add_argument('-j', '--jar',
                            default="/projects/ps-yeolab/software/picard-tools-1.93/DownsampleSam.jar",
                            type=str, action='store',
                            help='The Java Archive from PicardTools to use')
        parser.add_argument('-t', '--iter-per-percentage',
                            type=str, action='store', default=3,
                            help='For each fraction downsampling, how many '
                                 'iterations to perform')
        parser.add_argument('-s', '--step-size', type=float, action='store',
                            default=0.05,
                            help='Step size of downsampling, default is 0.05 '
                                 'for downsampling every 0.05 fraction of the '
                                 'reads in '
                                 'the bam file, resulting in 0.05, 0.10, '
                                 '0.15, '
                                 'etc downsampled files.')
        parser.add_argument('-r', '--random-seed-base', type=int,
                            action='store', default=2014,
                            help='To make sure this exact downsampling is '
                                 'reproducible but still random for different '
                                 'iterations, use this value + iteration '
                                 'number as the random seed.')
        parser.add_argument('-o', '--out-dir', type=str,
                            action='store', default='./',
                            help='Where you want to save the downsampled bams'
                                 '. Default is the current directory.')
        parser.add_argument('-n', '--name', default='downsample',
                            action='store', type=str,
                            help='The name of the submitted job in the queue')
        parser.add_argument('--out-sh', action='store', type=str,
                            required=False,
                            help='The sh file written and submitted to the '
                                 'cluster. Default is the name of the job + "'
                                 '.sh"')
        parser.add_argument('--do-not-submit', required=False,
                            action='store_true', default=False,
                            help='Flag to not actually submit the job but '
                                 'just write the sh file (for testing)')
        parser.add_argument('--queue-type', required=False, type=str,
                            action='store', default='PBS',
                            help='Type of the queue to submit to. For testing '
                                 'purposes on non-server devices, e.g. laptops')
        if inOpts is None:
            self.args = vars(self.parser.parse_args())
        else:
            self.args = vars(self.parser.parse_args(inOpts))

    def do_usage_and_die(self, str):
        '''
        If a critical error is encountered, where it is suspected that the
        program is not being called with consistent parameters or data, this
        method will write out an error string (str), then terminate execution
        of the program.
        '''
        import sys

        print >> sys.stderr, str
        self.parser.print_usage()
        return 2

# Class: Usage
class Usage(Exception):
    '''
    Used to signal a Usage error, evoking a usage statement and eventual
    exit when raised
    '''

    def __init__(self, msg):
        self.msg = msg

# Class: Downsample
class Downsample(object):
    def __init__(self, bams, sample_ids, jar, iter_per_percentage, step_size,
                 random_seed_base, out_dir, name, out_sh=None, submit=True,
                 queue_type='PBS'):
        """Any CamelCase here is directly copied from the STAR inputs for
        complete compatibility
        """
        # Make the directory if it's not there already
        try:
            os.mkdir(out_dir)
        except OSError:
            pass

        downsample_command = 'java -jar {}'.format(jar)

        commands = []
        for bam, sample_id in zip(bams, sample_ids):
            for downsample_prob in np.arange(step_size, 1, step_size):
                for i in range(iter_per_percentage):
                    out_bam = '{}_prob{}_iter{}.bam'.format(sample_id,
                                                            downsample_prob, i)
                    random_seed = random_seed_base + i
                    commands.append(
                        '{} INPUT={} OUTPUT={} RANDOM_SEED={} PROBABILITY={} CREATE_INDEX=true'.format(
                            downsample_command,
                            bam,
                            out_bam,
                            random_seed,
                            downsample_prob))

        sub = Submitter(sh_filename=out_sh, queue_type=queue_type,
                        commands=commands, job_name=name,
                        walltime='1:00:00', nodes=1, ppn=1, queue='home',
                        array=True,
                        max_running=20)
        sub.write_sh(submit=submit)

if __name__ == '__main__':
    try:
        cl = CommandLine()

        job_name = cl.args['name']
        out_dir = cl.args['out_dir']
        out_sh = name = job_name + '.sh' if cl.args['out_sh'] is None \
            else cl.args['out_sh']
        submit = not cl.args['do_not_submit']
        queue_type = cl.args['queue_type']

        bam = cl.args['bam']
        sample_id = cl.args['sample_id']
        if bam is not None and sample_id is None:
            raise ValueError('If "--bam" is provided, "--sample-id" is also '
                             'required.')

        if cl.args['bams'] is not None:
            bams = []
            sample_ids = []
            with open(cl.args['bams']) as f:
                for line in f:
                    line = line.strip().split()
                    bams.append(line[0])
                    sample_ids.append(line[1])
        else:
            bams = [bam]
            sample_ids = [sample_id]

        jar = cl.args['jar']
        iter_per_percentage = cl.args['iter_per_percentage']
        step_size = cl.args['step_size']
        random_seed_base = cl.args['random_seed_base']

        Downsample(bams, sample_ids, jar, iter_per_percentage, step_size,
                   random_seed_base, out_dir, name, out_sh, submit, queue_type)

    except Usage, err:
        cl.do_usage_and_die()<|endoftext|>


<commit_before>from django.contrib import admin

from xtest_app.models.array_field import ArrayFieldEntry as ArrayEntry
from xtest_app.models.basic_field import BasicRelatedEntry, BasicAuthor, BasicBlog
from xtest_app.models.embedded_field import EmbeddedFieldEntry as EmbeddedEntry
# from xtest_app.models.misc_field import ListEntry, DictEntry
# from xtest_app.models.reference_field import ReferenceEntry, ReferenceAuthor

admin.site.register(BasicAuthor)
admin.site.register(BasicBlog)
admin.site.register(ArrayEntry)

admin.site.register(BasicRelatedEntry)
admin.site.register(EmbeddedEntry)
# admin.site.register(EmbeddedDateEntry)
# admin.site.register(ReferenceEntry)
# admin.site.register(ReferenceAuthor)
# admin.site.register(ListEntry)
# admin.site.register(DictEntry)

<commit_msg>EmbeddedField, ArrayField, EmbeddedNestedField, EmbeddedInternalField: all tests pass<commit_after>from django.contrib import admin

from xtest_app.models.array_field import ArrayFieldEntry as ArrayEntry
from xtest_app.models.basic_field import BasicRelatedEntry, BasicAuthor, BasicBlog
from xtest_app.models.embedded_field import EmbeddedFieldEntry as EmbeddedEntry
# from xtest_app.models.misc_field import ListEntry, DictEntry
# from xtest_app.models.reference_field import ReferenceEntry, ReferenceAuthor

admin.site.register(BasicAuthor)
admin.site.register(BasicBlog)
admin.site.register(ArrayEntry)

admin.site.register(BasicRelatedEntry)
# admin.site.register(EmbeddedEntry)
# admin.site.register(EmbeddedDateEntry)
# admin.site.register(ReferenceEntry)
# admin.site.register(ReferenceAuthor)
# admin.site.register(ListEntry)
# admin.site.register(DictEntry)

<|endoftext|>


<commit_before># Module:   constants
# Date:     4th February 2009
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""Global Constants

This module implements required shared global constants.
"""

from circuits import __version__

SERVER_PROTOCOL = (1, 1)
SERVER_VERSION = "circuits.web/%s" % __version__
SERVER_URL = "http://circuitsframework.com/"

DEFAULT_ERROR_MESSAGE = """\
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

  <meta http-equiv="Content-Type" content="text/html; charset=utf-8"></meta>
  <title>%(code)d %(name)s</title>

%(name)s

  %(description)s

%(traceback)s

   Powered by %(version)s

"""

HTTP_STATUS_CODES = {
    100: "Continue",
    101: "Switching Protocols",
    102: "Processing",
    200: "OK",
    201: "Created",
    202: "Accepted",
    203: "Non Authoritative Information",
    204: "No Content",
    205: "Reset Content",
    206: "Partial Content",
    207: "Multi Status",
    226: "IM Used",
    300: "Multiple Choices",
    301: "Moved Permanently",
    302: "Found",
    303: "See Other",
    304: "Not Modified",
    305: "Use Proxy",
    307: "Temporary Redirect",
    400: "Bad Request",
    401: "Unauthorized",
    402: "Payment Required",
    403: "Forbidden",
    404: "Not Found",
    405: "Method Not Allowed",
    406: "Not Acceptable",
    407: "Proxy Authentication Required",
    408: "Request Timeout",
    409: "Conflict",
    410: "Gone",
    411: "Length Required",
    412: "Precondition Failed",
    413: "Request Entity Too Large",
    414: "Request URI Too Long",
    415: "Unsupported Media Type",
    416: "Requested Range Not Satisfiable",
    417: "Expectation Failed",
    418: "I\"m a teapot",
    422: "Unprocessable Entity",
    423: "Locked",
    424: "Failed Dependency",
    426: "Upgrade Required",
    449: "Retry With",
    500: "Internal Server Error",
    501: "Not Implemented",
    502: "Bad Gateway",
    503: "Service Unavailable",
    504: "Gateway Timeout",
    505: "HTTP Version Not Supported",
    507: "Insufficient Storage",
    510: "Not Extended"
}
<commit_msg>Updated URL of X-Powered-By to http://circuitsweb.com/<commit_after># Module:   constants
# Date:     4th February 2009
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""Global Constants

This module implements required shared global constants.
"""

from circuits import __version__

SERVER_PROTOCOL = (1, 1)
SERVER_VERSION = "circuits.web/%s" % __version__
SERVER_URL = "http://circuitsweb.com/"

DEFAULT_ERROR_MESSAGE = """\
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

  <meta http-equiv="Content-Type" content="text/html; charset=utf-8"></meta>
  <title>%(code)d %(name)s</title>

%(name)s

  %(description)s

%(traceback)s

   Powered by %(version)s

"""

HTTP_STATUS_CODES = {
    100: "Continue",
    101: "Switching Protocols",
    102: "Processing",
    200: "OK",
    201: "Created",
    202: "Accepted",
    203: "Non Authoritative Information",
    204: "No Content",
    205: "Reset Content",
    206: "Partial Content",
    207: "Multi Status",
    226: "IM Used",
    300: "Multiple Choices",
    301: "Moved Permanently",
    302: "Found",
    303: "See Other",
    304: "Not Modified",
    305: "Use Proxy",
    307: "Temporary Redirect",
    400: "Bad Request",
    401: "Unauthorized",
    402: "Payment Required",
    403: "Forbidden",
    404: "Not Found",
    405: "Method Not Allowed",
    406: "Not Acceptable",
    407: "Proxy Authentication Required",
    408: "Request Timeout",
    409: "Conflict",
    410: "Gone",
    411: "Length Required",
    412: "Precondition Failed",
    413: "Request Entity Too Large",
    414: "Request URI Too Long",
    415: "Unsupported Media Type",
    416: "Requested Range Not Satisfiable",
    417: "Expectation Failed",
    418: "I\"m a teapot",
    422: "Unprocessable Entity",
    423: "Locked",
    424: "Failed Dependency",
    426: "Upgrade Required",
    449: "Retry With",
    500: "Internal Server Error",
    501: "Not Implemented",
    502: "Bad Gateway",
    503: "Service Unavailable",
    504: "Gateway Timeout",
    505: "HTTP Version Not Supported",
    507: "Insufficient Storage",
    510: "Not Extended"
}
<|endoftext|>


<commit_before>from corehq.pillows.core import DATE_FORMATS_ARR, DATE_FORMATS_STRING
from corehq.pillows.mappings import NULL_VALUE
from corehq.util.elastic import prefix_for_tests
from pillowtop.es_utils import ElasticsearchIndexInfo, CASE_HQ_INDEX_NAME

CASE_INDEX = prefix_for_tests("hqcases_2016-03-04")
CASE_ES_TYPE = 'case'

CASE_MAPPING = {
    'date_detection': False,
    'date_formats': DATE_FORMATS_ARR,
    'dynamic': False,
    '_meta': {
        'comment': '',
        'created': '2013-09-19 @dmyung'
    },
    'properties': {
        'actions': {'dynamic': False,
                    'properties': {
                        'action_type': {'type': 'string'},
                        'date': {'format': DATE_FORMATS_STRING,
                                 'type': 'date'},
                        'doc_type': {'index': 'not_analyzed',
                                     'type': 'string'},
                        'indices': {'dynamic': False,
                                    'properties': {
                                        'doc_type': {'index': 'not_analyzed',
                                                     'type': 'string'},
                                        'identifier': {'type': 'string'},
                                        'referenced_id': {'type': 'string'},
                                        'referenced_type': {'type': 'string'}
                                    },
                                    'type': 'object'},
                        'server_date': {'format': DATE_FORMATS_STRING,
                                        'type': 'date'},
                        'sync_log_id': {'type': 'string'},
                        'user_id': {'type': 'string'},
                        'xform_id': {'type': 'string'},
                        'xform_name': {'type': 'string'},
                        'xform_xmlns': {'type': 'string'}
                    },
                    'type': 'nested'},
        'closed': {'type': 'boolean'},
        # todo; check with team
        'closed_by': {"type": "string", "index": "not_analyzed"},
        'closed_on': {'format': DATE_FORMATS_STRING,
                      'type': 'date'},
        'computed_': {'enabled': False,
                      'type': 'object'},
        'computed_modified_on_': {'format': DATE_FORMATS_STRING,
                                  'type': 'date'},
        'doc_type': {'index': 'not_analyzed',
                     'type': 'string'},
        'inserted_at': {"type": "date", "format": DATE_FORMATS_STRING},
        'domain': {'fields': {'domain': {'index': 'analyzed',
                                         'type': 'string'},
                              'exact': {'index': 'not_analyzed',
                                        'type': 'string'}},
                   'type': 'multi_field'},
        'export_tag': {'type': 'string'},
        'external_id': {'fields': {'exact': {'index': 'not_analyzed',
                                             'type': 'string'},
                                   'external_id': {'index': 'analyzed',
                                                   'type': 'string'}},
                        'type': 'multi_field'},
        'indices': {'dynamic': False,
                    'properties': {'doc_type': {'index': 'not_analyzed',
                                                'type': 'string'},
                                   'identifier': {'type': 'string'},
                                   'referenced_id': {'type': 'string'},
                                   'referenced_type': {'type': 'string'}},
                    'type': 'object'},
        'initial_processing_complete': {'type': 'boolean'},
        'location_id': {'type': 'string'},
        'modified_on': {'format': DATE_FORMATS_STRING,
                        'type': 'date'},
        'name': {'fields': {'exact': {'index': 'not_analyzed',
                                      'type': 'string'},
                            'name': {'index': 'analyzed',
                                     'type': 'string'}},
                 'type': 'multi_field'},
        # todo; check with team
        'opened_by': {"type": "string", "index": "not_analyzed"},
        'opened_on': {'format': DATE_FORMATS_STRING,
                      'type': 'date'},
        # Todo; check with team whether okay to update this without triggering reindex for other envs
        "owner_id": {"type": "string", "index": "not_analyzed"},
        'owner_type': {'type': 'string', "index": "not_analyzed", "null_value": NULL_VALUE},
        'referrals': {'enabled': False, 'type': 'object'},
        'server_modified_on': {'format': DATE_FORMATS_STRING,
                               'type': 'date'},
        'type': {'fields': {'exact': {'index': 'not_analyzed',
                                      'type': 'string'},
                            'type': {'index': 'analyzed',
                                     'type': 'string'}},
                 'type': 'multi_field'},
        'user_id': {'type': 'string'},
        'version': {'type': 'string'},
        'xform_ids': {'index': 'not_analyzed',
                      'type': 'string'},
        'contact_phone_number': {'index': 'not_analyzed',
                                 'type': 'string'},
        'backend_id': {'type': 'string', 'index': 'not_analyzed'},
    }
}

CASE_ES_ALIAS = prefix_for_tests("hqcases")

CASE_INDEX_INFO = ElasticsearchIndexInfo(
    index=CASE_INDEX,
    alias=CASE_ES_ALIAS,
    type=CASE_ES_TYPE,
    mapping=CASE_MAPPING,
    hq_index_name=CASE_HQ_INDEX_NAME
)
<commit_msg>remove todo comments<commit_after>from corehq.pillows.core import DATE_FORMATS_ARR, DATE_FORMATS_STRING
from corehq.pillows.mappings import NULL_VALUE
from corehq.util.elastic import prefix_for_tests
from pillowtop.es_utils import ElasticsearchIndexInfo, CASE_HQ_INDEX_NAME

CASE_INDEX = prefix_for_tests("hqcases_2016-03-04")
CASE_ES_TYPE = 'case'

CASE_MAPPING = {
    'date_detection': False,
    'date_formats': DATE_FORMATS_ARR,
    'dynamic': False,
    '_meta': {
        'comment': '',
        'created': '2013-09-19 @dmyung'
    },
    'properties': {
        'actions': {'dynamic': False,
                    'properties': {
                        'action_type': {'type': 'string'},
                        'date': {'format': DATE_FORMATS_STRING,
                                 'type': 'date'},
                        'doc_type': {'index': 'not_analyzed',
                                     'type': 'string'},
                        'indices': {'dynamic': False,
                                    'properties': {
                                        'doc_type': {'index': 'not_analyzed',
                                                     'type': 'string'},
                                        'identifier': {'type': 'string'},
                                        'referenced_id': {'type': 'string'},
                                        'referenced_type': {'type': 'string'}
                                    },
                                    'type': 'object'},
                        'server_date': {'format': DATE_FORMATS_STRING,
                                        'type': 'date'},
                        'sync_log_id': {'type': 'string'},
                        'user_id': {'type': 'string'},
                        'xform_id': {'type': 'string'},
                        'xform_name': {'type': 'string'},
                        'xform_xmlns': {'type': 'string'}
                    },
                    'type': 'nested'},
        'closed': {'type': 'boolean'},
        'closed_by': {"type": "string", "index": "not_analyzed"},
        'closed_on': {'format': DATE_FORMATS_STRING,
                      'type': 'date'},
        'computed_': {'enabled': False,
                      'type': 'object'},
        'computed_modified_on_': {'format': DATE_FORMATS_STRING,
                                  'type': 'date'},
        'doc_type': {'index': 'not_analyzed',
                     'type': 'string'},
        'inserted_at': {"type": "date", "format": DATE_FORMATS_STRING},
        'domain': {'fields': {'domain': {'index': 'analyzed',
                                         'type': 'string'},
                              'exact': {'index': 'not_analyzed',
                                        'type': 'string'}},
                   'type': 'multi_field'},
        'export_tag': {'type': 'string'},
        'external_id': {'fields': {'exact': {'index': 'not_analyzed',
                                             'type': 'string'},
                                   'external_id': {'index': 'analyzed',
                                                   'type': 'string'}},
                        'type': 'multi_field'},
        'indices': {'dynamic': False,
                    'properties': {'doc_type': {'index': 'not_analyzed',
                                                'type': 'string'},
                                   'identifier': {'type': 'string'},
                                   'referenced_id': {'type': 'string'},
                                   'referenced_type': {'type': 'string'}},
                    'type': 'object'},
        'initial_processing_complete': {'type': 'boolean'},
        'location_id': {'type': 'string'},
        'modified_on': {'format': DATE_FORMATS_STRING,
                        'type': 'date'},
        'name': {'fields': {'exact': {'index': 'not_analyzed',
                                      'type': 'string'},
                            'name': {'index': 'analyzed',
                                     'type': 'string'}},
                 'type': 'multi_field'},
        'opened_by': {"type": "string", "index": "not_analyzed"},
        'opened_on': {'format': DATE_FORMATS_STRING,
                      'type': 'date'},
        "owner_id": {"type": "string", "index": "not_analyzed"},
        'owner_type': {'type': 'string', "index": "not_analyzed", "null_value": NULL_VALUE},
        'referrals': {'enabled': False, 'type': 'object'},
        'server_modified_on': {'format': DATE_FORMATS_STRING,
                               'type': 'date'},
        'type': {'fields': {'exact': {'index': 'not_analyzed',
                                      'type': 'string'},
                            'type': {'index': 'analyzed',
                                     'type': 'string'}},
                 'type': 'multi_field'},
        'user_id': {'type': 'string'},
        'version': {'type': 'string'},
        'xform_ids': {'index': 'not_analyzed',
                      'type': 'string'},
        'contact_phone_number': {'index': 'not_analyzed',
                                 'type': 'string'},
        'backend_id': {'type': 'string', 'index': 'not_analyzed'},
    }
}

CASE_ES_ALIAS = prefix_for_tests("hqcases")

CASE_INDEX_INFO = ElasticsearchIndexInfo(
    index=CASE_INDEX,
    alias=CASE_ES_ALIAS,
    type=CASE_ES_TYPE,
    mapping=CASE_MAPPING,
    hq_index_name=CASE_HQ_INDEX_NAME
)
<|endoftext|>