<commit_before>from contextlib import contextmanager

from django.conf import settings
import sqlalchemy
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm.session import sessionmaker
from django.core import signals

DEFAULT_ENGINE_ID = 'default'
UCR_ENGINE_ID = 'ucr'
ICDS_UCR_ENGINE_ID = 'icds-ucr'
ICDS_TEST_UCR_ENGINE_ID = 'icds-test-ucr'

def create_engine(connection_string=None):
    connection_string = connection_string or settings.SQL_REPORTING_DATABASE_URL
    # paramstyle='format' allows you to use column names that include the ')' character
    # otherwise queries will sometimes be misformated/error when formatting
    # https://github.com/zzzeek/sqlalchemy/blob/ff20903/lib/sqlalchemy/dialects/postgresql/psycopg2.py#L173
    return sqlalchemy.create_engine(connection_string, paramstyle='format')

class SessionHelper(object):
    """
    Shim class helper for a single connection/session factory
    """

    def __init__(self, connection_string):
        self.engine = create_engine(connection_string)
        self._session_factory = sessionmaker(bind=self.engine)
        # Session is the actual constructor object
        self.Session = scoped_session(self._session_factory)

    @property
    def session_context(self):
        @contextmanager
        def session_scope():
            """Provide a transactional scope around a series of operations."""
            session = self.Session()
            try:
                yield session
                session.commit()
            except:
                session.rollback()
                raise
            finally:
                session.close()

        return session_scope

class ConnectionManager(object):
    """
    Object for dealing with sqlalchemy engines and sessions.
    """

    def __init__(self):
        self._session_helpers = {}

    def _get_or_create_helper(self, engine_id):
        if engine_id not in self._session_helpers:
            self._session_helpers[engine_id] = SessionHelper(self.get_connection_string(engine_id))
        return self._session_helpers[engine_id]

    def get_session_helper(self, engine_id=DEFAULT_ENGINE_ID):
        """
        Returns the SessionHelper object associated with this engine id
        """
        # making this separate from _get_or_create in case we ever want to fail differently here
        return self._get_or_create_helper(engine_id)

    def get_scoped_session(self, engine_id=DEFAULT_ENGINE_ID):
        """
        This returns a handle to a thread-locally scoped session object.
        """
        return self.get_session_helper(engine_id).Session

    def get_engine(self, engine_id=DEFAULT_ENGINE_ID):
        """
        Get an engine by ID. This should be a unique field identifying the connection,
        e.g. "report-db-1"
        """
        return self._get_or_create_helper(engine_id).engine

    def close_scoped_sessions(self):
        for helper in self._session_helpers.values():
            helper.Session.remove()

    def dispose_engine(self, engine_id=DEFAULT_ENGINE_ID):
        """
        If found, closes the active sessions associated with an an engine and disposes it.
        Also removes it from the session manager.
        If not found, does nothing.
        """
        if engine_id in self._session_helpers:
            helper = self._session_helpers.pop(engine_id)
            helper.Session.remove()
            helper.engine.dispose()

    def dispose_all(self):
        """
        Dispose all engines associated with this. Useful for tests.
        """
        for engine_id in self._session_helpers.keys():
            self.dispose_engine(engine_id)

    def get_connection_string(self, engine_id):
        db_connection_map = {
            DEFAULT_ENGINE_ID: settings.SQL_REPORTING_DATABASE_URL,
            UCR_ENGINE_ID: settings.UCR_DATABASE_URL,
        }
        if hasattr(settings, 'ICDS_UCR_DATABASE_ALIAS') and settings.ICDS_UCR_DATABASE_ALIAS in settings.DATABASES:
            db_connection_map[ICDS_UCR_ENGINE_ID] = \
                "postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}".format(
                    **settings.DATABASES[settings.ICDS_UCR_DATABASE_ALIAS]
                )
        if hasattr(
            settings, 'ICDS_UCR_TEST_DATABASE_ALIAS'
        ) and settings.ICDS_UCR_TEST_DATABASE_ALIAS in settings.DATABASES:
            db_connection_map[
                ICDS_TEST_UCR_ENGINE_ID
            ] = "postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}".format(
                **settings.DATABASES[settings.ICDS_UCR_TEST_DATABASE_ALIAS]
            )
        for custom_engine_id, custom_db_url in settings.CUSTOM_DATABASES:
            db_connection_map[custom_engine_id] = custom_db_url
        return db_connection_map.get(engine_id, settings.SQL_REPORTING_DATABASE_URL)

connection_manager = ConnectionManager()
Session = connection_manager.get_scoped_session(DEFAULT_ENGINE_ID)

# Register an event that closes the database connection
# when a Django request is finished.
# This will rollback any open transactions.
def _close_connections(**kwargs):
    Session.remove()  # todo: unclear whether this is necessary
    connection_manager.close_scoped_sessions()

signals.request_finished.connect(_close_connections)
<commit_msg>refactor connection manager<commit_after>from contextlib import contextmanager

from django.conf import settings
import sqlalchemy
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm.session import sessionmaker
from django.core import signals

DEFAULT_ENGINE_ID = 'default'
UCR_ENGINE_ID = 'ucr'
ICDS_UCR_ENGINE_ID = 'icds-ucr'
ICDS_TEST_UCR_ENGINE_ID = 'icds-test-ucr'

def create_engine(connection_string=None):
    connection_string = connection_string or settings.SQL_REPORTING_DATABASE_URL
    # paramstyle='format' allows you to use column names that include the ')' character
    # otherwise queries will sometimes be misformated/error when formatting
    # https://github.com/zzzeek/sqlalchemy/blob/ff20903/lib/sqlalchemy/dialects/postgresql/psycopg2.py#L173
    return sqlalchemy.create_engine(connection_string, paramstyle='format')

class SessionHelper(object):
    """
    Shim class helper for a single connection/session factory
    """

    def __init__(self, connection_string):
        self.engine = create_engine(connection_string)
        self._session_factory = sessionmaker(bind=self.engine)
        # Session is the actual constructor object
        self.Session = scoped_session(self._session_factory)

    @property
    def session_context(self):
        @contextmanager
        def session_scope():
            """Provide a transactional scope around a series of operations."""
            session = self.Session()
            try:
                yield session
                session.commit()
            except:
                session.rollback()
                raise
            finally:
                session.close()

        return session_scope

class ConnectionManager(object):
    """
    Object for dealing with sqlalchemy engines and sessions.
    """

    def __init__(self):
        self._session_helpers = {}
        self.db_connection_map = {
            DEFAULT_ENGINE_ID: settings.SQL_REPORTING_DATABASE_URL,
            UCR_ENGINE_ID: settings.UCR_DATABASE_URL,
        }
        self._populate_connection_map()

    def _get_or_create_helper(self, engine_id):
        if engine_id not in self._session_helpers:
            self._session_helpers[engine_id] = SessionHelper(self.get_connection_string(engine_id))
        return self._session_helpers[engine_id]

    def get_session_helper(self, engine_id=DEFAULT_ENGINE_ID):
        """
        Returns the SessionHelper object associated with this engine id
        """
        # making this separate from _get_or_create in case we ever want to fail differently here
        return self._get_or_create_helper(engine_id)

    def get_scoped_session(self, engine_id=DEFAULT_ENGINE_ID):
        """
        This returns a handle to a thread-locally scoped session object.
        """
        return self.get_session_helper(engine_id).Session

    def get_engine(self, engine_id=DEFAULT_ENGINE_ID):
        """
        Get an engine by ID. This should be a unique field identifying the connection,
        e.g. "report-db-1"
        """
        return self._get_or_create_helper(engine_id).engine

    def close_scoped_sessions(self):
        for helper in self._session_helpers.values():
            helper.Session.remove()

    def dispose_engine(self, engine_id=DEFAULT_ENGINE_ID):
        """
        If found, closes the active sessions associated with an an engine and disposes it.
        Also removes it from the session manager.
        If not found, does nothing.
        """
        if engine_id in self._session_helpers:
            helper = self._session_helpers.pop(engine_id)
            helper.Session.remove()
            helper.engine.dispose()

    def dispose_all(self):
        """
        Dispose all engines associated with this. Useful for tests.
        """
        for engine_id in self._session_helpers.keys():
            self.dispose_engine(engine_id)

    def get_connection_string(self, engine_id):
        return self.db_connection_map.get(engine_id, settings.SQL_REPORTING_DATABASE_URL)

    def _populate_connection_map(self):
        self._add_django_db(ICDS_UCR_ENGINE_ID, 'ICDS_UCR_DATABASE_ALIAS')
        self._add_django_db(ICDS_TEST_UCR_ENGINE_ID, 'ICDS_UCR_TEST_DATABASE_ALIAS')
        for custom_engine_id, custom_db_url in settings.CUSTOM_DATABASES:
            self.db_connection_map[custom_engine_id] = custom_db_url

    def _add_django_db(self, engine_id, django_db_alias):
        if self._django_db_exists(django_db_alias):
            connection_string = self._connection_string_from_django(settings.ICDS_UCR_DATABASE_ALIAS)
            self.db_connection_map[engine_id] = connection_string

    def _connection_string_from_django(self, django_alias):
        return "postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}".format(
                **settings.DATABASES[django_alias]
            )

    def _django_db_exists(self, django_alias):
        return getattr(settings, django_alias, None) in settings.DATABASES

connection_manager = ConnectionManager()
Session = connection_manager.get_scoped_session(DEFAULT_ENGINE_ID)

# Register an event that closes the database connection
# when a Django request is finished.
# This will rollback any open transactions.
def _close_connections(**kwargs):
    Session.remove()  # todo: unclear whether this is necessary
    connection_manager.close_scoped_sessions()

signals.request_finished.connect(_close_connections)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# - # Copyright 2016 Max Fischer
# - #
# - # Licensed under the Apache License, Version 2.0 (the "License");
# - # you may not use this file except in compliance with the License.
# - # You may obtain a copy of the License at
# - #
# - #     http://www.apache.org/licenses/LICENSE-2.0
# - #
# - # Unless required by applicable law or agreed to in writing, software
# - # distributed under the License is distributed on an "AS IS" BASIS,
# - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# - # See the License for the specific language governing permissions and
# - # limitations under the License.
#
# CPy2Py documentation build configuration file, created by
# sphinx-quickstart on Wed Mar 30 13:54:26 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.imgmath',
    'sphinx.ext.viewcode',
    'sphinx.ext.githubpages',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'CPy2Py'
copyright = u'2016, Max Fischer'
author = u'Max Fischer'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
import cpy2py
# The short X.Y version.
version = cpy2py.meta.__version__
# The full version, including alpha/beta/rc tags.
release = cpy2py.meta.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
# html_theme = 'alabaster'
html_theme = 'sphinxdoc'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default.
# html_title = u'CPy2Py v0.9a1'

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['.static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
# html_extra_path = []

# If not None, a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
# The empty string is equivalent to '%b %d, %Y'.
# html_last_updated_fmt = None

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'
# html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# 'ja' uses this config value.
# 'zh' user can custom change `jieba` dictionary path.
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'CPy2Pydoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # 'preamble': '',

    # Latex figure (float) alignment
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'CPy2Py.tex', u'CPy2Py Documentation',
     u'Max Fischer', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'cpy2py', u'CPy2Py Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'CPy2Py', u'CPy2Py Documentation',
     author, 'CPy2Py', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
# epub_basename = project

# The HTML theme for the epub output. Since the default themes are not
# optimized for small screen space, using the same theme for HTML and epub
# output is usually not wise. This defaults to 'epub', a theme designed to save
# visual space.
# epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
# epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
# epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
# epub_identifier = ''

# A unique identification for the text.
# epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
# epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
# epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
# epub_pre_files = []

# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
# epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
# epub_tocdepth = 3

# Allow duplicate toc entries.
# epub_tocdup = True

# Choose between 'default' and 'includehidden'.
# epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
# epub_fix_images = False

# Scale large images.
# epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# epub_show_urls = 'inline'

# If false, no index is generated.
# epub_use_index = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}
<commit_msg>changed docs metadata<commit_after># -*- coding: utf-8 -*-
# - # Copyright 2016 Max Fischer
# - #
# - # Licensed under the Apache License, Version 2.0 (the "License");
# - # you may not use this file except in compliance with the License.
# - # You may obtain a copy of the License at
# - #
# - #     http://www.apache.org/licenses/LICENSE-2.0
# - #
# - # Unless required by applicable law or agreed to in writing, software
# - # distributed under the License is distributed on an "AS IS" BASIS,
# - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# - # See the License for the specific language governing permissions and
# - # limitations under the License.
#
# CPy2Py documentation build configuration file, created by
# sphinx-quickstart on Wed Mar 30 13:54:26 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.imgmath',
    'sphinx.ext.viewcode',
    'sphinx.ext.githubpages',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'CPy2Py'
copyright = u'2016-2018, Max Fischer'
author = u'Max Fischer'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
import cpy2py
# The short X.Y version.
version = cpy2py.meta.__version__
# The full version, including alpha/beta/rc tags.
release = cpy2py.meta.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
# html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.
# "<project> v<release> documentation" by default.
# html_title = u'CPy2Py v0.9a1'

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['.static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
# html_extra_path = []

# If not None, a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
# The empty string is equivalent to '%b %d, %Y'.
# html_last_updated_fmt = None

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'
# html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# 'ja' uses this config value.
# 'zh' user can custom change `jieba` dictionary path.
# html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'CPy2Pydoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # 'preamble': '',

    # Latex figure (float) alignment
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'CPy2Py.tex', u'CPy2Py Documentation',
     u'Max Fischer', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'cpy2py', u'CPy2Py Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'CPy2Py', u'CPy2Py Documentation',
     author, 'CPy2Py', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
# epub_basename = project

# The HTML theme for the epub output. Since the default themes are not
# optimized for small screen space, using the same theme for HTML and epub
# output is usually not wise. This defaults to 'epub', a theme designed to save
# visual space.
# epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
# epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
# epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
# epub_identifier = ''

# A unique identification for the text.
# epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
# epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
# epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
# epub_pre_files = []

# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
# epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
# epub_tocdepth = 3

# Allow duplicate toc entries.
# epub_tocdup = True

# Choose between 'default' and 'includehidden'.
# epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
# epub_fix_images = False

# Scale large images.
# epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# epub_show_urls = 'inline'

# If false, no index is generated.
# epub_use_index = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}
<|endoftext|>


<commit_before>import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
results = []
for i in range(10):
    results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
data = results
modified_data = []
for row in data:
    d = {}
    d.update(row)
    d.update(row['resource'])
    d.update(row['resource']['view_count'])
    d.update(row['classification'])
    # https://data.cityofnewyork.us/Public-Safety/Disposition-Of-Offensive-Language-Allegations-2007/xah7-gu5w
    # https://data.cityofnewyork.us/resource/xah7-gu5w.json
    # use permalink
    d['api_url'] = d['permalink'].replace('/d/', '/resource/') + '.json'
    # ?$select=count(*)
    d['number_of_rows'] =
    modified_data.append(d)
print r.db('public').table('datasets').insert(modified_data).run(conflict='update')
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_created_at'), index=r.desc('socrata_created_at')).limit(int(1)).run())[0]['socrata_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/pu5n-trf4.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
import dateutil
import dateutil.parser
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for row in data:
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_cad_event'] = True
    row['is_rms_event'] = False
    for key in row.keys():
        if key.startswith(':id'):
            row['id'] = row[':id']
            del row[':id']
        elif key.startswith(':'):
            row['socrata_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
print r.db('public').table('police_response_events').insert(data).run(conflict='update')

t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_rms_created_at'), index=r.desc('socrata_rms_created_at')).limit(int(1)).run())[0]['socrata_rms_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/y7pv-r3kh.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
print 'got the data'
#print data
import dateutil
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for i, row in enumerate(data):
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_rms_event'] = True
    #print 'gon', row['general_offense_number']
    try:
        row['id'] = list(r.db('public').table('police_response_events').get_all(row['general_offense_number'], index='general_offense_number').run())[0]['id']
    except:
        #print row.keys()
        row['id'] = row[':id']
    for key in row.keys():
        if key.startswith(':'):
            row['socrata_rms_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
    change = r.db('public').table('police_response_events').insert(row).run(conflict='update')
    print change<commit_msg>auto commit<commit_after>import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
results = []
for i in range(10):
    results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
data = results
modified_data = []
for row in data:
    d = {}
    d.update(row)
    d.update(row['resource'])
    d.update(row['resource']['view_count'])
    d.update(row['classification'])
    # https://data.cityofnewyork.us/Public-Safety/Disposition-Of-Offensive-Language-Allegations-2007/xah7-gu5w
    # https://data.cityofnewyork.us/resource/xah7-gu5w.json
    # use permalink
    d['api_url'] = d['permalink'].replace('/d/', '/resource/') + '.json'
    # ?$select=count(*)

    d['number_of_rows'] =
    modified_data.append(d)
print r.db('public').table('datasets').insert(modified_data).run(conflict='update')
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_created_at'), index=r.desc('socrata_created_at')).limit(int(1)).run())[0]['socrata_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/pu5n-trf4.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
import dateutil
import dateutil.parser
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for row in data:
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_cad_event'] = True
    row['is_rms_event'] = False
    for key in row.keys():
        if key.startswith(':id'):
            row['id'] = row[':id']
            del row[':id']
        elif key.startswith(':'):
            row['socrata_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
print r.db('public').table('police_response_events').insert(data).run(conflict='update')

t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_rms_created_at'), index=r.desc('socrata_rms_created_at')).limit(int(1)).run())[0]['socrata_rms_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/y7pv-r3kh.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
print 'got the data'
#print data
import dateutil
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for i, row in enumerate(data):
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_rms_event'] = True
    #print 'gon', row['general_offense_number']
    try:
        row['id'] = list(r.db('public').table('police_response_events').get_all(row['general_offense_number'], index='general_offense_number').run())[0]['id']
    except:
        #print row.keys()
        row['id'] = row[':id']
    for key in row.keys():
        if key.startswith(':'):
            row['socrata_rms_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
    change = r.db('public').table('police_response_events').insert(row).run(conflict='update')
    print change<|endoftext|>


<commit_before>from datetime import date

from django.conf import settings

import pytz

from custom.icds_reports.data_pull.data_pulls import (
    AndhraPradeshMonthly,
    MonthlyPerformance,
    VHSNDMonthlyReport,
)

ISSUE_TRACKER_APP_ID = '48cc1709b7f62ffea24cc6634a005734'

INDIA_TIMEZONE = pytz.timezone('Asia/Kolkata')

TABLEAU_TICKET_URL = settings.TABLEAU_URL_ROOT + "trusted/"
TABLEAU_VIEW_URL = settings.TABLEAU_URL_ROOT + "#/views/"
TABLEAU_USERNAME = "reportviewer"
TABLEAU_INVALID_TOKEN = '-1'

BHD_ROLE = 'BHD (For VL Dashboard Testing)'

UCR_PILLOWS = ['kafka-ucr-static', 'kafka-ucr-static-cases',
               'kafka-ucr-static-forms', 'kafka-ucr-static-awc-location',
               'kafka-ucr-main']

class NavigationSections:
    MATERNAL_CHILD = 'maternal_child'
    ICDS_CAS_REACH = 'icds_cas_reach'
    DEMOGRAPHICS = 'demographics'
    AWC_INFRASTRUCTURE = 'awc_infrastructure'

class SDDSections:
    PW_LW_CHILDREN = 'pw_lw_children'
    CHILDREN = 'children'

class LocationTypes(object):
    STATE = 'state'
    DISTRICT = 'district'
    BLOCK = 'block'
    SUPERVISOR = 'supervisor'
    AWC = 'awc'

class AggregationLevels(object):
    STATE = 1
    DISTRICT = 2
    BLOCK = 3
    SUPERVISOR = 4
    AWC = 5

class ChartColors(object):
    PINK = '#fcb18d'
    ORANGE = '#fa683c'
    RED = '#bf231d'
    BLUE = '#005ebd'

class MapColors(object):
    RED = '#de2d26'
    ORANGE = '#fc9272'
    BLUE = '#006fdf'
    PINK = '#fee0d2'
    GREY = '#9D9D9D'

LOCATION_TYPES = [
    LocationTypes.STATE,
    LocationTypes.DISTRICT,
    LocationTypes.BLOCK,
    LocationTypes.SUPERVISOR,
    LocationTypes.AWC
]

HELPDESK_ROLES = [
    'BHD',
    'DHD',
    'CPMU',
    'SHD',
    'Test BHD (For VL Dashboard QA)',
    'Test DHD (For VL Dashboard QA)',
    'Test SHD (For VL Dashboard QA)',
    'Test CPMU (For VL Dashboard QA)'
]

CHILDREN_EXPORT = 1
PREGNANT_WOMEN_EXPORT = 2
DEMOGRAPHICS_EXPORT = 3
SYSTEM_USAGE_EXPORT = 4
AWC_INFRASTRUCTURE_EXPORT = 5
GROWTH_MONITORING_LIST_EXPORT = 6
ISSNIP_MONTHLY_REGISTER_PDF = 7
AWW_INCENTIVE_REPORT = 8
LS_REPORT_EXPORT = 9
THR_REPORT_EXPORT = 10
DASHBOARD_USAGE_EXPORT = 11
SERVICE_DELIVERY_REPORT = 12
CHILD_GROWTH_TRACKER_REPORT = 13

AGG_COMP_FEEDING_TABLE = 'icds_dashboard_comp_feed_form'
AGG_CCS_RECORD_CF_TABLE = 'icds_dashboard_ccs_record_cf_forms'
AGG_CCS_RECORD_PNC_TABLE = 'icds_dashboard_ccs_record_postnatal_forms'
AGG_CHILD_HEALTH_PNC_TABLE = 'icds_dashboard_child_health_postnatal_forms'
AGG_CHILD_HEALTH_THR_TABLE = 'icds_dashboard_child_health_thr_forms'
AGG_CCS_RECORD_THR_TABLE = 'icds_dashboard_ccs_record_thr_forms'
AGG_CCS_RECORD_BP_TABLE = 'icds_dashboard_ccs_record_bp_forms'
AGG_CCS_RECORD_DELIVERY_TABLE = 'icds_dashboard_ccs_record_delivery_forms'
AGG_DAILY_FEEDING_TABLE = 'icds_dashboard_daily_feeding_forms'
AGG_GROWTH_MONITORING_TABLE = 'icds_dashboard_growth_monitoring_forms'
AGG_INFRASTRUCTURE_TABLE = 'icds_dashboard_infrastructure_forms'
AWW_INCENTIVE_TABLE = 'icds_dashboard_aww_incentive'
AGG_LS_AWC_VISIT_TABLE = 'icds_dashboard_ls_awc_visits_forms'
AGG_LS_VHND_TABLE = 'icds_dashboard_ls_vhnd_forms'
AGG_LS_BENEFICIARY_TABLE = 'icds_dashboard_ls_beneficiary_forms'
AGG_THR_V2_TABLE = 'icds_dashboard_thr_v2'
AGG_DASHBOARD_ACTIVITY = 'icds_dashboard_user_activity'
AGG_ADOLESCENT_GIRLS_REGISTRATION_TABLE = 'icds_dashboard_adolescent_girls_registration'
AGG_GOV_DASHBOARD_TABLE = 'agg_gov_dashboard'
AGG_MIGRATION_TABLE = 'icds_dashboard_migration_forms'
AGG_AVAILING_SERVICES_TABLE = 'icds_dashboard_availing_service_forms'

AWC_LOCATION_TABLE_ID = 'static-awc_location'
USAGE_TABLE_ID = 'static-usage_forms'
HOUSEHOLD_TABLE_ID = 'static-household_cases'
AWW_USER_TABLE_ID = 'static-commcare_user_cases'
DAILY_FEEDING_TABLE_ID = 'static-daily_feeding_forms'
AGG_SDR_TABLE = 'agg_service_delivery_report'
BIHAR_API_DEMOGRAPHICS_TABLE = 'bihar_api_demographics'
BIHAR_API_MOTHER_DETAILS_TABLE = 'bihar_api_mother_details'
CHILD_VACCINE_TABLE = 'child_vaccines'
CHILD_DELIVERY_FORM_ID = 'static-child_delivery_forms'

DASHBOARD_DOMAIN = 'icds-dashboard-qa' if settings.SERVER_ENVIRONMENT == 'india' else 'icds-cas'

THREE_MONTHS = 60 * 60 * 24 * 95

VALID_LEVELS_FOR_DUMP = [
    '1',  # state
    '2',  # district
    '3',  # block
]

DISTRIBUTED_TABLES = [
    (AGG_CCS_RECORD_DELIVERY_TABLE, 'supervisor_id'),
    (AGG_COMP_FEEDING_TABLE, 'supervisor_id'),
    (AGG_CCS_RECORD_CF_TABLE, 'supervisor_id'),
    (AGG_CHILD_HEALTH_THR_TABLE, 'supervisor_id'),
    (AGG_GROWTH_MONITORING_TABLE, 'supervisor_id'),
    (AGG_CHILD_HEALTH_PNC_TABLE, 'supervisor_id'),
    (AGG_CCS_RECORD_PNC_TABLE, 'supervisor_id'),
    (AGG_CCS_RECORD_BP_TABLE, 'supervisor_id'),
    (AGG_CCS_RECORD_THR_TABLE, 'supervisor_id'),
    (AGG_DAILY_FEEDING_TABLE, 'supervisor_id'),
    ('child_health_monthly', 'supervisor_id'),
    ('ccs_record_monthly', 'supervisor_id'),
    ('daily_attendance', 'supervisor_id'),
]

REFERENCE_TABLES = [
    'awc_location',
    'icds_months'
]

AADHAR_SEEDED_BENEFICIARIES = 'Aadhaar-seeded Beneficiaries'
CHILDREN_ENROLLED_FOR_ANGANWADI_SERVICES = 'Children enrolled for Anganwadi Services'
PREGNANT_WOMEN_ENROLLED_FOR_ANGANWADI_SERVICES = 'Pregnant women enrolled for Anganwadi Services'
LACTATING_WOMEN_ENROLLED_FOR_ANGANWADI_SERVICES = 'Lactating women enrolled for Anganwadi Services'
ADOLESCENT_GIRLS_ENROLLED_FOR_ANGANWADI_SERVICES = 'Adolescent girls enrolled for Anganwadi Services'

OUT_OF_SCHOOL_ADOLESCENT_GIRLS_11_14_YEARS = 'Out of school Adolescent girls (11-14 years)'
NUM_OF_ADOLESCENT_GIRLS_11_14_YEARS = 'Number of adolescent girls (11-14 years)'
NUM_OUT_OF_SCHOOL_ADOLESCENT_GIRLS_11_14_YEARS = 'Number of out of school adolescent girls (11-14 years)'

CAS_API_PAGE_SIZE = 10000

CUSTOM_DATA_PULLS = {
    AndhraPradeshMonthly.slug: AndhraPradeshMonthly,
    MonthlyPerformance.slug: MonthlyPerformance,
    VHSNDMonthlyReport.slug: VHSNDMonthlyReport,
}
<commit_msg>updated dashboard domain to include icds-staging<commit_after>from datetime import date

from django.conf import settings

import pytz

from custom.icds_reports.data_pull.data_pulls import (
    AndhraPradeshMonthly,
    MonthlyPerformance,
    VHSNDMonthlyReport,
)

ISSUE_TRACKER_APP_ID = '48cc1709b7f62ffea24cc6634a005734'

INDIA_TIMEZONE = pytz.timezone('Asia/Kolkata')

TABLEAU_TICKET_URL = settings.TABLEAU_URL_ROOT + "trusted/"
TABLEAU_VIEW_URL = settings.TABLEAU_URL_ROOT + "#/views/"
TABLEAU_USERNAME = "reportviewer"
TABLEAU_INVALID_TOKEN = '-1'

BHD_ROLE = 'BHD (For VL Dashboard Testing)'

UCR_PILLOWS = ['kafka-ucr-static', 'kafka-ucr-static-cases',
               'kafka-ucr-static-forms', 'kafka-ucr-static-awc-location',
               'kafka-ucr-main']

class NavigationSections:
    MATERNAL_CHILD = 'maternal_child'
    ICDS_CAS_REACH = 'icds_cas_reach'
    DEMOGRAPHICS = 'demographics'
    AWC_INFRASTRUCTURE = 'awc_infrastructure'

class SDDSections:
    PW_LW_CHILDREN = 'pw_lw_children'
    CHILDREN = 'children'

class LocationTypes(object):
    STATE = 'state'
    DISTRICT = 'district'
    BLOCK = 'block'
    SUPERVISOR = 'supervisor'
    AWC = 'awc'

class AggregationLevels(object):
    STATE = 1
    DISTRICT = 2
    BLOCK = 3
    SUPERVISOR = 4
    AWC = 5

class ChartColors(object):
    PINK = '#fcb18d'
    ORANGE = '#fa683c'
    RED = '#bf231d'
    BLUE = '#005ebd'

class MapColors(object):
    RED = '#de2d26'
    ORANGE = '#fc9272'
    BLUE = '#006fdf'
    PINK = '#fee0d2'
    GREY = '#9D9D9D'

LOCATION_TYPES = [
    LocationTypes.STATE,
    LocationTypes.DISTRICT,
    LocationTypes.BLOCK,
    LocationTypes.SUPERVISOR,
    LocationTypes.AWC
]

HELPDESK_ROLES = [
    'BHD',
    'DHD',
    'CPMU',
    'SHD',
    'Test BHD (For VL Dashboard QA)',
    'Test DHD (For VL Dashboard QA)',
    'Test SHD (For VL Dashboard QA)',
    'Test CPMU (For VL Dashboard QA)'
]

CHILDREN_EXPORT = 1
PREGNANT_WOMEN_EXPORT = 2
DEMOGRAPHICS_EXPORT = 3
SYSTEM_USAGE_EXPORT = 4
AWC_INFRASTRUCTURE_EXPORT = 5
GROWTH_MONITORING_LIST_EXPORT = 6
ISSNIP_MONTHLY_REGISTER_PDF = 7
AWW_INCENTIVE_REPORT = 8
LS_REPORT_EXPORT = 9
THR_REPORT_EXPORT = 10
DASHBOARD_USAGE_EXPORT = 11
SERVICE_DELIVERY_REPORT = 12
CHILD_GROWTH_TRACKER_REPORT = 13

AGG_COMP_FEEDING_TABLE = 'icds_dashboard_comp_feed_form'
AGG_CCS_RECORD_CF_TABLE = 'icds_dashboard_ccs_record_cf_forms'
AGG_CCS_RECORD_PNC_TABLE = 'icds_dashboard_ccs_record_postnatal_forms'
AGG_CHILD_HEALTH_PNC_TABLE = 'icds_dashboard_child_health_postnatal_forms'
AGG_CHILD_HEALTH_THR_TABLE = 'icds_dashboard_child_health_thr_forms'
AGG_CCS_RECORD_THR_TABLE = 'icds_dashboard_ccs_record_thr_forms'
AGG_CCS_RECORD_BP_TABLE = 'icds_dashboard_ccs_record_bp_forms'
AGG_CCS_RECORD_DELIVERY_TABLE = 'icds_dashboard_ccs_record_delivery_forms'
AGG_DAILY_FEEDING_TABLE = 'icds_dashboard_daily_feeding_forms'
AGG_GROWTH_MONITORING_TABLE = 'icds_dashboard_growth_monitoring_forms'
AGG_INFRASTRUCTURE_TABLE = 'icds_dashboard_infrastructure_forms'
AWW_INCENTIVE_TABLE = 'icds_dashboard_aww_incentive'
AGG_LS_AWC_VISIT_TABLE = 'icds_dashboard_ls_awc_visits_forms'
AGG_LS_VHND_TABLE = 'icds_dashboard_ls_vhnd_forms'
AGG_LS_BENEFICIARY_TABLE = 'icds_dashboard_ls_beneficiary_forms'
AGG_THR_V2_TABLE = 'icds_dashboard_thr_v2'
AGG_DASHBOARD_ACTIVITY = 'icds_dashboard_user_activity'
AGG_ADOLESCENT_GIRLS_REGISTRATION_TABLE = 'icds_dashboard_adolescent_girls_registration'
AGG_GOV_DASHBOARD_TABLE = 'agg_gov_dashboard'
AGG_MIGRATION_TABLE = 'icds_dashboard_migration_forms'
AGG_AVAILING_SERVICES_TABLE = 'icds_dashboard_availing_service_forms'

AWC_LOCATION_TABLE_ID = 'static-awc_location'
USAGE_TABLE_ID = 'static-usage_forms'
HOUSEHOLD_TABLE_ID = 'static-household_cases'
AWW_USER_TABLE_ID = 'static-commcare_user_cases'
DAILY_FEEDING_TABLE_ID = 'static-daily_feeding_forms'
AGG_SDR_TABLE = 'agg_service_delivery_report'
BIHAR_API_DEMOGRAPHICS_TABLE = 'bihar_api_demographics'
BIHAR_API_MOTHER_DETAILS_TABLE = 'bihar_api_mother_details'
CHILD_VACCINE_TABLE = 'child_vaccines'
CHILD_DELIVERY_FORM_ID = 'static-child_delivery_forms'

DASHBOARD_DOMAIN = 'icds-dashboard-qa' if settings.SERVER_ENVIRONMENT in ('india','icds-staging') else 'icds-cas'

THREE_MONTHS = 60 * 60 * 24 * 95

VALID_LEVELS_FOR_DUMP = [
    '1',  # state
    '2',  # district
    '3',  # block
]

DISTRIBUTED_TABLES = [
    (AGG_CCS_RECORD_DELIVERY_TABLE, 'supervisor_id'),
    (AGG_COMP_FEEDING_TABLE, 'supervisor_id'),
    (AGG_CCS_RECORD_CF_TABLE, 'supervisor_id'),
    (AGG_CHILD_HEALTH_THR_TABLE, 'supervisor_id'),
    (AGG_GROWTH_MONITORING_TABLE, 'supervisor_id'),
    (AGG_CHILD_HEALTH_PNC_TABLE, 'supervisor_id'),
    (AGG_CCS_RECORD_PNC_TABLE, 'supervisor_id'),
    (AGG_CCS_RECORD_BP_TABLE, 'supervisor_id'),
    (AGG_CCS_RECORD_THR_TABLE, 'supervisor_id'),
    (AGG_DAILY_FEEDING_TABLE, 'supervisor_id'),
    ('child_health_monthly', 'supervisor_id'),
    ('ccs_record_monthly', 'supervisor_id'),
    ('daily_attendance', 'supervisor_id'),
]

REFERENCE_TABLES = [
    'awc_location',
    'icds_months'
]

AADHAR_SEEDED_BENEFICIARIES = 'Aadhaar-seeded Beneficiaries'
CHILDREN_ENROLLED_FOR_ANGANWADI_SERVICES = 'Children enrolled for Anganwadi Services'
PREGNANT_WOMEN_ENROLLED_FOR_ANGANWADI_SERVICES = 'Pregnant women enrolled for Anganwadi Services'
LACTATING_WOMEN_ENROLLED_FOR_ANGANWADI_SERVICES = 'Lactating women enrolled for Anganwadi Services'
ADOLESCENT_GIRLS_ENROLLED_FOR_ANGANWADI_SERVICES = 'Adolescent girls enrolled for Anganwadi Services'

OUT_OF_SCHOOL_ADOLESCENT_GIRLS_11_14_YEARS = 'Out of school Adolescent girls (11-14 years)'
NUM_OF_ADOLESCENT_GIRLS_11_14_YEARS = 'Number of adolescent girls (11-14 years)'
NUM_OUT_OF_SCHOOL_ADOLESCENT_GIRLS_11_14_YEARS = 'Number of out of school adolescent girls (11-14 years)'

CAS_API_PAGE_SIZE = 10000

CUSTOM_DATA_PULLS = {
    AndhraPradeshMonthly.slug: AndhraPradeshMonthly,
    MonthlyPerformance.slug: MonthlyPerformance,
    VHSNDMonthlyReport.slug: VHSNDMonthlyReport,
}
<|endoftext|>


<commit_before>from __future__ import absolute_import
from django import settings
from corehq.fluff.calculators.case import CasePropertyFilter
import fluff
from fluff.pillow import get_multi_fluff_pillow
from couchforms.models import XFormInstance
from fluff.filters import ORFilter, ANDFilter, CustomFilter
from casexml.apps.case.models import CommCareCase
from corehq.apps.change_feed.topics import CASE, FORM, CASE_SQL, FORM_SQL
from corehq.fluff.calculators.xform import FormPropertyFilter
from custom.intrahealth import (
    COMMANDE_XMLNSES,
    INTRAHEALTH_DOMAINS,
    LIVRAISON_XMLNSES,
    OPERATEUR_XMLNSES,
    RAPTURE_XMLNSES,
    report_calcs,
)
from custom.intrahealth.utils import (
    get_district_id,
    get_district_name,
    get_location_id,
    get_location_id_by_type,
    get_month,
    get_pps_name,
    get_products,
    get_products_id,
    get_real_date,
    get_region_id,
    get_rupture_products,
    get_rupture_products_ids,
    IsExistFormPropertyFilter,
)
from custom.utils.utils import flat_field

IH_DELETED_TYPES = ('XFormArchived', 'XFormDuplicate', 'XFormDeprecated', 'XFormError')
IH_DELETED_CASE_TYPES = ('CommCareCase-Deleted', )

def _get_all_forms():
    form_filters = []
    for xmlns in OPERATEUR_XMLNSES:
        form_filters.append(FormPropertyFilter(xmlns=xmlns))
    return form_filters

class CouvertureFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ORFilter([
        FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0]),
        FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1]),
    ])

    domains = INTRAHEALTH_DOMAINS
    group_by = ('domain', fluff.AttributeGetter('location_id', get_location_id))
    deleted_types = IH_DELETED_TYPES

    location_id = flat_field(get_location_id)
    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    pps_name = flat_field(lambda f: get_pps_name(f))
    district_name = flat_field(lambda f: get_district_name(f))
    month = flat_field(lambda f: get_month(f, 'real_date'))
    real_date_repeat = flat_field(get_real_date)
    registered = report_calcs.PPSRegistered()
    planned = report_calcs.PPSPlaned()

class TauxDeSatisfactionFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ORFilter([
        FormPropertyFilter(xmlns=COMMANDE_XMLNSES[0]),
        FormPropertyFilter(xmlns=COMMANDE_XMLNSES[1]),
        FormPropertyFilter(xmlns=COMMANDE_XMLNSES[2])
    ])
    deleted_types = IH_DELETED_TYPES

    domains = INTRAHEALTH_DOMAINS
    group_by = (fluff.AttributeGetter('product_name', lambda f: get_products(f, 'productName')),
                fluff.AttributeGetter('product_id', lambda f: get_products_id(f, 'productName')))

    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    commandes = report_calcs.TauxCalculator(property_name='amountOrdered')
    recus = report_calcs.TauxCalculator(property_name='amountReceived')

class IntraHealthFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ORFilter(
        [
            ANDFilter([
                FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0]),
                IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0],
                                          property_path="form",
                                          property_value='district_name'),
                IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0], property_path="form",
                                          property_value='PPS_name')
            ]),
            ANDFilter([
                FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1]),
                IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1],
                                          property_path="form",
                                          property_value='district_name'),
                IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1], property_path="form",
                                          property_value='PPS_name')
            ]),
        ]
    )
    domains = INTRAHEALTH_DOMAINS
    deleted_types = IH_DELETED_TYPES
    group_by = (fluff.AttributeGetter('product_name', lambda f: get_products(f, 'product_name')),
                fluff.AttributeGetter('product_id', lambda f: get_products_id(f, 'product_name')))

    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    PPS_name = flat_field(lambda f: get_pps_name(f))
    district_name = flat_field(lambda f: get_district_name(f))
    location_id = flat_field(get_location_id)

    actual_consumption = report_calcs.PPSConsumption()
    billed_consumption = report_calcs.PPSConsumption(field='billed_consumption')
    stock = report_calcs.PPSConsumption('old_stock_total')
    total_stock = report_calcs.PPSConsumption('total_stock')
    quantity = report_calcs.PPSConsumption('display_total_stock')
    cmm = report_calcs.PPSConsumption('default_consumption')

class RecapPassageFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ANDFilter([
        ORFilter(
            [FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0]), FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1])]
        ),
        ORFilter([
            IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0], property_path="form", property_value='PPS_name'),
            IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1], property_path="form", property_value='PPS_name')
        ])

    ])

    domains = INTRAHEALTH_DOMAINS
    deleted_types = IH_DELETED_TYPES
    group_by = (fluff.AttributeGetter('product_name', lambda f: get_products(f, 'product_name')),
                fluff.AttributeGetter('product_id', lambda f: get_products_id(f, 'product_name')))

    location_id = flat_field(get_location_id)
    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    real_date_repeat = flat_field(get_real_date)
    PPS_name = flat_field(lambda f: f.form['PPS_name'])

    product = report_calcs.RecapPassage()

class TauxDeRuptureFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ANDFilter([
        FormPropertyFilter(xmlns=RAPTURE_XMLNSES[0]),
        IsExistFormPropertyFilter(xmlns=RAPTURE_XMLNSES[0], property_path="form", property_value='district')
    ])
    domains = INTRAHEALTH_DOMAINS
    deleted_types = IH_DELETED_TYPES
    group_by = (fluff.AttributeGetter('product_name', lambda f: get_rupture_products(f)),
                fluff.AttributeGetter('product_id', lambda f: get_rupture_products_ids(f)))

    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    district_name = flat_field(lambda f: f.form['district'])
    PPS_name = flat_field(lambda f: CommCareCase.get(f.form['case']['@case_id']).name)

    total_stock = report_calcs.RupturesDeStocks('pps_stocked_out')

class LivraisonFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = FormPropertyFilter(xmlns=LIVRAISON_XMLNSES[0])

    domains = INTRAHEALTH_DOMAINS
    group_by = ('domain', )
    deleted_types = IH_DELETED_TYPES

    month = flat_field(lambda f: get_month(f, 'mois_visite'))
    duree_moyenne_livraison = report_calcs.DureeMoyenneLivraison()

    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: CommCareCase.get(f.form['case']['@case_id']).location_id)
    district_name = flat_field(lambda f: CommCareCase.get(f.form['case']['@case_id']).name)

class RecouvrementFluff(fluff.IndicatorDocument):
    document_class = CommCareCase
    kafka_topic = CASE_SQL if settings.SERVER_ENVIRONMENT == 'pna' else CASE

    document_filter = ANDFilter([
        CasePropertyFilter(type='payment'),
        CustomFilter(lambda c: hasattr(c, 'district_name'))
    ])

    domains = INTRAHEALTH_DOMAINS
    deleted_types = IH_DELETED_CASE_TYPES
    group_by = ('domain', fluff.AttributeGetter('district_name',
                                                lambda case: case.get_case_property('district_name')))

    region_id = flat_field(lambda c: get_region_id(c))
    district_id = flat_field(lambda c: get_district_id(c))
    district_name = flat_field(lambda c: c['district_name'])

    payments = report_calcs.Recouvrement()

RecouvrementFluffPillow = RecouvrementFluff.pillow()

def IntraHealthFormFluffPillow(delete_filtered=False):
    return get_multi_fluff_pillow(
        indicator_classes=[
            TauxDeSatisfactionFluff,
            CouvertureFluff,
            RecapPassageFluff,
            IntraHealthFluff,
            TauxDeRuptureFluff,
            LivraisonFluff,
        ],
        name='IntraHealthFormFluff',
        kafka_topic=FORM_SQL if settings.SERVER_ENVIRONMENT == 'pna' else FORM,
        delete_filtered=delete_filtered
    )

# Remove these once they are out of the pillow retry queue
CouvertureFluffPillow = CouvertureFluff.pillow()
IntraHealthFluffPillow = IntraHealthFluff.pillow()
<commit_msg>Fix import.<commit_after>from __future__ import absolute_import
from django.conf import settings
from corehq.fluff.calculators.case import CasePropertyFilter
import fluff
from fluff.pillow import get_multi_fluff_pillow
from couchforms.models import XFormInstance
from fluff.filters import ORFilter, ANDFilter, CustomFilter
from casexml.apps.case.models import CommCareCase
from corehq.apps.change_feed.topics import CASE, FORM, CASE_SQL, FORM_SQL
from corehq.fluff.calculators.xform import FormPropertyFilter
from custom.intrahealth import (
    COMMANDE_XMLNSES,
    INTRAHEALTH_DOMAINS,
    LIVRAISON_XMLNSES,
    OPERATEUR_XMLNSES,
    RAPTURE_XMLNSES,
    report_calcs,
)
from custom.intrahealth.utils import (
    get_district_id,
    get_district_name,
    get_location_id,
    get_location_id_by_type,
    get_month,
    get_pps_name,
    get_products,
    get_products_id,
    get_real_date,
    get_region_id,
    get_rupture_products,
    get_rupture_products_ids,
    IsExistFormPropertyFilter,
)
from custom.utils.utils import flat_field

IH_DELETED_TYPES = ('XFormArchived', 'XFormDuplicate', 'XFormDeprecated', 'XFormError')
IH_DELETED_CASE_TYPES = ('CommCareCase-Deleted', )

def _get_all_forms():
    form_filters = []
    for xmlns in OPERATEUR_XMLNSES:
        form_filters.append(FormPropertyFilter(xmlns=xmlns))
    return form_filters

class CouvertureFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ORFilter([
        FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0]),
        FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1]),
    ])

    domains = INTRAHEALTH_DOMAINS
    group_by = ('domain', fluff.AttributeGetter('location_id', get_location_id))
    deleted_types = IH_DELETED_TYPES

    location_id = flat_field(get_location_id)
    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    pps_name = flat_field(lambda f: get_pps_name(f))
    district_name = flat_field(lambda f: get_district_name(f))
    month = flat_field(lambda f: get_month(f, 'real_date'))
    real_date_repeat = flat_field(get_real_date)
    registered = report_calcs.PPSRegistered()
    planned = report_calcs.PPSPlaned()

class TauxDeSatisfactionFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ORFilter([
        FormPropertyFilter(xmlns=COMMANDE_XMLNSES[0]),
        FormPropertyFilter(xmlns=COMMANDE_XMLNSES[1]),
        FormPropertyFilter(xmlns=COMMANDE_XMLNSES[2])
    ])
    deleted_types = IH_DELETED_TYPES

    domains = INTRAHEALTH_DOMAINS
    group_by = (fluff.AttributeGetter('product_name', lambda f: get_products(f, 'productName')),
                fluff.AttributeGetter('product_id', lambda f: get_products_id(f, 'productName')))

    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    commandes = report_calcs.TauxCalculator(property_name='amountOrdered')
    recus = report_calcs.TauxCalculator(property_name='amountReceived')

class IntraHealthFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ORFilter(
        [
            ANDFilter([
                FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0]),
                IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0],
                                          property_path="form",
                                          property_value='district_name'),
                IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0], property_path="form",
                                          property_value='PPS_name')
            ]),
            ANDFilter([
                FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1]),
                IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1],
                                          property_path="form",
                                          property_value='district_name'),
                IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1], property_path="form",
                                          property_value='PPS_name')
            ]),
        ]
    )
    domains = INTRAHEALTH_DOMAINS
    deleted_types = IH_DELETED_TYPES
    group_by = (fluff.AttributeGetter('product_name', lambda f: get_products(f, 'product_name')),
                fluff.AttributeGetter('product_id', lambda f: get_products_id(f, 'product_name')))

    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    PPS_name = flat_field(lambda f: get_pps_name(f))
    district_name = flat_field(lambda f: get_district_name(f))
    location_id = flat_field(get_location_id)

    actual_consumption = report_calcs.PPSConsumption()
    billed_consumption = report_calcs.PPSConsumption(field='billed_consumption')
    stock = report_calcs.PPSConsumption('old_stock_total')
    total_stock = report_calcs.PPSConsumption('total_stock')
    quantity = report_calcs.PPSConsumption('display_total_stock')
    cmm = report_calcs.PPSConsumption('default_consumption')

class RecapPassageFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ANDFilter([
        ORFilter(
            [FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0]), FormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1])]
        ),
        ORFilter([
            IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[0], property_path="form", property_value='PPS_name'),
            IsExistFormPropertyFilter(xmlns=OPERATEUR_XMLNSES[1], property_path="form", property_value='PPS_name')
        ])

    ])

    domains = INTRAHEALTH_DOMAINS
    deleted_types = IH_DELETED_TYPES
    group_by = (fluff.AttributeGetter('product_name', lambda f: get_products(f, 'product_name')),
                fluff.AttributeGetter('product_id', lambda f: get_products_id(f, 'product_name')))

    location_id = flat_field(get_location_id)
    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    real_date_repeat = flat_field(get_real_date)
    PPS_name = flat_field(lambda f: f.form['PPS_name'])

    product = report_calcs.RecapPassage()

class TauxDeRuptureFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = ANDFilter([
        FormPropertyFilter(xmlns=RAPTURE_XMLNSES[0]),
        IsExistFormPropertyFilter(xmlns=RAPTURE_XMLNSES[0], property_path="form", property_value='district')
    ])
    domains = INTRAHEALTH_DOMAINS
    deleted_types = IH_DELETED_TYPES
    group_by = (fluff.AttributeGetter('product_name', lambda f: get_rupture_products(f)),
                fluff.AttributeGetter('product_id', lambda f: get_rupture_products_ids(f)))

    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: get_location_id_by_type(form=f, type='district'))
    district_name = flat_field(lambda f: f.form['district'])
    PPS_name = flat_field(lambda f: CommCareCase.get(f.form['case']['@case_id']).name)

    total_stock = report_calcs.RupturesDeStocks('pps_stocked_out')

class LivraisonFluff(fluff.IndicatorDocument):
    document_class = XFormInstance
    document_filter = FormPropertyFilter(xmlns=LIVRAISON_XMLNSES[0])

    domains = INTRAHEALTH_DOMAINS
    group_by = ('domain', )
    deleted_types = IH_DELETED_TYPES

    month = flat_field(lambda f: get_month(f, 'mois_visite'))
    duree_moyenne_livraison = report_calcs.DureeMoyenneLivraison()

    region_id = flat_field(lambda f: get_location_id_by_type(form=f, type=u'r\xe9gion'))
    district_id = flat_field(lambda f: CommCareCase.get(f.form['case']['@case_id']).location_id)
    district_name = flat_field(lambda f: CommCareCase.get(f.form['case']['@case_id']).name)

class RecouvrementFluff(fluff.IndicatorDocument):
    document_class = CommCareCase
    kafka_topic = CASE_SQL if settings.SERVER_ENVIRONMENT == 'pna' else CASE

    document_filter = ANDFilter([
        CasePropertyFilter(type='payment'),
        CustomFilter(lambda c: hasattr(c, 'district_name'))
    ])

    domains = INTRAHEALTH_DOMAINS
    deleted_types = IH_DELETED_CASE_TYPES
    group_by = ('domain', fluff.AttributeGetter('district_name',
                                                lambda case: case.get_case_property('district_name')))

    region_id = flat_field(lambda c: get_region_id(c))
    district_id = flat_field(lambda c: get_district_id(c))
    district_name = flat_field(lambda c: c['district_name'])

    payments = report_calcs.Recouvrement()

RecouvrementFluffPillow = RecouvrementFluff.pillow()

def IntraHealthFormFluffPillow(delete_filtered=False):
    return get_multi_fluff_pillow(
        indicator_classes=[
            TauxDeSatisfactionFluff,
            CouvertureFluff,
            RecapPassageFluff,
            IntraHealthFluff,
            TauxDeRuptureFluff,
            LivraisonFluff,
        ],
        name='IntraHealthFormFluff',
        kafka_topic=FORM_SQL if settings.SERVER_ENVIRONMENT == 'pna' else FORM,
        delete_filtered=delete_filtered
    )

# Remove these once they are out of the pillow retry queue
CouvertureFluffPillow = CouvertureFluff.pillow()
IntraHealthFluffPillow = IntraHealthFluff.pillow()
<|endoftext|>


<commit_before># Copyright (c) 2013, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import cybox
import cybox.bindings.file_object as file_binding
from cybox.common import (DateTime, ExtractedFeatures, HashList, HexBinary,
        ObjectProperties, String, UnsignedLong)

class FilePath(String):
    _binding = file_binding
    _binding_class = file_binding.FilePathType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'

    def __init__(self, *args, **kwargs):
        String.__init__(self, *args, **kwargs)
        self.fully_qualified = None

    def is_plain(self):
        return (super(FilePath, self).is_plain() and
                self.fully_qualified is None)

    def to_obj(self):
        filepath_obj = String.to_obj(self)
        if self.fully_qualified is not None:
            filepath_obj.set_fully_qualified(self.fully_qualified)
        return filepath_obj

    def to_dict(self):
        filepath_dict = String.to_dict(self)
        if self.fully_qualified is not None:
            filepath_dict['fully_qualified'] = self.fully_qualified
        return filepath_dict

    @staticmethod
    def from_obj(filepath_obj):
        if not filepath_obj:
            return None
        filepath = FilePath()
        filepath._populate_from_obj(filepath_obj)
        filepath.fully_qualified = filepath_obj.get_fully_qualified()
        return filepath

    @staticmethod
    def from_dict(filepath_dict):
        if not filepath_dict:
            return None
        filepath = FilePath()
        # TODO: properly handle plain (non-dict) value
        filepath._populate_from_dict(filepath_dict)
        filepath.fully_qualified = filepath_dict.get('fully_qualified')
        return filepath

class File(ObjectProperties):
    _binding = file_binding
    _binding_class = file_binding.FileObjectType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'
    _XSI_NS = "FileObj"
    _XSI_TYPE = "FileObjectType"

    is_packed = cybox.TypedField("is_packed")
    file_name = cybox.TypedField("File_Name", String)
    file_path = cybox.TypedField("File_Path", FilePath)
    device_path = cybox.TypedField("Device_Path", String)
    full_path = cybox.TypedField("Full_Path", String)
    file_extension = cybox.TypedField("File_Extension", String)
    size_in_bytes = cybox.TypedField("Size_In_Bytes", UnsignedLong)
    magic_number = cybox.TypedField("Magic_Number", HexBinary)
    file_format = cybox.TypedField("File_Format", String)
    modified_time = cybox.TypedField("Modified_Time", String)
    accessed_time = cybox.TypedField("Accessed_Time", String)
    created_time = cybox.TypedField("Created_Time", DateTime)
    hashes = cybox.TypedField("Hashes", HashList)
    extracted_features = cybox.TypedField("Extracted_Features",
                                          ExtractedFeatures)

    # Not supported yet:
    # - Digital_Signatures
    # - File_Attributes_List
    # - Permissions
    # - User_Owner
    # - Packer_List
    # - Peak_Entropy
    # - Sym_Links
    # - Extracted_Features
    # - Byte Runs

    def __init__(self):
        super(File, self).__init__()
        self.is_packed = None

    @property
    def md5(self):
        if self.hashes is None:
            return None
        return self.hashes.md5

    @md5.setter
    def md5(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.md5 = value

    @property
    def sha1(self):
        if self.hashes is None:
            return None
        return self.hashes.sha1

    @sha1.setter
    def sha1(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.sha1 = value

    @property
    def sha256(self):
        if self.hashes is None:
            return None
        return self.hashes.sha256

    @sha256.setter
    def sha256(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.sha256 = value

    @property
    def size(self):
        """`size` is an alias for `size_in_bytes`"""
        return self.size_in_bytes

    @size.setter
    def size(self, value):
        """`size` is an alias for `size_in_bytes`"""
        self.size_in_bytes = value

    def add_hash(self, hash_):
        if hash_ is not None:
            if self.hashes is None:
                self.hashes = HashList()
            self.hashes.append(hash_)

class Packer(cybox.Entity):
    _binding = file_binding
    _binding_class = file_binding.PackerType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'

    name = cybox.TypedField("Name", String)
    version = cybox.TypedField("Version", String)
    entry_point = cybox.TypedField("Entry_Point", HexBinary)
    signature = cybox.TypedField("Signature", String)
    type_ = cybox.TypedField("Type", String)
    #TODO: add Detected_Entrypoint_Signatures and EP_Jump_Codes

class PackerList(cybox.EntityList):
    _binding = file_binding
    _binding_class = file_binding.PackerListType
    _binding_var = "Packer"
    _contained_type = Packer
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'
<commit_msg>Added dictionary instance checking to from_dict in FilePath<commit_after># Copyright (c) 2013, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import cybox
import cybox.bindings.file_object as file_binding
from cybox.common import (DateTime, ExtractedFeatures, HashList, HexBinary,
        ObjectProperties, String, UnsignedLong)

class FilePath(String):
    _binding = file_binding
    _binding_class = file_binding.FilePathType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'

    def __init__(self, *args, **kwargs):
        String.__init__(self, *args, **kwargs)
        self.fully_qualified = None

    def is_plain(self):
        return (super(FilePath, self).is_plain() and
                self.fully_qualified is None)

    def to_obj(self):
        filepath_obj = String.to_obj(self)
        if self.fully_qualified is not None:
            filepath_obj.set_fully_qualified(self.fully_qualified)
        return filepath_obj

    def to_dict(self):
        filepath_dict = String.to_dict(self)
        if self.fully_qualified is not None:
            filepath_dict['fully_qualified'] = self.fully_qualified
        return filepath_dict

    @staticmethod
    def from_obj(filepath_obj):
        if not filepath_obj:
            return None
        filepath = FilePath()
        filepath._populate_from_obj(filepath_obj)
        filepath.fully_qualified = filepath_obj.get_fully_qualified()
        return filepath

    @staticmethod
    def from_dict(filepath_dict):
        if not filepath_dict:
            return None
        filepath = FilePath()
        filepath._populate_from_dict(filepath_dict)
        if isinstance(filepath_dict, dict):
            filepath.fully_qualified = filepath_dict.get('fully_qualified')
        return filepath

class File(ObjectProperties):
    _binding = file_binding
    _binding_class = file_binding.FileObjectType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'
    _XSI_NS = "FileObj"
    _XSI_TYPE = "FileObjectType"

    is_packed = cybox.TypedField("is_packed")
    file_name = cybox.TypedField("File_Name", String)
    file_path = cybox.TypedField("File_Path", FilePath)
    device_path = cybox.TypedField("Device_Path", String)
    full_path = cybox.TypedField("Full_Path", String)
    file_extension = cybox.TypedField("File_Extension", String)
    size_in_bytes = cybox.TypedField("Size_In_Bytes", UnsignedLong)
    magic_number = cybox.TypedField("Magic_Number", HexBinary)
    file_format = cybox.TypedField("File_Format", String)
    modified_time = cybox.TypedField("Modified_Time", String)
    accessed_time = cybox.TypedField("Accessed_Time", String)
    created_time = cybox.TypedField("Created_Time", DateTime)
    hashes = cybox.TypedField("Hashes", HashList)
    extracted_features = cybox.TypedField("Extracted_Features",
                                          ExtractedFeatures)

    # Not supported yet:
    # - Digital_Signatures
    # - File_Attributes_List
    # - Permissions
    # - User_Owner
    # - Packer_List
    # - Peak_Entropy
    # - Sym_Links
    # - Extracted_Features
    # - Byte Runs

    def __init__(self):
        super(File, self).__init__()
        self.is_packed = None

    @property
    def md5(self):
        if self.hashes is None:
            return None
        return self.hashes.md5

    @md5.setter
    def md5(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.md5 = value

    @property
    def sha1(self):
        if self.hashes is None:
            return None
        return self.hashes.sha1

    @sha1.setter
    def sha1(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.sha1 = value

    @property
    def sha256(self):
        if self.hashes is None:
            return None
        return self.hashes.sha256

    @sha256.setter
    def sha256(self, value):
        if self.hashes is None:
            self.hashes = HashList()
        self.hashes.sha256 = value

    @property
    def size(self):
        """`size` is an alias for `size_in_bytes`"""
        return self.size_in_bytes

    @size.setter
    def size(self, value):
        """`size` is an alias for `size_in_bytes`"""
        self.size_in_bytes = value

    def add_hash(self, hash_):
        if hash_ is not None:
            if self.hashes is None:
                self.hashes = HashList()
            self.hashes.append(hash_)

class Packer(cybox.Entity):
    _binding = file_binding
    _binding_class = file_binding.PackerType
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'

    name = cybox.TypedField("Name", String)
    version = cybox.TypedField("Version", String)
    entry_point = cybox.TypedField("Entry_Point", HexBinary)
    signature = cybox.TypedField("Signature", String)
    type_ = cybox.TypedField("Type", String)
    #TODO: add Detected_Entrypoint_Signatures and EP_Jump_Codes

class PackerList(cybox.EntityList):
    _binding = file_binding
    _binding_class = file_binding.PackerListType
    _binding_var = "Packer"
    _contained_type = Packer
    _namespace = 'http://cybox.mitre.org/objects#FileObject-2'
<|endoftext|>