<commit_before>import os

# Import global settings to make it easier to extend settings.
from django.conf.global_settings import *

# Import the project module to calculate directories relative to the module
# location.
PROJECT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../..')
PROJECT_ROOT, PROJECT_MODULE_NAME = os.path.split(
    os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
)

# List all Django apps here. Note that standard Python libraries should not
# be added to this list since Django will not recognize them as apps anyway.
# An app is really only an "app" if a `models` module or package is defined.
# Read more about projects vs. apps here:
# https://docs.djangoproject.com/en/1.3/intro/tutorial01/#creating-models
INSTALLED_APPS = (
    'omop_harvest',

    'south',
    'serrano',
    'avocado',
    'modeltree',
    'haystack',

    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.humanize',
    'django.contrib.markup',
    'django.contrib.messages',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',

    'registration',
)

#
# ADMINISTRATIVE
#

# Admins receive any error messages by email if DEBUG is False
ADMINS = ()

# Managers receive broken link emails if SEND_BROKEN_LINK_EMAILS is True
MANAGERS = ADMINS

# List of IP addresses which will show debug comments
INTERNAL_IPS = ('127.0.0.1', '::1')

DEBUG = True
TEMPLATE_DEBUG = DEBUG

DIR = os.path.abspath(os.path.dirname(__file__))

#
# DATABASES
# Each database can be specified here, but passwords should be in a separate
# file that is not versioned. Use ``local_settings.py``.
#

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(PROJECT_PATH, 'omop_harvest.db')
    }
}

#
# LOCALITY
#

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = None

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = False

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = False

#
# STATIC AND MEDIA
# The application's static files should be placed in the STATIC_ROOT in
# addition to other static files found in third-party apps. The MEDIA_ROOT
# is intended for user uploaded files.
#

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = os.path.join(PROJECT_PATH, '_site/media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = os.path.join(PROJECT_PATH, '_site/static')

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# URL prefix for admin static files -- CSS, JavaScript and images.
# Make sure to use a trailing slash.
# Examples: "http://foo.com/static/admin/", "/static/admin/".
ADMIN_MEDIA_PREFIX = '/static/admin/'

# Additional locations of static files
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
STATICFILES_DIRS = ()

#
# TEMPLATES
#

# Project level templates and template directories that override
# third-party app templates.
TEMPLATE_DIRS = ()

# Context processors are simply functions that return a dict which augments the
# template context.
TEMPLATE_CONTEXT_PROCESSORS += (
    'django.core.context_processors.request',
    'omop_harvest.context_processors.static',
)

#
# URLS
#

# FORCE_SCRIPT_NAME overrides the interpreted 'SCRIPT_NAME' provided by the
# web server. since the URLs below are used for various purposes outside of
# the WSGI application (static and media files), these need to be updated to
# reflect this discrepancy.
FORCE_SCRIPT_NAME = ''
LOGIN_URL = FORCE_SCRIPT_NAME + '/login/'
LOGIN_REDIRECT_URL = FORCE_SCRIPT_NAME + '/query/'
LOGOUT_URL = '/logout/'
ROOT_URLCONF = 'omop_harvest.conf.urls'

# For non-publicly accessible applications, the siteauth app can be used to
# restrict access site-wide.
# SITEAUTH_ACCESS_ORDER = 'allow/deny'
#
SITEAUTH_ALLOW_URLS = (
    r'^log(in|out)/',
    r'^password/reset/',
    r'^(register|verify)/',
)

SITEAUTH_DENY_URLS = (
    r'^workspace/',
    r'^workspace/discover/',
    r'^query/',
    r'^results/+',
    r'^api/+',
    r'^details/\d+/',
    r'^moderate/+',
    r'^verify/+',
)

#
# MIDDLEWARE
#

MIDDLEWARE_CLASSES = (
    'django.middleware.gzip.GZipMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'siteauth.middleware.SiteAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'serrano.middleware.SessionMiddleware',
)

#
# EMAIL
#

SUPPORT_EMAIL = 'cbmisupport@email.chop.edu'
DEFAULT_FROM_EMAIL = 'cbmisupport@email.chop.edu'
EMAIL_SUBJECT_PREFIX = '[omop_harvest] '
SEND_BROKEN_LINK_EMAILS = False

#
# LOGGING
#

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters' :{
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        }
    },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'default': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': 'logs/omop.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount': 5,
            'formatter':'standard',
        },
        'request_handler': {
                'level':'DEBUG',
                'class':'logging.handlers.RotatingFileHandler',
                'filename': 'logs/omop_request.log',
                'maxBytes': 1024*1024*5, # 5 MB
                'backupCount': 5,
                'formatter':'standard',
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console':{
            'level': 'DEBUG',
            'class':'logging.StreamHandler',
        }
    },
    'loggers': {
        '': {
            'handlers': ['default'],
            'level': 'DEBUG',
            'propagate': True
        },
        'django.request': {
            'handlers': ['request_handler'],
            'level': 'DEBUG',
            'propagate': False
        },
        'avocado':{
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': True,
        },
        'serrano':{
            'handlers':['console'],
            'level': 'DEBUG',
            'propagate':True,
        },
    }
}

#
# CACHE
#

# For production environments, the memcached backend is highly recommended
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'unique',
        'KEY_PREFIX': 'omop_harvest',
        'VERSION': 1,
    }
}

CACHE_MIDDLEWARE_SECONDS = 0

# This is not necessary to set if the above `KEY_PREFIX` value is set since
# the `KEY_PREFIX` namespaces all cache set by this application
CACHE_MIDDLEWARE_KEY_PREFIX = 'omop_harvest'

#
# SESSIONS AND COOKIES
#

CSRF_COOKIE_NAME = 'omop_harvest_csrftoken'

# SESSION_COOKIE_AGE = 60 * 20
SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
SESSION_COOKIE_NAME = 'omop_harvest_sessionid'
SESSION_EXPIRE_AT_BROWSER_CLOSE = False
SESSION_SAVE_EVERY_REQUEST = False

#
# OTHER PROJECT SETTINGS
#

# USE_ETAGS = True
IGNORABLE_404_PATHS = (
    r'robots.txt$',
    r'favicon.ico$',
)

#
# VARIOUS APP SETTINGS
#

# The primary key of the ``Site`` object for the Sites Framework
SITE_ID = 1

#
# ModelTrees Configuration
#

MODELTREES = {
    'default': {
        'model': 'omop_harvest.Person',
    }
}

#
# Haystack Configuration
#
HAYSTACK_SITECONF = 'avocado.search_sites'

HAYSTACK_SEARCH_ENGINE = 'whoosh'

HAYSTACK_WHOOSH_PATH = os.path.join(os.path.dirname(__file__), 'whoosh.index')

AVOCADO = {
    'DATA_CHACHE_ENABLED': False,
    'METADATA_MIGRATION_APP': 'omop_harvest',
}
<commit_msg>Re-enabled avocado data cache<commit_after>import os

# Import global settings to make it easier to extend settings.
from django.conf.global_settings import *

# Import the project module to calculate directories relative to the module
# location.
PROJECT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../..')
PROJECT_ROOT, PROJECT_MODULE_NAME = os.path.split(
    os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
)

# List all Django apps here. Note that standard Python libraries should not
# be added to this list since Django will not recognize them as apps anyway.
# An app is really only an "app" if a `models` module or package is defined.
# Read more about projects vs. apps here:
# https://docs.djangoproject.com/en/1.3/intro/tutorial01/#creating-models
INSTALLED_APPS = (
    'omop_harvest',

    'south',
    'serrano',
    'avocado',
    'modeltree',
    'haystack',

    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.humanize',
    'django.contrib.markup',
    'django.contrib.messages',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',

    'registration',
)

#
# ADMINISTRATIVE
#

# Admins receive any error messages by email if DEBUG is False
ADMINS = ()

# Managers receive broken link emails if SEND_BROKEN_LINK_EMAILS is True
MANAGERS = ADMINS

# List of IP addresses which will show debug comments
INTERNAL_IPS = ('127.0.0.1', '::1')

DEBUG = True
TEMPLATE_DEBUG = DEBUG

DIR = os.path.abspath(os.path.dirname(__file__))

#
# DATABASES
# Each database can be specified here, but passwords should be in a separate
# file that is not versioned. Use ``local_settings.py``.
#

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(PROJECT_PATH, 'omop_harvest.db')
    }
}

#
# LOCALITY
#

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = None

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = False

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = False

#
# STATIC AND MEDIA
# The application's static files should be placed in the STATIC_ROOT in
# addition to other static files found in third-party apps. The MEDIA_ROOT
# is intended for user uploaded files.
#

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = os.path.join(PROJECT_PATH, '_site/media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = os.path.join(PROJECT_PATH, '_site/static')

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# URL prefix for admin static files -- CSS, JavaScript and images.
# Make sure to use a trailing slash.
# Examples: "http://foo.com/static/admin/", "/static/admin/".
ADMIN_MEDIA_PREFIX = '/static/admin/'

# Additional locations of static files
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
STATICFILES_DIRS = ()

#
# TEMPLATES
#

# Project level templates and template directories that override
# third-party app templates.
TEMPLATE_DIRS = ()

# Context processors are simply functions that return a dict which augments the
# template context.
TEMPLATE_CONTEXT_PROCESSORS += (
    'django.core.context_processors.request',
    'omop_harvest.context_processors.static',
)

#
# URLS
#

# FORCE_SCRIPT_NAME overrides the interpreted 'SCRIPT_NAME' provided by the
# web server. since the URLs below are used for various purposes outside of
# the WSGI application (static and media files), these need to be updated to
# reflect this discrepancy.
FORCE_SCRIPT_NAME = ''
LOGIN_URL = FORCE_SCRIPT_NAME + '/login/'
LOGIN_REDIRECT_URL = FORCE_SCRIPT_NAME + '/query/'
LOGOUT_URL = '/logout/'
ROOT_URLCONF = 'omop_harvest.conf.urls'

# For non-publicly accessible applications, the siteauth app can be used to
# restrict access site-wide.
# SITEAUTH_ACCESS_ORDER = 'allow/deny'
#
SITEAUTH_ALLOW_URLS = (
    r'^log(in|out)/',
    r'^password/reset/',
    r'^(register|verify)/',
)

SITEAUTH_DENY_URLS = (
    r'^workspace/',
    r'^workspace/discover/',
    r'^query/',
    r'^results/+',
    r'^api/+',
    r'^details/\d+/',
    r'^moderate/+',
    r'^verify/+',
)

#
# MIDDLEWARE
#

MIDDLEWARE_CLASSES = (
    'django.middleware.gzip.GZipMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'siteauth.middleware.SiteAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'serrano.middleware.SessionMiddleware',
)

#
# EMAIL
#

SUPPORT_EMAIL = 'cbmisupport@email.chop.edu'
DEFAULT_FROM_EMAIL = 'cbmisupport@email.chop.edu'
EMAIL_SUBJECT_PREFIX = '[omop_harvest] '
SEND_BROKEN_LINK_EMAILS = False

#
# LOGGING
#

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters' :{
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        }
    },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'default': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': 'logs/omop.log',
            'maxBytes': 1024*1024*5, # 5 MB
            'backupCount': 5,
            'formatter':'standard',
        },
        'request_handler': {
                'level':'DEBUG',
                'class':'logging.handlers.RotatingFileHandler',
                'filename': 'logs/omop_request.log',
                'maxBytes': 1024*1024*5, # 5 MB
                'backupCount': 5,
                'formatter':'standard',
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console':{
            'level': 'DEBUG',
            'class':'logging.StreamHandler',
        }
    },
    'loggers': {
        '': {
            'handlers': ['default'],
            'level': 'DEBUG',
            'propagate': True
        },
        'django.request': {
            'handlers': ['request_handler'],
            'level': 'DEBUG',
            'propagate': False
        },
        'avocado':{
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': True,
        },
        'serrano':{
            'handlers':['console'],
            'level': 'DEBUG',
            'propagate':True,
        },
    }
}

#
# CACHE
#

# For production environments, the memcached backend is highly recommended
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'unique',
        'KEY_PREFIX': 'omop_harvest',
        'VERSION': 1,
    }
}

CACHE_MIDDLEWARE_SECONDS = 0

# This is not necessary to set if the above `KEY_PREFIX` value is set since
# the `KEY_PREFIX` namespaces all cache set by this application
CACHE_MIDDLEWARE_KEY_PREFIX = 'omop_harvest'

#
# SESSIONS AND COOKIES
#

CSRF_COOKIE_NAME = 'omop_harvest_csrftoken'

# SESSION_COOKIE_AGE = 60 * 20
SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
SESSION_COOKIE_NAME = 'omop_harvest_sessionid'
SESSION_EXPIRE_AT_BROWSER_CLOSE = False
SESSION_SAVE_EVERY_REQUEST = False

#
# OTHER PROJECT SETTINGS
#

# USE_ETAGS = True
IGNORABLE_404_PATHS = (
    r'robots.txt$',
    r'favicon.ico$',
)

#
# VARIOUS APP SETTINGS
#

# The primary key of the ``Site`` object for the Sites Framework
SITE_ID = 1

#
# ModelTrees Configuration
#

MODELTREES = {
    'default': {
        'model': 'omop_harvest.Person',
    }
}

#
# Haystack Configuration
#
HAYSTACK_SITECONF = 'avocado.search_sites'

HAYSTACK_SEARCH_ENGINE = 'whoosh'

HAYSTACK_WHOOSH_PATH = os.path.join(os.path.dirname(__file__), 'whoosh.index')

AVOCADO = {
    'METADATA_MIGRATION_APP': 'omop_harvest',
}
<|endoftext|>


<commit_before>def elastic_mapping(resource_type, change_type):
    mapping = {
        "mappings": {
            resource_type: {
                "properties": {
                    "resync_id":{
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "location": {
                        "type": "nested",
                        "properties": {
                            "value": {
                                "type": "string",
                                "index": "not_analyzed"
                            },
                            "type": {
                                "type": "string",
                                "index": "not_analyzed"
                            }
                        }
                    },
                    "length": {
                        "type": "integer",
                        "index": "not_analyzed"
                    },
                    "md5": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "mime": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "lastmod": {
                        "type": "date",
                        "format": "yyyyMMdd'T'HHmmssZ"
                    },
                    "resource_set": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "ln": {
                        "type": "nested",
                        "index_name": "link",
                        "properties": {
                            "rel": {
                                "type": "string",
                                "index": "not_analyzed"
                            },
                            "href": {
                                "type": "nested",
                                "properties": {
                                    "value": {
                                        "type": "string",
                                        "index": "not_analyzed"
                                    },
                                    "type": {
                                        "type": "string",
                                        "index": "not_analyzed"
                                    }
                                }
                            },
                            "mime": {
                                "type": "string",
                                "index": "not_analyzed"
                            }
                        }
                    }
                },
                "timestamp": {
                    "type": "date",
                    "format": "yyyyMMdd'T'HHmmssZ"
                }
            },
            change_type: {
                "properties": {
                    "location": {
                        "type": "nested",
                        "properties": {
                            "value": {
                                "type": "string",
                                "index": "not_analyzed"
                            },
                            "type": {
                                "type": "string",
                                "index": "not_analyzed"
                            }
                        }
                    },
                    "lastmod": {
                        "type": "date",
                        "format": "yyyyMMdd'T'HHmmssZ"
                    },
                    "change": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "resource_set": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "timestamp": {
                        "type": "date",
                        "format": "yyyyMMdd'T'HHmmssZ"
                    },
                }
            }
        }
    }
    return mapping
<commit_msg>Fix date format ES mapping<commit_after>def elastic_mapping(resource_type, change_type):
    mapping = {
        "mappings": {
            resource_type: {
                "properties": {
                    "resync_id":{
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "location": {
                        "type": "nested",
                        "properties": {
                            "value": {
                                "type": "string",
                                "index": "not_analyzed"
                            },
                            "type": {
                                "type": "string",
                                "index": "not_analyzed"
                            }
                        }
                    },
                    "length": {
                        "type": "integer",
                        "index": "not_analyzed"
                    },
                    "md5": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "mime": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "lastmod": {
                        "type": "date",
                        "format": "yyyy-MM-dd\'T\'HH:mm:ssZ"
                    },
                    "resource_set": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "ln": {
                        "type": "nested",
                        "index_name": "link",
                        "properties": {
                            "rel": {
                                "type": "string",
                                "index": "not_analyzed"
                            },
                            "href": {
                                "type": "nested",
                                "properties": {
                                    "value": {
                                        "type": "string",
                                        "index": "not_analyzed"
                                    },
                                    "type": {
                                        "type": "string",
                                        "index": "not_analyzed"
                                    }
                                }
                            },
                            "mime": {
                                "type": "string",
                                "index": "not_analyzed"
                            }
                        }
                    },
                    "timestamp": {
                        "type": "date",
                        "format": "yyyy-MM-dd\'T\'HH:mm:ssZ"
                    }
                }
            },
            change_type: {
                "properties": {
                    "location": {
                        "type": "nested",
                        "properties": {
                            "value": {
                                "type": "string",
                                "index": "not_analyzed"
                            },
                            "type": {
                                "type": "string",
                                "index": "not_analyzed"
                            }
                        }
                    },
                    "lastmod": {
                        "type": "date",
                        "format": "yyyy-MM-dd\'T\'HH:mm:ssZ"
                    },
                    "change": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "resource_set": {
                        "type": "string",
                        "index": "not_analyzed"
                    },
                    "datetime": {
                        "type": "date",
                        "format": "yyyy-MM-dd\'T\'HH:mm:ssZ"
                    },
                    "timestamp": {
                        "type": "date",
                        "format": "yyyy-MM-dd\'T\'HH:mm:ssZ"
                    }
                }
            }
        }
    }
    return mapping
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import collections
import re
import requests
import sys
import time
import yaml

# fun hack to order yaml by key
# @todo - master plan is to move from yaml to mongodb
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_representer(dumper, data):
    return dumper.represent_dict(data.iteritems())
def dict_constructor(loader, node):
    return collections.OrderedDict(loader.construct_pairs(node))
yaml.add_representer(collections.OrderedDict, dict_representer)
yaml.add_constructor(_mapping_tag, dict_constructor)

# get our data
with open('data.yml', 'r') as stream:
    try:
        data = yaml.load(stream)
    except yaml.YAMLError as exception:
        print 'There was a problem loading the yml file...'
        print(exception)

# find regulatory code changes
print '\nLOOKING FOR CODE CHANGES'
for indication in data:

    # what indication?
    print '\n' + indication

    for website in data[indication]:

        # what website?
        print website

        # try and get the html
        trys = 0
        while True:
            try:

                # make the request
                url = 'http://' + website
                headers = {'user-agent': 'Moai'}
                html_content = requests.get(url, headers=headers, timeout=3).text

                # search for the code using the regex defined per website
                live_matches = re.findall(data[indication][website]['regex'], re.sub('<[^<]+?>', '', html_content));

                # get the most recent date with a code
                for date in reversed(data[indication][website]['dates']):
                    if data[indication][website]['dates'][date].has_key('code'):
                        most_recent_date = date
                        break
                    else:
                        most_recent_date =''
                # get the most recent data code
                if data[indication][website]['dates'][most_recent_date].has_key('code'):
                    most_recent_date_code = data[indication][website]['dates'][most_recent_date]['code']
                else:
                    most_recent_date_code = ''

                # if we find a match
                if len(live_matches) > 0:
                    if str(most_recent_date_code) == str(live_matches[0]):
                        print '\t- The most recent code that we have [' + str(most_recent_date) + '][' + str(most_recent_date_code) + '] matches what we just found [' + str(live_matches[0]) + ']'
                    else:
                        print '\t- Found a different code than the one we have, writing [' + str(live_matches[0]) + '] to data.yml'
                        data[indication][website]['dates'][int(time.strftime("%Y%m%d"))] = { 'code' : str(live_matches[0]) }
                        with open('data.yml', 'w') as outfile:
                            yaml.dump(data, outfile, default_flow_style=False)

                # if we dont find a match
                else:
                    print '\t- Did not find any live matches, please check that the regex is current.'

                break

            # catch any exceptions
            except requests.exceptions.RequestException as e:
                print('\t- Exception: ' + str(e))
            finally:
                trys = trys + 1
                time.sleep(3)
                if trys == 2:
                    print '\t- Tried getting the content ' + str(trys) + ' times, skipping...'
                    break

# determine if there is a 443 listener
print '\nLOOKING FOR HTTPS SUPPORT'
for indication in data:

    # what indication?
    print '\n' + indication

    for website in data[indication]:

        # what website?
        print website

        # try and get the html
        trys = 0
        https = False
        while True:
            try:

                # make the request
                url = 'https://' + website
                headers = {'user-agent': 'Moai'}
                requests.get(url, headers=headers, timeout=3)
                https = True
                break

            # catch any exceptions
            except requests.exceptions.RequestException as e:
                print('\t- Exception: ' + str(e))
                https = False
            finally:
                trys = trys + 1
                time.sleep(3)
                if trys == 2:
                    print '\t- Tried validating HTTPS support ' + str(trys) + ' times, skipping...'
                    break

        # get the most recent date
        most_recent_date = data[indication][website]['dates'].keys()[-1]
        # get the most recent https status
        if data[indication][website]['dates'][most_recent_date].has_key('https'):
            most_recent_date_https = data[indication][website]['dates'][most_recent_date]['https']
        else:
            most_recent_date_https = ''

        if str(most_recent_date_https) == str(https):
            print '\t- The most recent https status that we have [' + str(most_recent_date) + '][' + str(most_recent_date_https) + '] matches what we just found [' + str(https) + ']'
        else:
            print '\t- Found a different https status than the one we have, writing [' + str(https) + '] to data.yml'
            if data[indication][website]['dates'].has_key(int(time.strftime("%Y%m%d"))) and data[indication][website]['dates'][int(time.strftime("%Y%m%d"))].has_key('code'):
                data[indication][website]['dates'][int(time.strftime("%Y%m%d"))] = { 'code' : data[indication][website]['dates'][int(time.strftime("%Y%m%d"))]['code'], 'https' : str(https) }
            else:
                data[indication][website]['dates'][int(time.strftime("%Y%m%d"))] = { 'https' : str(https) }
            with open('data.yml', 'w') as outfile:
                yaml.dump(data, outfile, default_flow_style=False)

# generate images and README content
print '\nGENERATING IMAGES'
import matplotlib
# force matplotlib to not use any Xwindows backend
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pandas as pd
# set global styles
matplotlib.rcParams.update({'font.size': 6})

content = ''

for indication in data:

    content += '\n'
    content += '\n### ' + str(indication)
    content += '\n| Drug | Company | Generic | HTTPS | Update frequency |'
    content += '\n| ---- | ------- | ------- | ----- | ---------------- |'

    for website in data[indication]:

        # what website?
        print website

        dates = []

        for date in data[indication][website]['dates']:
            if data[indication][website]['dates'][date].has_key('code'):
                dates.append(str(date))

        X = pd.to_datetime(dates)
        print X
        fig, ax = plt.subplots(figsize=(6,0.4))
        ax.scatter(X, [1]*len(X), marker='v', s=50, color='#306caa')
        fig.autofmt_xdate()

        # turn off unncessary items
        ax.yaxis.set_visible(False)
        ax.spines['right'].set_visible(False)
        ax.spines['left'].set_visible(False)
        ax.spines['top'].set_visible(False)
        ax.xaxis.set_ticks_position('bottom')

        ax.get_yaxis().set_ticklabels([])
        day = pd.to_timedelta("1", unit='D')
        plt.xlim(X[0] - day, X[-1] + day)
        plt.savefig('data/' + website.replace("/","-") + '.png', bbox_inches='tight')

        plt.close('all')

        # get the most recent date with https status
        for date in reversed(data[indication][website]['dates']):
            if data[indication][website]['dates'][date].has_key('https'):
                https = data[indication][website]['dates'][date]['https']
                if https == 'True':
                    https = ':white_check_mark:'
                else:
                    https = ':x:'
                break

        content += '\n| [{0}](http://{0}) | {1} | {2} | [{3}](https://{0}) | ![{4}](data/{4}.png) |'.format( website , data[indication][website]['drug']['company'] , data[indication][website]['drug']['generic'] , https , website.replace("/","-") )

# generate README.md
f = open('README.md', 'w')
f.write('''
# moai
:moyai: Pharmaceutical competitive intelligence through product website FDA OPDP update frequency.

![Moai](moai.jpg)

Moai /ˈmoʊ.aɪ/ provides competitive intelligence by tracking the unique regulatory code on United States pharmaceutical websites that are mandated by the FDA. This provides insight as to when, and how often, a website is updated.

HTTPS is also tracked. Sadly, many website infrastructures do not provide HTTPS and subsequently [provides no data security](https://www.chromium.org/Home/chromium-security/marking-http-as-non-secure) to its vistors. Here's a shameless plug for our website and workflow management platorm [Catapult](https://github.com/devopsgroup-io/catapult), which enforces best practice security.

| ![Charles](moai-charles.jpg) | Meet Charles, the moaiBOT. He scours websites daily, looking for changes.
Charles likes fishing and long walks on the beach. |
| -- | -- |

The below data is free, looking for a complete picture with valuable insights? Please contact us at info@devopsgroup.io to learn more.
{0}
'''.format(content))
f.close()
<commit_msg>typos<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import collections
import re
import requests
import sys
import time
import yaml

# fun hack to order yaml by key
# @todo - master plan is to move from yaml to mongodb
_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
def dict_representer(dumper, data):
    return dumper.represent_dict(data.iteritems())
def dict_constructor(loader, node):
    return collections.OrderedDict(loader.construct_pairs(node))
yaml.add_representer(collections.OrderedDict, dict_representer)
yaml.add_constructor(_mapping_tag, dict_constructor)

# get our data
with open('data.yml', 'r') as stream:
    try:
        data = yaml.load(stream)
    except yaml.YAMLError as exception:
        print 'There was a problem loading the yml file...'
        print(exception)

# find regulatory code changes
print '\nLOOKING FOR CODE CHANGES'
for indication in data:

    # what indication?
    print '\n' + indication

    for website in data[indication]:

        # what website?
        print website

        # try and get the html
        trys = 0
        while True:
            try:

                # make the request
                url = 'http://' + website
                headers = {'user-agent': 'Moai'}
                html_content = requests.get(url, headers=headers, timeout=3).text

                # search for the code using the regex defined per website
                live_matches = re.findall(data[indication][website]['regex'], re.sub('<[^<]+?>', '', html_content));

                # get the most recent date with a code
                for date in reversed(data[indication][website]['dates']):
                    if data[indication][website]['dates'][date].has_key('code'):
                        most_recent_date = date
                        break
                    else:
                        most_recent_date =''
                # get the most recent data code
                if data[indication][website]['dates'][most_recent_date].has_key('code'):
                    most_recent_date_code = data[indication][website]['dates'][most_recent_date]['code']
                else:
                    most_recent_date_code = ''

                # if we find a match
                if len(live_matches) > 0:
                    if str(most_recent_date_code) == str(live_matches[0]):
                        print '\t- The most recent code that we have [' + str(most_recent_date) + '][' + str(most_recent_date_code) + '] matches what we just found [' + str(live_matches[0]) + ']'
                    else:
                        print '\t- Found a different code than the one we have, writing [' + str(live_matches[0]) + '] to data.yml'
                        data[indication][website]['dates'][int(time.strftime("%Y%m%d"))] = { 'code' : str(live_matches[0]) }
                        with open('data.yml', 'w') as outfile:
                            yaml.dump(data, outfile, default_flow_style=False)

                # if we dont find a match
                else:
                    print '\t- Did not find any live matches, please check that the regex is current.'

                break

            # catch any exceptions
            except requests.exceptions.RequestException as e:
                print('\t- Exception: ' + str(e))
            finally:
                trys = trys + 1
                time.sleep(3)
                if trys == 2:
                    print '\t- Tried getting the content ' + str(trys) + ' times, skipping...'
                    break

# determine if there is a 443 listener
print '\nLOOKING FOR HTTPS SUPPORT'
for indication in data:

    # what indication?
    print '\n' + indication

    for website in data[indication]:

        # what website?
        print website

        # try and get the html
        trys = 0
        https = False
        while True:
            try:

                # make the request
                url = 'https://' + website
                headers = {'user-agent': 'Moai'}
                requests.get(url, headers=headers, timeout=3)
                https = True
                break

            # catch any exceptions
            except requests.exceptions.RequestException as e:
                print('\t- Exception: ' + str(e))
                https = False
            finally:
                trys = trys + 1
                time.sleep(3)
                if trys == 2:
                    print '\t- Tried validating HTTPS support ' + str(trys) + ' times, skipping...'
                    break

        # get the most recent date
        most_recent_date = data[indication][website]['dates'].keys()[-1]
        # get the most recent https status
        if data[indication][website]['dates'][most_recent_date].has_key('https'):
            most_recent_date_https = data[indication][website]['dates'][most_recent_date]['https']
        else:
            most_recent_date_https = ''

        if str(most_recent_date_https) == str(https):
            print '\t- The most recent https status that we have [' + str(most_recent_date) + '][' + str(most_recent_date_https) + '] matches what we just found [' + str(https) + ']'
        else:
            print '\t- Found a different https status than the one we have, writing [' + str(https) + '] to data.yml'
            if data[indication][website]['dates'].has_key(int(time.strftime("%Y%m%d"))) and data[indication][website]['dates'][int(time.strftime("%Y%m%d"))].has_key('code'):
                data[indication][website]['dates'][int(time.strftime("%Y%m%d"))] = { 'code' : data[indication][website]['dates'][int(time.strftime("%Y%m%d"))]['code'], 'https' : str(https) }
            else:
                data[indication][website]['dates'][int(time.strftime("%Y%m%d"))] = { 'https' : str(https) }
            with open('data.yml', 'w') as outfile:
                yaml.dump(data, outfile, default_flow_style=False)

# generate images and README content
print '\nGENERATING IMAGES'
import matplotlib
# force matplotlib to not use any Xwindows backend
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pandas as pd
# set global styles
matplotlib.rcParams.update({'font.size': 6})

content = ''

for indication in data:

    content += '\n'
    content += '\n### ' + str(indication)
    content += '\n| Drug | Company | Generic | HTTPS | Update frequency |'
    content += '\n| ---- | ------- | ------- | ----- | ---------------- |'

    for website in data[indication]:

        # what website?
        print website

        dates = []

        for date in data[indication][website]['dates']:
            if data[indication][website]['dates'][date].has_key('code'):
                dates.append(str(date))

        X = pd.to_datetime(dates)
        print X
        fig, ax = plt.subplots(figsize=(6,0.4))
        ax.scatter(X, [1]*len(X), marker='v', s=50, color='#306caa')
        fig.autofmt_xdate()

        # turn off unncessary items
        ax.yaxis.set_visible(False)
        ax.spines['right'].set_visible(False)
        ax.spines['left'].set_visible(False)
        ax.spines['top'].set_visible(False)
        ax.xaxis.set_ticks_position('bottom')

        ax.get_yaxis().set_ticklabels([])
        day = pd.to_timedelta("1", unit='D')
        plt.xlim(X[0] - day, X[-1] + day)
        plt.savefig('data/' + website.replace("/","-") + '.png', bbox_inches='tight')

        plt.close('all')

        # get the most recent date with https status
        for date in reversed(data[indication][website]['dates']):
            if data[indication][website]['dates'][date].has_key('https'):
                https = data[indication][website]['dates'][date]['https']
                if https == 'True':
                    https = ':white_check_mark:'
                else:
                    https = ':x:'
                break

        content += '\n| [{0}](http://{0}) | {1} | {2} | [{3}](https://{0}) | ![{4}](data/{4}.png) |'.format( website , data[indication][website]['drug']['company'] , data[indication][website]['drug']['generic'] , https , website.replace("/","-") )

# generate README.md
f = open('README.md', 'w')
f.write('''
# moai
:moyai: Pharmaceutical competitive intelligence through product website FDA OPDP update frequency.

![Moai](moai.jpg)

Moai /ˈmoʊ.aɪ/ provides competitive intelligence by tracking the unique regulatory code on United States pharmaceutical websites that are mandated by the FDA. This provides insight as to when, and how often, a website is updated.

HTTPS is also tracked. Sadly, many website infrastructures do not provide HTTPS and subsequently [provides no data security](https://www.chromium.org/Home/chromium-security/marking-http-as-non-secure) to its visitors. Here's a shameless plug for our website and workflow management platform [Catapult](https://github.com/devopsgroup-io/catapult), which enforces best practice security.

| ![Charles](moai-charles.jpg) | Meet Charles, the moaiBOT. He scours websites daily, looking for changes.
Charles likes fishing and long walks on the beach. |
| -- | -- |

The below data is free, looking for a complete picture with valuable insights? Please contact us at info@devopsgroup.io to learn more.
{0}
'''.format(content))
f.close()
<|endoftext|>


<commit_before>#!/usr/bin/python3

### Author: Vinoth Manoharan, Zoho Corp
### Language : Python
### Tested in Ubuntu

import sys,json

#if any impacting changes to this plugin kindly increment the plugin version here.
PLUGIN_VERSION = "1"

#Setting this to true will alert you when there is a communication problem while posting plugin data to server
HEARTBEAT="true"

PROC_FILE = "/proc/sys/fs/file-nr"

METRIC_UNITS = {'open_files': 'units', 'total_files': 'units'}

def metricCollector():
    data = {}
    try:
        open_nr, free_nr, max = open(PROC_FILE).readline().split("\t")
        open_files = int(open_nr) - int(free_nr)
        data["open_files"] = open_files
        data["total_files"] = int(max)
    except Exception as e:
        print ("status error ",e)
        data["error"] = str(e)
    data["units"] = METRIC_UNITS
    return data

if __name__ == "__main__":
    result = metricCollector()

    print(json.dumps(result, indent=4, sort_keys=True))<commit_msg>default python path changed to /usr/bin/python<commit_after>#!/usr/bin/python

### Author: Vinoth Manoharan, Zoho Corp
### Language : Python
### Tested in Ubuntu

import sys,json

#if any impacting changes to this plugin kindly increment the plugin version here.
PLUGIN_VERSION = "1"

#Setting this to true will alert you when there is a communication problem while posting plugin data to server
HEARTBEAT="true"

PROC_FILE = "/proc/sys/fs/file-nr"

METRIC_UNITS = {'open_files': 'units', 'total_files': 'units'}

def metricCollector():
    data = {}
    try:
        open_nr, free_nr, max = open(PROC_FILE).readline().split("\t")
        open_files = int(open_nr) - int(free_nr)
        data["open_files"] = open_files
        data["total_files"] = int(max)
    except Exception as e:
        print ("status error ",e)
        data["error"] = str(e)
    data["units"] = METRIC_UNITS
    return data

if __name__ == "__main__":
    result = metricCollector()

    print(json.dumps(result, indent=4, sort_keys=True))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import datetime, json, logging, os, pprint
from .models import Accession, CatEdit
from django.conf import settings as project_settings
from django.contrib.auth import logout
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render

log = logging.getLogger(__name__)

def hi( request ):
    """ Returns simplest response. """
    now = datetime.datetime.now()
    return HttpResponse( '
hi

( %s )
' % now )

def index( request ):
    """ Returns simplest response. """
    now = datetime.datetime.now()
    return HttpResponse( '
index

( %s )
' % now )

# @bul_login
def index( request, response_format=None ):
    # import time
    # from datetime import date
    # from django.db import connection, transaction
    # from django.template import loader, RequestContext
    # context = RequestContext( request )
    context = {}
    params = request.GET
    if params.has_key( 'format' ):
        requested_format = params['format']
    else:
        requested_format = None
    #Accessions
    #Months to reports
    acc_months = Accession.objects.dates('created', 'month', order='DESC')
    #Years to report
    acc_years = Accession.objects.dates('created', 'year', order='DESC')
    context['acc_months'] = acc_months
    context['acc_years'] = acc_years

    #Cataloging
    cat_months = CatEdit.objects.dates('edit_date', 'month', order='DESC')
    #Years to report
    cat_years = CatEdit.objects.dates('edit_date', 'year', order='DESC')
    context['cat_months'] = cat_months
    context['cat_years'] = cat_years

    # context['settings_app'] = settings_app
    resp = render( request, u'tech_services_reports_templates/index.html', context )
    # return render_to_response('index.html', context)
<commit_msg>returns resp.<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import datetime, json, logging, os, pprint
from .models import Accession, CatEdit
from django.conf import settings as project_settings
from django.contrib.auth import logout
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render

log = logging.getLogger(__name__)

def hi( request ):
    """ Returns simplest response. """
    now = datetime.datetime.now()
    return HttpResponse( '
hi

( %s )
' % now )

def index( request ):
    """ Returns simplest response. """
    now = datetime.datetime.now()
    return HttpResponse( '
index

( %s )
' % now )

# @bul_login
def index( request, response_format=None ):
    log.debug( 'starting index()' )
    # import time
    # from datetime import date
    # from django.db import connection, transaction
    # from django.template import loader, RequestContext
    # context = RequestContext( request )
    context = {}
    params = request.GET
    if params.has_key( 'format' ):
        requested_format = params['format']
    else:
        requested_format = None
    #Accessions
    #Months to reports
    acc_months = Accession.objects.dates('created', 'month', order='DESC')
    #Years to report
    acc_years = Accession.objects.dates('created', 'year', order='DESC')
    context['acc_months'] = acc_months
    context['acc_years'] = acc_years

    #Cataloging
    cat_months = CatEdit.objects.dates('edit_date', 'month', order='DESC')
    #Years to report
    cat_years = CatEdit.objects.dates('edit_date', 'year', order='DESC')
    context['cat_months'] = cat_months
    context['cat_years'] = cat_years

    # context['settings_app'] = settings_app
    log.debug( 'context, ```{}```'.format(pprint.pformat(context)) )
    resp = render( request, u'tech_services_reports_templates/index.html', context )
    # return render_to_response('index.html', context)
    return resp
<|endoftext|>


<commit_before>from openbudget.settings.base import *

######################################################################
##                                                                  ##
## RENAME THIS FILE TO local.py to provide your own local overrides ##
##                                                                  ##
######################################################################

#EMAIL_HOST_USER = ''

#EMAIL_HOST_PASSWORD = ''

#ADMINS = (
#    ('', ''),
#)

# Uncomment MIDDLEWARE_CLASSES, INSTALLED_APPS, INTERNAL_IPS, and
# DEBUG_TOOLBAR_CONFIG for debugging tools.
#
#MIDDLEWARE_CLASSES += (
#    'debug_toolbar.middleware.DebugToolbarMiddleware',
#    'django_pdb.middleware.PdbMiddleware',
#)
#
#INSTALLED_APPS += (
#    'debug_toolbar',
#    'django_pdb',
#)
#
#INTERNAL_IPS = ('127.0.0.1',)
#
#DEBUG_TOOLBAR_CONFIG = {
#    'INTERCEPT_REDIRECTS': False,
#}

# Uncomment to use Postgresql in development.
# You'll also need some extra dependencies:
# pip install -r requirements/deploy.txt
#DATABASES = {
#    'default': {
#        'ENGINE': 'django.db.backends.postgresql_psycopg2',
#        'NAME': '',
#        'USER': '',
#        'PASSWORD': '',
#        'HOST': '',
#        'PORT': '',
#    }
#}

# Uncomment to enable caching during development
#CACHES = {
#    'default': {
#        'BACKEND': 'redis_cache.RedisCache',
#        'LOCATION': REDIS['HOST'] + ':' + str(REDIS['PORT']),
#        'OPTIONS': {
#            'DB': REDIS['DB'],
#            'PARSER_CLASS': 'redis.connection.HiredisParser'
#        },
#    },
#}
<commit_msg>Some small updates to settings.example_local<commit_after># Uncomment if you are overriding any settings that are defined in settings.base
#from openbudget.settings.base import *

######################################################################
##                                                                  ##
## RENAME THIS FILE TO local.py to provide your own local overrides ##
##                                                                  ##
######################################################################

#EMAIL_HOST_USER = ''

#EMAIL_HOST_PASSWORD = ''

#ADMINS = (
#    ('', ''),
#)

# Uncomment MIDDLEWARE_CLASSES, INSTALLED_APPS, INTERNAL_IPS, and
# DEBUG_TOOLBAR_CONFIG for debugging tools.
#
#MIDDLEWARE_CLASSES += (
#    'debug_toolbar.middleware.DebugToolbarMiddleware',
#    'django_pdb.middleware.PdbMiddleware',
#)
#
#INSTALLED_APPS += (
#    'debug_toolbar',
#    'django_pdb',
#)
#
#INTERNAL_IPS = ('127.0.0.1',)
#
#DEBUG_TOOLBAR_CONFIG = {
#    'INTERCEPT_REDIRECTS': False,
#}

# Uncomment to use Postgresql in development.
# You'll also need some extra dependencies:
# pip install -r requirements/deploy.txt
#DATABASES = {
#    'default': {
#        'ENGINE': 'django.db.backends.postgresql_psycopg2',
#        'NAME': '',
#        'USER': '',
#        'PASSWORD': '',
#        'HOST': '',
#        'PORT': '',
#    }
#}

# Uncomment to enable caching during development
#CACHES = {
#    'default': {
#        'BACKEND': 'redis_cache.RedisCache',
#        'LOCATION': REDIS['HOST'] + ':' + str(REDIS['PORT']),
#        'OPTIONS': {
#            'DB': REDIS['DB'],
#            'PARSER_CLASS': 'redis.connection.HiredisParser'
#        },
#    },
#}

# Uncomment to enable redis as a celery backend in development
# You'll also need some extra dependencies:
# pip install -r requirements/deploy.txt
#BROKER_URL = REDIS_URL
#
#CELERY_RESULT_BACKEND = BROKER_URL
#
#CELERY_RESULT_DBURI = ''
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import subprocess
import os
import pdb
import filecmp
from sys import platform as _platform
import tempfile
import time
import argparse
import getpass

# Currently, destination manifest will be overwritten. user input should be required. source manifest will not be overwritten, it will be read.
parser = argparse.ArgumentParser(description='Copy directory with checksum comparison and manifest generation.'
                                 ' Written by Kieran O\'Leary.')
parser.add_argument('source', help='Input directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('-b', '-benchmark', action='store_true', help='display benchmark')

args = parser.parse_args()

source               = args.source
source_parent_dir    = os.path.dirname(source)
normpath             = os.path.normpath(source)
dirname              = os.path.split(os.path.basename(source))[1]
relative_path        = normpath.split(os.sep)[-1]

destination                    = args.destination # or hardcode
manifest_destination           = destination + '/%s_manifest.md5' % dirname
destination_final_path         = destination + '/%s' % dirname
manifest                       = source_parent_dir + '/%s_manifest.md5' % relative_path
log_name_source                = source_parent_dir + '/%s_ifi_events_log.log' % relative_path
log_name_destination           = destination + '/%s_ifi_events_log.log' % dirname

def generate_log(log, what2log):
    if not os.path.isfile(log):
        with open(log,"wb") as fo:
            fo.write(time.strftime("%Y-%m-%dT%H:%M:%S ") + getpass.getuser() + ' ' + what2log + ' \n')
    else:
        with open(log,"ab") as fo:
            fo.write(time.strftime("%Y-%m-%dT%H:%M:%S ") + getpass.getuser() + ' ' + what2log + ' \n')

generate_log(log_name_source, 'move.py started.')
generate_log(log_name_source, 'Source: %s' % source)
generate_log(log_name_source, 'Destination: %s'  % destination)
def display_benchmark():
    print 'SOURCE MANIFEST TIME', source_manifest_time
    print 'COPY TIME', copy_time
    print 'DESTINATION MANIFEST TIME', destination_manifest_time

def test_write_capabilities(directory):
    if os.path.isdir(directory):
        temp = tempfile.mkstemp(dir=directory, suffix='.tmp')
        os.close(temp[0]) # Needed for windows.
        os.remove(temp[1])
    elif os.path.isfile(directory):
        print '\nFile transfer is not currently supported, only directories.\n'
        generate_log(log_name_source, 'Error: Attempted file transfer. Source and Destination must be a directory')
        generate_log(log_name_source, 'move.py exit')
        sys.exit()
    else:
        print ' %s is either not a directory or it does not exist' % directory
        generate_log(log_name_source, ' %s is either not a directory or it does not exist' % directory)
        generate_log(log_name_source, 'move.py exit')
        sys.exit()

def remove_bad_files(root_dir):
    rm_these = ['.DS_Store', 'Thumbs.db', 'desktop.ini']
    for root, dirs, files in os.walk(root_dir):
        for name in files:
            path = os.path.join(root, name)
            for i in rm_these:
                if name == i:
                    print '***********************' + 'removing: ' + path
                    generate_log(log_name_source, 'EVENT = Unwanted file removal - %s was removed' % path)
                    os.remove(path)

def make_manifest(manifest_dir, relative_manifest_path, manifest_textfile):
    os.chdir(manifest_dir)
    if os.path.isfile(manifest_destination):
        print 'Destination manifest already exists'
    manifest_generator = subprocess.check_output(['md5deep', '-ler', relative_manifest_path])
    manifest_list = manifest_generator.splitlines()
    files_in_manifest = len(manifest_list)
    # http://stackoverflow.com/a/31306961/2188572
    manifest_list = sorted(manifest_list,  key=lambda x:(x[34:]))
    with open(manifest_textfile,"wb") as fo:
        for i in manifest_list:
            fo.write(i + '\n')
    return files_in_manifest

def copy_dir():
    if _platform == "win32":
        subprocess.call(['robocopy',source, destination_final_path, '/E'])
        generate_log(log_name_source, 'EVENT = File Transfer - Windows O.S - Software=Robocopy')
        print destination_final_path
    elif _platform == "darwin":
        # https://github.com/amiaopensource/ltopers/blob/master/writelto#L51
        cmd = ['gcp','--preserve=mode,timestamps', '-nRv',source, destination_final_path]
        generate_log(log_name_source, 'EVENT = File Transfer - OSX - Software=gcp')

        subprocess.call(cmd)
    elif _platform == "linux2":
        # https://github.com/amiaopensource/ltopers/blob/master/writelto#L51
        cmd = [ 'cp','--preserve=mode,timestamps', '-nRv',source, destination_final_path]
        generate_log(log_name_source, 'EVENT = File Transfer - Linux- Software=cp')
        subprocess.call(cmd)

def check_overwrite(file2check):
    if os.path.isfile(file2check):
        print 'A manifest already exists at your destination. Overwrite? Y/N?'
        overwrite_destination_manifest = ''
        while overwrite_destination_manifest not in ('Y','y','N','n'):
            overwrite_destination_manifest = raw_input()
            if overwrite_destination_manifest not in ('Y','y','N','n'):
                print 'Incorrect input. Please enter Y or N'
        return overwrite_destination_manifest
def manifest_file_count(manifest2check):
    if os.path.isfile(manifest2check):
        print 'A manifest already exists'
        with open(manifest2check, "r") as fo:
            manifest_lines = [line.split(',') for line in fo.readlines()]
            count_in_manifest =  len(manifest_lines)
    return count_in_manifest

def check_overwrite_dir(dir2check):
    if os.path.isdir(dir2check):
        print 'A directory already exists at your destination. Overwrite? Y/N?'
        overwrite_destination_dir = ''
        while overwrite_destination_dir not in ('Y','y','N','n'):
            overwrite_destination_dir = raw_input()
            if overwrite_destination_dir not in ('Y','y','N','n'):
                print 'Incorrect input. Please enter Y or N'
        return overwrite_destination_dir

try:
    test_write_capabilities(source_parent_dir)
except OSError:
            print 'You cannot write to your source directory!'
            generate_log(log_name_source, 'EVENT = I/O Test - Failure - No write access to source directory.')
            sys.exit()

try:
    test_write_capabilities(destination)
except OSError:
            print 'You cannot write to your destination!'
            generate_log(log_name_source, 'EVENT = I/O Test - Failure - No write access to destination directory.')
            sys.exit()
overwrite_destination_manifest = check_overwrite(manifest_destination)
overwrite_destination_dir = check_overwrite_dir(destination_final_path)

remove_bad_files(source)
source_count = 0

for root, directories, filenames in os.walk(source):
    for files in filenames:
            source_count +=1 #works in windows at least
print source_count

if os.path.isfile(manifest):
    count_in_manifest = manifest_file_count(manifest)
    if source_count != count_in_manifest:
        print 'This manifest may be outdated as the number of files in your directory does not match the number of files in the manifest'
        generate_log(log_name_source, 'EVENT = Existing source manifest check - Failure - The number of files in the source directory is not equal to the number of files in the source manifest ')
        sys.exit()
source_manifest_start_time = time.time()

if not os.path.isfile(manifest):
    try:
        print 'Generating source manifest'
        make_manifest(source_parent_dir, relative_path,manifest)
        generate_log(log_name_source, 'EVENT = Generating source manifest')

    except OSError:
            print 'You do not have access to this directory. Perhaps it is read only, or the wrong file system\n'
            sys.exit()

source_manifest_time = time.time() - source_manifest_start_time

copy_start_time = time.time()
if overwrite_destination_dir not in ('N','n'):
    if overwrite_destination_dir != None:
        generate_log(log_name_source, 'EVENT = File Transfer Overwrite - Destination directory already exists - Overwriting.')
    copy_dir()
else:
    generate_log(log_name_source, 'EVENT = File Transfer Overwrite - Destination directory already exists - Not Overwriting.')

copy_time = time.time() - copy_start_time

start_destination_manifest_time = time.time()
if overwrite_destination_manifest not in ('N','n'):
    if overwrite_destination_manifest == None:
        generate_log(log_name_source, 'EVENT = Destination Manifest Generation')
    else:
        generate_log(log_name_source, 'EVENT = Destination Manifest Overwrite - Destination manifest already exists - Overwriting.')
    print 'Generating destination manifest'
    files_in_manifest = make_manifest(destination,dirname, manifest_destination)
else:
    generate_log(log_name_source, 'EVENT = File Transfer Overwrite - Destination directory already exists - Not Overwriting.')
remove_bad_files(destination_final_path)

destination_manifest_time = time.time() - start_destination_manifest_time
destination_count = 0

for root, directories, filenames in os.walk(destination_final_path):
    for files in filenames:
            destination_count +=1 #works in windows at least
print destination_count

if filecmp.cmp(manifest, manifest_destination, shallow=False):
    print "Your files have reached their destination and the checksums match"
    generate_log(log_name_source, 'EVENT = File Transfer Judgement - Success')
else:
    print "***********YOUR CHECKSUMS DO NOT MATCH*************"
    if overwrite_destination_manifest not in ('N','n'):
        generate_log(log_name_source, 'EVENT = File Transfer Outcome - Failure')
        print ' There are: \n %s files in your destination manifest \n' % files_in_manifest
        print ' %s files in your destination \n %s files at source' % (destination_count, source_count)
        generate_log(log_name_source, 'EVENT = File Transfer Failure Explanation -  %s files in your destination,  %s files at source' % (destination_count, source_count))
    else:
        print ' %s files in your destination \n %s files at source' % (destination_count, source_count)
if args.b:
    display_benchmark()
<commit_msg>move.py - source manifest/log is now saved to desktop<commit_after>#!/usr/bin/env python

import sys
import subprocess
import os
import pdb
import filecmp
from sys import platform as _platform
import tempfile
import time
import argparse
import getpass

# Currently, destination manifest will be overwritten. user input should be required. source manifest will not be overwritten, it will be read.
parser = argparse.ArgumentParser(description='Copy directory with checksum comparison and manifest generation.'
                                 ' Written by Kieran O\'Leary.')
parser.add_argument('source', help='Input directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('-b', '-benchmark', action='store_true', help='display benchmark')

args = parser.parse_args()

source               = args.source
source_parent_dir    = os.path.dirname(source)
normpath             = os.path.normpath(source)
dirname              = os.path.split(os.path.basename(source))[1]
relative_path        = normpath.split(os.sep)[-1]

destination                    = args.destination # or hardcode
manifest_destination           = destination + '/%s_manifest.md5' % dirname
destination_final_path         = destination + '/%s' % dirname
manifest_ = os.path.basename(args.source) + '_source_manifest' + time.strftime("_%Y_%m_%dT%H_%M_%S")
manifest = os.path.expanduser("~/Desktop/%s.md5") % manifest_
log_name_source_                = os.path.basename(args.source) + time.strftime("_%Y_%m_%dT%H_%M_%S")
log_name_source = os.path.expanduser("~/Desktop/%s.log") % log_name_source_
log_name_destination           = destination + '/%s_ifi_events_log.log' % dirname

def generate_log(log, what2log):
    if not os.path.isfile(log):
        with open(log,"wb") as fo:
            fo.write(time.strftime("%Y-%m-%dT%H:%M:%S ") + getpass.getuser() + ' ' + what2log + ' \n')
    else:
        with open(log,"ab") as fo:
            fo.write(time.strftime("%Y-%m-%dT%H:%M:%S ") + getpass.getuser() + ' ' + what2log + ' \n')

generate_log(log_name_source, 'move.py started.')
generate_log(log_name_source, 'Source: %s' % source)
generate_log(log_name_source, 'Destination: %s'  % destination)
def display_benchmark():
    print 'SOURCE MANIFEST TIME', source_manifest_time
    print 'COPY TIME', copy_time
    print 'DESTINATION MANIFEST TIME', destination_manifest_time

def test_write_capabilities(directory):
    if os.path.isdir(directory):
        temp = tempfile.mkstemp(dir=directory, suffix='.tmp')
        os.close(temp[0]) # Needed for windows.
        os.remove(temp[1])
    elif os.path.isfile(directory):
        print '\nFile transfer is not currently supported, only directories.\n'
        generate_log(log_name_source, 'Error: Attempted file transfer. Source and Destination must be a directory')
        generate_log(log_name_source, 'move.py exit')
        sys.exit()
    else:
        print ' %s is either not a directory or it does not exist' % directory
        generate_log(log_name_source, ' %s is either not a directory or it does not exist' % directory)
        generate_log(log_name_source, 'move.py exit')
        sys.exit()

def remove_bad_files(root_dir):
    rm_these = ['.DS_Store', 'Thumbs.db', 'desktop.ini']
    for root, dirs, files in os.walk(root_dir):
        for name in files:
            path = os.path.join(root, name)
            for i in rm_these:
                if name == i:
                    print '***********************' + 'removing: ' + path
                    generate_log(log_name_source, 'EVENT = Unwanted file removal - %s was removed' % path)
                    os.remove(path)

def make_manifest(manifest_dir, relative_manifest_path, manifest_textfile):
    os.chdir(manifest_dir)
    if os.path.isfile(manifest_destination):
        print 'Destination manifest already exists'
    manifest_generator = subprocess.check_output(['md5deep', '-ler', relative_manifest_path])
    manifest_list = manifest_generator.splitlines()
    files_in_manifest = len(manifest_list)
    # http://stackoverflow.com/a/31306961/2188572
    manifest_list = sorted(manifest_list,  key=lambda x:(x[34:]))
    with open(manifest_textfile,"wb") as fo:
        for i in manifest_list:
            fo.write(i + '\n')
    return files_in_manifest

def copy_dir():
    if _platform == "win32":
        subprocess.call(['robocopy',source, destination_final_path, '/E'])
        generate_log(log_name_source, 'EVENT = File Transfer - Windows O.S - Software=Robocopy')
        print destination_final_path
    elif _platform == "darwin":
        # https://github.com/amiaopensource/ltopers/blob/master/writelto#L51
        cmd = ['gcp','--preserve=mode,timestamps', '-nRv',source, destination_final_path]
        generate_log(log_name_source, 'EVENT = File Transfer - OSX - Software=gcp')

        subprocess.call(cmd)
    elif _platform == "linux2":
        # https://github.com/amiaopensource/ltopers/blob/master/writelto#L51
        cmd = [ 'cp','--preserve=mode,timestamps', '-nRv',source, destination_final_path]
        generate_log(log_name_source, 'EVENT = File Transfer - Linux- Software=cp')
        subprocess.call(cmd)

def check_overwrite(file2check):
    if os.path.isfile(file2check):
        print 'A manifest already exists at your destination. Overwrite? Y/N?'
        overwrite_destination_manifest = ''
        while overwrite_destination_manifest not in ('Y','y','N','n'):
            overwrite_destination_manifest = raw_input()
            if overwrite_destination_manifest not in ('Y','y','N','n'):
                print 'Incorrect input. Please enter Y or N'
        return overwrite_destination_manifest
def manifest_file_count(manifest2check):
    if os.path.isfile(manifest2check):
        print 'A manifest already exists'
        with open(manifest2check, "r") as fo:
            manifest_lines = [line.split(',') for line in fo.readlines()]
            count_in_manifest =  len(manifest_lines)
    return count_in_manifest

def check_overwrite_dir(dir2check):
    if os.path.isdir(dir2check):
        print 'A directory already exists at your destination. Overwrite? Y/N?'
        overwrite_destination_dir = ''
        while overwrite_destination_dir not in ('Y','y','N','n'):
            overwrite_destination_dir = raw_input()
            if overwrite_destination_dir not in ('Y','y','N','n'):
                print 'Incorrect input. Please enter Y or N'
        return overwrite_destination_dir

try:
    test_write_capabilities(source_parent_dir)
except OSError:
            print 'You cannot write to your source directory!'
            generate_log(log_name_source, 'EVENT = I/O Test - Failure - No write access to source directory.')
            sys.exit()

try:
    test_write_capabilities(destination)
except OSError:
            print 'You cannot write to your destination!'
            generate_log(log_name_source, 'EVENT = I/O Test - Failure - No write access to destination directory.')
            sys.exit()
overwrite_destination_manifest = check_overwrite(manifest_destination)
overwrite_destination_dir = check_overwrite_dir(destination_final_path)

remove_bad_files(source)
source_count = 0

for root, directories, filenames in os.walk(source):
    for files in filenames:
            source_count +=1 #works in windows at least
print source_count

if os.path.isfile(manifest):
    count_in_manifest = manifest_file_count(manifest)
    if source_count != count_in_manifest:
        print 'This manifest may be outdated as the number of files in your directory does not match the number of files in the manifest'
        generate_log(log_name_source, 'EVENT = Existing source manifest check - Failure - The number of files in the source directory is not equal to the number of files in the source manifest ')
        sys.exit()
source_manifest_start_time = time.time()

if not os.path.isfile(manifest):
    try:
        print 'Generating source manifest'
        make_manifest(source_parent_dir, relative_path,manifest)
        generate_log(log_name_source, 'EVENT = Generating source manifest')

    except OSError:
            print 'You do not have access to this directory. Perhaps it is read only, or the wrong file system\n'
            sys.exit()

source_manifest_time = time.time() - source_manifest_start_time

copy_start_time = time.time()
if overwrite_destination_dir not in ('N','n'):
    if overwrite_destination_dir != None:
        generate_log(log_name_source, 'EVENT = File Transfer Overwrite - Destination directory already exists - Overwriting.')
    copy_dir()
else:
    generate_log(log_name_source, 'EVENT = File Transfer Overwrite - Destination directory already exists - Not Overwriting.')

copy_time = time.time() - copy_start_time

start_destination_manifest_time = time.time()
if overwrite_destination_manifest not in ('N','n'):
    if overwrite_destination_manifest == None:
        generate_log(log_name_source, 'EVENT = Destination Manifest Generation')
    else:
        generate_log(log_name_source, 'EVENT = Destination Manifest Overwrite - Destination manifest already exists - Overwriting.')
    print 'Generating destination manifest'
    files_in_manifest = make_manifest(destination,dirname, manifest_destination)
else:
    generate_log(log_name_source, 'EVENT = File Transfer Overwrite - Destination directory already exists - Not Overwriting.')
remove_bad_files(destination_final_path)

destination_manifest_time = time.time() - start_destination_manifest_time
destination_count = 0

for root, directories, filenames in os.walk(destination_final_path):
    for files in filenames:
            destination_count +=1 #works in windows at least
print destination_count

if filecmp.cmp(manifest, manifest_destination, shallow=False):
    print "Your files have reached their destination and the checksums match"
    generate_log(log_name_source, 'EVENT = File Transfer Judgement - Success')
else:
    print "***********YOUR CHECKSUMS DO NOT MATCH*************"
    if overwrite_destination_manifest not in ('N','n'):
        generate_log(log_name_source, 'EVENT = File Transfer Outcome - Failure')
        print ' There are: \n %s files in your destination manifest \n' % files_in_manifest
        print ' %s files in your destination \n %s files at source' % (destination_count, source_count)
        generate_log(log_name_source, 'EVENT = File Transfer Failure Explanation -  %s files in your destination,  %s files at source' % (destination_count, source_count))
    else:
        print ' %s files in your destination \n %s files at source' % (destination_count, source_count)
if args.b:
    display_benchmark()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import division

import copy
import logging

from numpy import maximum as max_, minimum as min_
from openfisca_core import formulas, reforms, tools

from ..base import *  # noqa

log = logging.getLogger(__name__)

def _reductions_2013(accult, adhcga, cappme, creaen, daepad, deffor, dfppce, doment, domlog, donapd, duflot, ecpess,
                     garext, intagr, invfor, invlst, ip_net, locmeu, mecena, mohist, patnat, prcomp, repsoc, resimm,
                     rsceha, saldom, scelli, sofica, spfcpi, reduction_impot_exceptionnelle):
    '''
    Renvoie la somme des réductions d'impôt à intégrer pour l'année 2013
    '''
    total_reductions = (accult + adhcga + cappme + creaen + daepad + deffor + dfppce + doment + domlog + donapd +
                        duflot + ecpess + garext + intagr + invfor + invlst + locmeu + mecena + mohist + patnat +
                        prcomp + repsoc + resimm + rsceha + saldom + scelli + sofica + spfcpi +
                        reduction_impot_exceptionnelle)
    return min_(ip_net, total_reductions)

def build_reform_entity_class_by_key_plural(tax_benefit_system):
    reform_entity_class_by_key_plural = tax_benefit_system.entity_class_by_key_plural.copy()
    FoyersFiscaux = reform_entity_class_by_key_plural['foyers_fiscaux']
    ReformFoyersFiscaux = type('ReformFoyersFiscaux', (FoyersFiscaux, ),
        {'column_by_name': FoyersFiscaux.column_by_name.copy()})
    reform_entity_class_by_key_plural['foyers_fiscaux'] = ReformFoyersFiscaux

    # reduction_impot_exceptionnelle
    class reduction_impot_exceptionnelle(formulas.SimpleFormulaColumn):
        column = FloatCol
        entity_class = FoyersFiscaux
        label = u"Réduction d'impôt exceptionnelle"

        def function(self, rfr, nb_adult, nb_par, _P):
            parametres = _P.plfr2014.reduction_impot_exceptionnelle
            plafond = parametres.seuil * nb_adult + (nb_par - nb_adult) * 2 * parametres.majoration_seuil
            montant = parametres.montant_plafond * nb_adult
            return min_(max_(plafond + montant - rfr, 0), montant)

        def get_output_period(self, period):
            return period.start.offset('first-of', 'month').period('year')

    ReformFoyersFiscaux.column_by_name['reduction_impot_exceptionnelle'] = reduction_impot_exceptionnelle

    # reductions

    reductions_column = FoyersFiscaux.column_by_name['reductions']
    reform_reductions_column = tools.empty_clone(reductions_column)
    reform_reductions_column.__dict__ = reductions_column.__dict__.copy()

    reductions_formula_class_2013 = reform_reductions_column.formula_class.dated_formulas_class[-1]['formula_class']  # noqa
    reform_reductions_formula_class_2013 = type(
        'reform_reductions_formula_class_2013',
        (reductions_formula_class_2013, ),
        {'function': staticmethod(_reductions_2013)},
        )
    reform_reductions_formula_class_2013.extract_variables_name()

    reform_dated_formulas_class = reform_reductions_column.formula_class.dated_formulas_class[:]
    reform_dated_formulas_class[-1] = reform_dated_formulas_class[-1].copy()
    reform_dated_formulas_class[-1]['formula_class'] = reform_reductions_formula_class_2013

    reform_dated_formula_class = type('reform_dated_formula_class', (reform_reductions_column.formula_class, ),
        {'dated_formulas_class': reform_dated_formulas_class})

    reform_reductions_column.formula_class = reform_dated_formula_class

    ReformFoyersFiscaux.column_by_name['reductions'] = reform_reductions_column

    return reform_entity_class_by_key_plural

def build_new_legislation_nodes():
    return {
        "plfr2014": {
            "@type": "Node",
            "description": "Projet de loi de finance 2014",
            "children": {
                "reduction_impot_exceptionnelle": {
                    "@type": "Node",
                    "description": "Réduction d'impôt exceptionnelle",
                    "children": {
                        "montant_plafond": {
                            "@type": "Parameter",
                            "description": "Montant plafond par part pour les deux premières parts",
                            "format": "integer",
                            "unit": "currency",
                            "values": [{'start': u'2013-01-01', 'stop': u'2014-12-31', 'value': 350}],
                            },
                        "seuil": {
                            "@type": "Parameter",
                            "description": "Seuil (à partir duquel la réduction décroît) par part pour les deux "
                                           "premières parts",
                            "format": "integer",
                            "unit": "currency",
                            "values": [{'start': u'2013-01-01', 'stop': u'2014-12-31', 'value': 13795}],
                            },
                        "majoration_seuil": {
                            "@type": "Parameter",
                            "description": "Majoration du seuil par demi-part supplémentaire",
                            "format": "integer",
                            "unit": "currency",
                            "values": [{'start': u'2013-01-01', 'stop': u'2014-12-31', 'value': 3536}],
                            },
                        },
                    },
                },
            },
        }

def build_reform(tax_benefit_system):
    reference_legislation_json = tax_benefit_system.legislation_json
    reform_legislation_json = copy.deepcopy(reference_legislation_json)
    reform_legislation_json['children'].update(build_new_legislation_nodes())
    return reforms.Reform(
        entity_class_by_key_plural = build_reform_entity_class_by_key_plural(tax_benefit_system),
        legislation_json = reform_legislation_json,
        name = u'PLFR2014',
        reference = tax_benefit_system,
        )
<commit_msg>Remove extract_variables_name call<commit_after># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import division

import copy
import logging

from numpy import maximum as max_, minimum as min_
from openfisca_core import formulas, reforms, tools

from ..base import *  # noqa

log = logging.getLogger(__name__)

def _reductions_2013(accult, adhcga, cappme, creaen, daepad, deffor, dfppce, doment, domlog, donapd, duflot, ecpess,
                     garext, intagr, invfor, invlst, ip_net, locmeu, mecena, mohist, patnat, prcomp, repsoc, resimm,
                     rsceha, saldom, scelli, sofica, spfcpi, reduction_impot_exceptionnelle):
    '''
    Renvoie la somme des réductions d'impôt à intégrer pour l'année 2013
    '''
    total_reductions = (accult + adhcga + cappme + creaen + daepad + deffor + dfppce + doment + domlog + donapd +
                        duflot + ecpess + garext + intagr + invfor + invlst + locmeu + mecena + mohist + patnat +
                        prcomp + repsoc + resimm + rsceha + saldom + scelli + sofica + spfcpi +
                        reduction_impot_exceptionnelle)
    return min_(ip_net, total_reductions)

def build_reform_entity_class_by_key_plural(tax_benefit_system):
    reform_entity_class_by_key_plural = tax_benefit_system.entity_class_by_key_plural.copy()
    FoyersFiscaux = reform_entity_class_by_key_plural['foyers_fiscaux']
    ReformFoyersFiscaux = type('ReformFoyersFiscaux', (FoyersFiscaux, ),
        {'column_by_name': FoyersFiscaux.column_by_name.copy()})
    reform_entity_class_by_key_plural['foyers_fiscaux'] = ReformFoyersFiscaux

    # reduction_impot_exceptionnelle
    class reduction_impot_exceptionnelle(formulas.SimpleFormulaColumn):
        column = FloatCol
        entity_class = FoyersFiscaux
        label = u"Réduction d'impôt exceptionnelle"

        def function(self, rfr, nb_adult, nb_par, _P):
            parametres = _P.plfr2014.reduction_impot_exceptionnelle
            plafond = parametres.seuil * nb_adult + (nb_par - nb_adult) * 2 * parametres.majoration_seuil
            montant = parametres.montant_plafond * nb_adult
            return min_(max_(plafond + montant - rfr, 0), montant)

        def get_output_period(self, period):
            return period.start.offset('first-of', 'month').period('year')

    ReformFoyersFiscaux.column_by_name['reduction_impot_exceptionnelle'] = reduction_impot_exceptionnelle

    # reductions

    reductions_column = FoyersFiscaux.column_by_name['reductions']
    reform_reductions_column = tools.empty_clone(reductions_column)
    reform_reductions_column.__dict__ = reductions_column.__dict__.copy()

    reductions_formula_class_2013 = reform_reductions_column.formula_class.dated_formulas_class[-1]['formula_class']  # noqa
    reform_reductions_formula_class_2013 = type(
        'reform_reductions_formula_class_2013',
        (reductions_formula_class_2013, ),
        {'function': staticmethod(_reductions_2013)},
        )

    reform_dated_formulas_class = reform_reductions_column.formula_class.dated_formulas_class[:]
    reform_dated_formulas_class[-1] = reform_dated_formulas_class[-1].copy()
    reform_dated_formulas_class[-1]['formula_class'] = reform_reductions_formula_class_2013

    reform_dated_formula_class = type('reform_dated_formula_class', (reform_reductions_column.formula_class, ),
        {'dated_formulas_class': reform_dated_formulas_class})

    reform_reductions_column.formula_class = reform_dated_formula_class

    ReformFoyersFiscaux.column_by_name['reductions'] = reform_reductions_column

    return reform_entity_class_by_key_plural

def build_new_legislation_nodes():
    return {
        "plfr2014": {
            "@type": "Node",
            "description": "Projet de loi de finance 2014",
            "children": {
                "reduction_impot_exceptionnelle": {
                    "@type": "Node",
                    "description": "Réduction d'impôt exceptionnelle",
                    "children": {
                        "montant_plafond": {
                            "@type": "Parameter",
                            "description": "Montant plafond par part pour les deux premières parts",
                            "format": "integer",
                            "unit": "currency",
                            "values": [{'start': u'2013-01-01', 'stop': u'2014-12-31', 'value': 350}],
                            },
                        "seuil": {
                            "@type": "Parameter",
                            "description": "Seuil (à partir duquel la réduction décroît) par part pour les deux "
                                           "premières parts",
                            "format": "integer",
                            "unit": "currency",
                            "values": [{'start': u'2013-01-01', 'stop': u'2014-12-31', 'value': 13795}],
                            },
                        "majoration_seuil": {
                            "@type": "Parameter",
                            "description": "Majoration du seuil par demi-part supplémentaire",
                            "format": "integer",
                            "unit": "currency",
                            "values": [{'start': u'2013-01-01', 'stop': u'2014-12-31', 'value': 3536}],
                            },
                        },
                    },
                },
            },
        }

def build_reform(tax_benefit_system):
    reference_legislation_json = tax_benefit_system.legislation_json
    reform_legislation_json = copy.deepcopy(reference_legislation_json)
    reform_legislation_json['children'].update(build_new_legislation_nodes())
    return reforms.Reform(
        entity_class_by_key_plural = build_reform_entity_class_by_key_plural(tax_benefit_system),
        legislation_json = reform_legislation_json,
        name = u'PLFR2014',
        reference = tax_benefit_system,
        )
<|endoftext|>


<commit_before>import gzip
#
import boto
import warc
#
from boto.s3.key import Key
from gzipstream import GzipStreamFile
from mrjob.job import MRJob

class CCJob(MRJob):
  def process_record(self, record):
    """
    Override process_record with your mapper
    """
    raise NotImplementedError('Process record needs to be customized')

  def mapper(self, _, line):
    f = None
    ## If we're on EC2 or running on a Hadoop cluster, pull files via S3
    if self.options.runner in ['emr', 'hadoop']:
      # Connect to Amazon S3 using anonymous credentials
      conn = boto.connect_s3(anon=True)
      pds = conn.get_bucket('aws-publicdatasets')
      # Start a connection to one of the WARC files
      k = Key(pds, line)
      f = warc.WARCFile(fileobj=GzipStreamFile(k))
    ## If we're local, use files on the local file system
    else:
      print 'Loading local file {}'.format(line)
      f = warc.WARCFile(fileobj=gzip.open(line))
    ###
    for i, record in enumerate(f):
      for key, value in self.process_record(record):
        yield key, value
      self.increment_counter('commoncrawl', 'processed_records', 1)

  def combiner(self, key, value):
    yield key, sum(value)

  def reducer(self, key, value):
    yield key, sum(value)
<commit_msg>Add CCJob.run() to mrcc.py<commit_after>import gzip
#
import boto
import warc
#
from boto.s3.key import Key
from gzipstream import GzipStreamFile
from mrjob.job import MRJob

class CCJob(MRJob):
  def process_record(self, record):
    """
    Override process_record with your mapper
    """
    raise NotImplementedError('Process record needs to be customized')

  def mapper(self, _, line):
    f = None
    ## If we're on EC2 or running on a Hadoop cluster, pull files via S3
    if self.options.runner in ['emr', 'hadoop']:
      # Connect to Amazon S3 using anonymous credentials
      conn = boto.connect_s3(anon=True)
      pds = conn.get_bucket('aws-publicdatasets')
      # Start a connection to one of the WARC files
      k = Key(pds, line)
      f = warc.WARCFile(fileobj=GzipStreamFile(k))
    ## If we're local, use files on the local file system
    else:
      print 'Loading local file {}'.format(line)
      f = warc.WARCFile(fileobj=gzip.open(line))
    ###
    for i, record in enumerate(f):
      for key, value in self.process_record(record):
        yield key, value
      self.increment_counter('commoncrawl', 'processed_records', 1)

  def combiner(self, key, value):
    yield key, sum(value)

  def reducer(self, key, value):
    yield key, sum(value)

if __name__ == '__main__':
  CCJob.run()<|endoftext|>