<commit_before>from pyramid.httpexceptions import HTTPBadRequest
import rfc3987

from hel.utils import parse_search_phrase
from hel.utils.constants import Constants
from hel.utils.messages import Messages
from hel.utils.version import latest_version

def _only_one_param(func):
    func._only_one = True
    return func

class PackagesSearchParams:

    @_only_one_param
    def name(param):
        """Search by name"""

        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                if phrase not in pkg['name']:
                    success = False
                    break
            return success

        return search

    @_only_one_param
    def description(param):
        """Search by description"""

        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                if phrase not in pkg['description']:
                    success = False
                    break
            return success

        return search

    @_only_one_param
    def short_description(param):
        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                if phrase not in pkg['short_description']:
                    success = False
                    break
            return success

        return search

    @_only_one_param
    def authors(param):
        """Search by author name"""

        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                success_loop = False
                for author in pkg['authors']:
                    if phrase in author:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    def owners(param):

        def search(pkg):
            success = True
            for rq_owner in param:
                if rq_owner not in pkg['owners']:
                    return False
            return True

        return search

    @_only_one_param
    def license(param):
        """Search by license"""

        def search(pkg):
            return param in pkg['license']

        return search

    def tags(param):
        """Search by tag.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            for exp_tag in param:
                success_loop = False
                for tag in pkg['tags']:
                    if exp_tag == tag:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    def file_url(param):
        """Search by file URL.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            ver = pkg['versions'][str(latest_version(pkg))]
            for url in param:
                if url not in ver['files']:
                    success = False
                    break
            return success

        return search

    def file_dir(param):
        """Search by directory.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            ver = pkg['versions'][str(latest_version(pkg))]
            for d in param:
                success_loop = False
                for k, v in ver['files'].items():
                    if v['dir'] == d:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    def file_name(param):
        """Search by file name.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            ver = pkg['versions'][str(latest_version(pkg))]
            for name in param:
                success_loop = False
                for k, v in ver['files'].items():
                    if v['name'] == name:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    def dependency(param):
        """Returns packages depending on specific packages.

        Param format: <name>[:<version>[:<type>]]
        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            ver = pkg['versions'][str(latest_version(pkg))]
            for dep_info in param:
                dep = dep_info.split(':')
                name = dep[0]
                if len(dep) > 2:
                    dep_type = dep[2]
                if len(dep) > 1:
                    version = dep[1]
                if name not in ver['depends']:
                    success = False
                else:
                    if (len(dep) > 2 and
                            ver['depends'][name]
                            ['version'] != version):
                        success = False
                    else:
                        if (len(dep) > 2 and
                                ver['depends'][name]
                                ['type'] != dep_type):
                            success = False
            return success

        return search

    def screen_url(param):
        """Search by screenshot URL.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            for url in param:
                if url not in pkg['screenshots']:
                    success = False
            return success

        return search

    @_only_one_param
    def screen_desc(param):
        """Search by screenshot description"""

        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                success_loop = False
                for k, v in pkg['screenshots'].items():
                    if phrase in v:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    @_only_one_param
    def q(param):

        def search(pkg):
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                found = False
                for k in ["name", "description", "short_description"]:
                    if phrase in pkg[k].lower():
                        found = True
                        break
                if found:
                    continue

                for k in ["owners", "authors", "license", "tags"]:
                    for v in pkg[k]:
                        if phrase in v.lower():
                            found = True
                            break
                    if found:
                        break
                if found:
                    continue

                for k, screenshot in pkg["screenshots"].items():
                    if phrase in screenshot.lower():
                        found = True
                        break
                if found:
                    continue

                for k, version in pkg["versions"].items():
                    if phrase in version["changes"].lower():
                        found = True
                        break
                if found:
                    continue

                # If we reached this line, `continue` lines weren't executed.
                # It means that the phrase was not found.
                # As all phrases must be found, we terminate the loop.
                return False

            return True

        return search

class PackagesSearcher:

    def __init__(self, params):
        self.params = params

    def __call__(self):
        searchers = []
        for param_name, param in self.params.items():
            if hasattr(PackagesSearchParams, param_name):
                param_method = getattr(PackagesSearchParams, param_name)
                if (not hasattr(param_method, '_no_param') and
                        (len(param) == 0 or param[0] == '')):
                    raise HTTPBadRequest(
                        detail=Messages.no_values % param_name)
                if hasattr(param_method, '_only_one'):
                    if len(param) > 1:
                        raise HTTPBadRequest(
                            detail=Messages.too_many_values % (1, len(param),))
                    else:
                        param = param[0]
                search = param_method(param)
                searchers.append(search)
            else:
                raise HTTPBadRequest(
                    detail=Messages.bad_search_param % param_name)
        self.searchers = searchers
        return searchers

    def search(self, pkgs):
        packages = []
        for pkg in pkgs:
            packages.append(
                replace_chars_in_keys(
                    pkg, Constants.key_replace_char, '.'))
        result = []
        for pkg in packages:
            success = True
            for searcher in self.searchers:
                if not searcher(pkg):
                    success = False
                    break
            if success:
                result.append(pkg)
        return result

def check(value, expected_type, message=None):
    if type(value) != expected_type:
        raise HTTPBadRequest(detail=message)
    return value

def check_list_of_strs(value, message=None):
    check(value, list, message)
    for item in value:
        if type(item) != str:
            raise HTTPBadRequest(detail=message)
    return value

def replace_chars_in_keys(d, repl_chr, repl_to):
    try:
        result = {}
        for k, v in d.items():
            result[k.replace(repl_chr, repl_to)] = replace_chars_in_keys(
                v, repl_chr, repl_to)
    except (ValueError, TypeError, AttributeError):
        return d
    return result

def parse_url(url):
    try:
        matches = rfc3987.parse(url, rule='URI')
    except ValueError:
        raise HTTPBadRequest(detail=Messages.invalid_uri)
    if matches['scheme'] not in ['http', 'https']:
        raise HTTPBadRequest(detail=Messages.invalid_uri)
    matches['path'] = matches['path'] or '/'
    matches['fragment'] = None
    return rfc3987.compose(**matches)
<commit_msg>Also make search phrase case-insensitive<commit_after>from pyramid.httpexceptions import HTTPBadRequest
import rfc3987

from hel.utils import parse_search_phrase
from hel.utils.constants import Constants
from hel.utils.messages import Messages
from hel.utils.version import latest_version

def _only_one_param(func):
    func._only_one = True
    return func

class PackagesSearchParams:

    @_only_one_param
    def name(param):
        """Search by name"""

        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                if phrase not in pkg['name']:
                    success = False
                    break
            return success

        return search

    @_only_one_param
    def description(param):
        """Search by description"""

        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                if phrase not in pkg['description']:
                    success = False
                    break
            return success

        return search

    @_only_one_param
    def short_description(param):
        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                if phrase not in pkg['short_description']:
                    success = False
                    break
            return success

        return search

    @_only_one_param
    def authors(param):
        """Search by author name"""

        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                success_loop = False
                for author in pkg['authors']:
                    if phrase in author:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    def owners(param):

        def search(pkg):
            success = True
            for rq_owner in param:
                if rq_owner not in pkg['owners']:
                    return False
            return True

        return search

    @_only_one_param
    def license(param):
        """Search by license"""

        def search(pkg):
            return param in pkg['license']

        return search

    def tags(param):
        """Search by tag.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            for exp_tag in param:
                success_loop = False
                for tag in pkg['tags']:
                    if exp_tag == tag:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    def file_url(param):
        """Search by file URL.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            ver = pkg['versions'][str(latest_version(pkg))]
            for url in param:
                if url not in ver['files']:
                    success = False
                    break
            return success

        return search

    def file_dir(param):
        """Search by directory.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            ver = pkg['versions'][str(latest_version(pkg))]
            for d in param:
                success_loop = False
                for k, v in ver['files'].items():
                    if v['dir'] == d:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    def file_name(param):
        """Search by file name.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            ver = pkg['versions'][str(latest_version(pkg))]
            for name in param:
                success_loop = False
                for k, v in ver['files'].items():
                    if v['name'] == name:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    def dependency(param):
        """Returns packages depending on specific packages.

        Param format: <name>[:<version>[:<type>]]
        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            ver = pkg['versions'][str(latest_version(pkg))]
            for dep_info in param:
                dep = dep_info.split(':')
                name = dep[0]
                if len(dep) > 2:
                    dep_type = dep[2]
                if len(dep) > 1:
                    version = dep[1]
                if name not in ver['depends']:
                    success = False
                else:
                    if (len(dep) > 2 and
                            ver['depends'][name]
                            ['version'] != version):
                        success = False
                    else:
                        if (len(dep) > 2 and
                                ver['depends'][name]
                                ['type'] != dep_type):
                            success = False
            return success

        return search

    def screen_url(param):
        """Search by screenshot URL.

        Logical AND is used to concatenate params.
        """

        def search(pkg):
            success = True
            for url in param:
                if url not in pkg['screenshots']:
                    success = False
            return success

        return search

    @_only_one_param
    def screen_desc(param):
        """Search by screenshot description"""

        def search(pkg):
            success = True
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                success_loop = False
                for k, v in pkg['screenshots'].items():
                    if phrase in v:
                        success_loop = True
                        break
                if not success_loop:
                    success = False
                    break
            return success

        return search

    @_only_one_param
    def q(param):

        def search(pkg):
            phrases = parse_search_phrase(param)
            for phrase in phrases:
                found = False
                for k in ["name", "description", "short_description"]:
                    if phrase.lower() in pkg[k].lower():
                        found = True
                        break
                if found:
                    continue

                for k in ["owners", "authors", "license", "tags"]:
                    for v in pkg[k]:
                        if phrase.lower() in v.lower():
                            found = True
                            break
                    if found:
                        break
                if found:
                    continue

                for k, screenshot in pkg["screenshots"].items():
                    if phrase.lower() in screenshot.lower():
                        found = True
                        break
                if found:
                    continue

                for k, version in pkg["versions"].items():
                    if phrase.lower() in version["changes"].lower():
                        found = True
                        break
                if found:
                    continue

                # If we reached this line, `continue` lines weren't executed.
                # It means that the phrase was not found.
                # As all phrases must be found, we terminate the loop.
                return False

            return True

        return search

class PackagesSearcher:

    def __init__(self, params):
        self.params = params

    def __call__(self):
        searchers = []
        for param_name, param in self.params.items():
            if hasattr(PackagesSearchParams, param_name):
                param_method = getattr(PackagesSearchParams, param_name)
                if (not hasattr(param_method, '_no_param') and
                        (len(param) == 0 or param[0] == '')):
                    raise HTTPBadRequest(
                        detail=Messages.no_values % param_name)
                if hasattr(param_method, '_only_one'):
                    if len(param) > 1:
                        raise HTTPBadRequest(
                            detail=Messages.too_many_values % (1, len(param),))
                    else:
                        param = param[0]
                search = param_method(param)
                searchers.append(search)
            else:
                raise HTTPBadRequest(
                    detail=Messages.bad_search_param % param_name)
        self.searchers = searchers
        return searchers

    def search(self, pkgs):
        packages = []
        for pkg in pkgs:
            packages.append(
                replace_chars_in_keys(
                    pkg, Constants.key_replace_char, '.'))
        result = []
        for pkg in packages:
            success = True
            for searcher in self.searchers:
                if not searcher(pkg):
                    success = False
                    break
            if success:
                result.append(pkg)
        return result

def check(value, expected_type, message=None):
    if type(value) != expected_type:
        raise HTTPBadRequest(detail=message)
    return value

def check_list_of_strs(value, message=None):
    check(value, list, message)
    for item in value:
        if type(item) != str:
            raise HTTPBadRequest(detail=message)
    return value

def replace_chars_in_keys(d, repl_chr, repl_to):
    try:
        result = {}
        for k, v in d.items():
            result[k.replace(repl_chr, repl_to)] = replace_chars_in_keys(
                v, repl_chr, repl_to)
    except (ValueError, TypeError, AttributeError):
        return d
    return result

def parse_url(url):
    try:
        matches = rfc3987.parse(url, rule='URI')
    except ValueError:
        raise HTTPBadRequest(detail=Messages.invalid_uri)
    if matches['scheme'] not in ['http', 'https']:
        raise HTTPBadRequest(detail=Messages.invalid_uri)
    matches['path'] = matches['path'] or '/'
    matches['fragment'] = None
    return rfc3987.compose(**matches)
<|endoftext|>


<commit_before>"""
Django settings for heldev project.

Generated by 'django-admin startproject' using Django 1.8.3.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os

PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
BASE_DIR = os.path.dirname(PROJECT_DIR)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

# Application definition

INSTALLED_APPS = (
    'grappelli',
    'django.contrib.sites',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'taggit',
    'compressor',
    'modelcluster',

    'wagtail.wagtailcore',
    'wagtail.wagtailadmin',
    'wagtail.wagtailsearch',
    'wagtail.wagtailimages',
    'wagtail.wagtaildocs',
    'wagtail.wagtailsnippets',
    'wagtail.wagtailusers',
    'wagtail.wagtailsites',
    'wagtail.wagtailembeds',
    'wagtail.wagtailredirects',
    'wagtail.wagtailforms',

    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    'helusers.providers.helsinki',

    'blog',
    'djangobower',
    'helusers',

    'search',
    'home',
    'apimanager',
    'projects',
    'users',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',

    'wagtail.wagtailcore.middleware.SiteMiddleware',
    'wagtail.wagtailredirects.middleware.RedirectMiddleware',
)

ROOT_URLCONF = 'heldev.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(PROJECT_DIR, 'templates'),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'heldev.wsgi.application'

# Bower apps

BOWER_INSTALLED_APPS = (
    'compass-mixins',
    'susy',
    'jquery',
    'pushy',
    'imagesloaded',
    'imagefill',
    'compass-breakpoint'
)

COMPRESS_PRECOMPILERS = (
    ('text/x-scss', 'django_libsass.SassCompiler'),
)

# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'heldev'
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'djangobower.finders.BowerFinder',
    'compressor.finders.CompressorFinder',
)

STATICFILES_DIRS = (
    os.path.join(PROJECT_DIR, 'static'),
)

STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'

MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = '/media/'

# Wagtail settings

WAGTAIL_SITE_NAME = "heldev"

AUTH_USER_MODEL = 'users.User'
AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
)
SITE_ID = 1

SOCIALACCOUNT_PROVIDERS = {
    'helsinki': {
        'VERIFIED_EMAIL': True
    }
}
LOGIN_REDIRECT_URL = '/'

DEBUG = False
TEMPLATE_DEBUG = False

try:
    from local_settings import *
except ImportError:
    raise

if 'CUSTOM_INSTALLED_APPS' in locals():
    INSTALLED_APPS = INSTALLED_APPS + CUSTOM_INSTALLED_APPS

if 'SECRET_KEY' not in locals():
    secret_file = os.path.join(BASE_DIR, '.django_secret')
    try:
        SECRET_KEY = open(secret_file).read().strip()
    except IOError:
        import random
        system_random = random.SystemRandom()
        try:
            SECRET_KEY = ''.join([system_random.choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(64)])
            secret = open(secret_file, 'w')
            import os
            os.chmod(secret_file, 0o0600)
            secret.write(SECRET_KEY)
            secret.close()
        except IOError:
            Exception('Please create a %s file with random characters to generate your secret key!' % secret_file)
<commit_msg>Fix Helsinki OAuth settings<commit_after>"""
Django settings for heldev project.

Generated by 'django-admin startproject' using Django 1.8.3.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os

PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
BASE_DIR = os.path.dirname(PROJECT_DIR)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

# Application definition

INSTALLED_APPS = (
    'grappelli',
    'django.contrib.sites',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'taggit',
    'compressor',
    'modelcluster',

    'wagtail.wagtailcore',
    'wagtail.wagtailadmin',
    'wagtail.wagtailsearch',
    'wagtail.wagtailimages',
    'wagtail.wagtaildocs',
    'wagtail.wagtailsnippets',
    'wagtail.wagtailusers',
    'wagtail.wagtailsites',
    'wagtail.wagtailembeds',
    'wagtail.wagtailredirects',
    'wagtail.wagtailforms',

    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    'helusers.providers.helsinki',

    'blog',
    'djangobower',
    'helusers',

    'search',
    'home',
    'apimanager',
    'projects',
    'users',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',

    'wagtail.wagtailcore.middleware.SiteMiddleware',
    'wagtail.wagtailredirects.middleware.RedirectMiddleware',
)

ROOT_URLCONF = 'heldev.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(PROJECT_DIR, 'templates'),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'heldev.wsgi.application'

# Bower apps

BOWER_INSTALLED_APPS = (
    'compass-mixins',
    'susy',
    'jquery',
    'pushy',
    'imagesloaded',
    'imagefill',
    'compass-breakpoint'
)

COMPRESS_PRECOMPILERS = (
    ('text/x-scss', 'django_libsass.SassCompiler'),
)

# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'heldev'
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'djangobower.finders.BowerFinder',
    'compressor.finders.CompressorFinder',
)

STATICFILES_DIRS = (
    os.path.join(PROJECT_DIR, 'static'),
)

STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'

MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = '/media/'

# Wagtail settings

WAGTAIL_SITE_NAME = "heldev"

AUTH_USER_MODEL = 'users.User'
AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
)
SITE_ID = 1

SOCIALACCOUNT_PROVIDERS = {
    'helsinki': {
        'VERIFIED_EMAIL': True
    }
}
LOGIN_REDIRECT_URL = '/'
ACCOUNT_LOGOUT_ON_GET = True
SOCIALACCOUNT_ADAPTER = 'helusers.providers.helsinki.provider.SocialAccountAdapter'

DEBUG = False
TEMPLATE_DEBUG = False

try:
    from local_settings import *
except ImportError:
    raise

if 'CUSTOM_INSTALLED_APPS' in locals():
    INSTALLED_APPS = INSTALLED_APPS + CUSTOM_INSTALLED_APPS

if 'SECRET_KEY' not in locals():
    secret_file = os.path.join(BASE_DIR, '.django_secret')
    try:
        SECRET_KEY = open(secret_file).read().strip()
    except IOError:
        import random
        system_random = random.SystemRandom()
        try:
            SECRET_KEY = ''.join([system_random.choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(64)])
            secret = open(secret_file, 'w')
            import os
            os.chmod(secret_file, 0o0600)
            secret.write(SECRET_KEY)
            secret.close()
        except IOError:
            Exception('Please create a %s file with random characters to generate your secret key!' % secret_file)
<|endoftext|>


<commit_before>from blessings import Terminal
from pprint import pprint as ppr

term = Terminal()

# Display utilities

def _func_or_print(result, func):
    """Private function to either print or call a function on result"""
    if func is not None:
        func(result)
    else:
        print(result)

def _uncase_x(string, seperator):
    """Split by `seperator` and join the string;
    e.g. 'foo_bar' => foo bar """
    return ' '.join(string.split(str(seperator)))

def uncase_period(string):
    return _uncase_x(string, '.')

def uncase_hyphen(string):
    return _uncase_x(string, '-')

def uncase_snake(string):
    return _uncase_x(string, '_')

def uncase_snake_upper(string):
    return uncase_snake(string).upper()

def firstcaps(string):
    string = list(string.lower())
    string[0] = string[0].upper()
    return ''.join(string)

def title_case(string, seperator='_'):
    """Format a string into title case - e.g. 'some_word' => Some Word """
    return ' '.join(map(firstcaps, string.split(seperator)))

def divider(atom='_', newline=True):
    """Print a divider. Optionally override the unit of text to use
    (e.g ---, ...., ###)"""
    # Correct for longer than 1 character atoms, since it will take up
    # more than 80 chars.
    print(atom * (80 // len(atom)))
    if newline:
        print('\n')

def print_subdued(msg):
    """Print info-type text in dark."""
    vals = '{t.black}{}{t.normal}'
    print(vals.format(msg, t=term))

def print_info(msg, prefix='[INFO]'):
    """Print info-type text in red with a prefix"""
    vals = '{t.blue}{} {}{t.normal}'
    print(vals.format(prefix, msg, t=term))

def print_warning(msg, prefix='[WARN]'):
    """Print warning-type text in red with a prefix"""
    vals = '{t.yellow}{} {}{t.normal}'
    print(vals.format(prefix, msg, t=term))

def print_success(msg, prefix='[YAY]'):
    """Print success-type text in red with a prefix"""
    vals = '{t.green}{} {}{t.normal}'
    print(vals.format(prefix, msg, t=term))

def print_error(msg, prefix='[ERROR]'):
    """Print error-type text in red with a prefix"""
    vals = '{t.red}{} {}{t.normal}'
    print(vals.format(prefix, msg, t=term))

def prnt(title, result, func=None, newlines=False):
    """A more useful default print function for titles and accompanying
    content. Shows stylized title, with content below, and newlines.
    The content can optionally be formatted by a given `func`."""
    if newlines:
        print('\n')
    print('{t.green}{t.underline}{}{t.normal}'.format(title, t=term))
    _func_or_print(result, func)
    if newlines:
        print('\n')

def _make_padded_char(word, padding=5):
    """Create a string format token with padding based on the length
    of the given `word` * `padding`;
    e.g. 'cat' -> {:<3}"""
    return '{:<' + str(len(str(word)) + padding) + '}'

def make_padded_chars(words, seperator=' '):
    """Call `_make_padding_char` on a list of words.
    For example, to create a new format string to pad a list of values.
    (e.g. {:<3} {<:6} {<:9}"""
    fmt_string = ''
    for word in words:
        fmt_string += _make_padded_char(word) + seperator
    return fmt_string

def print_table(rows, formatter=None, striped=True):
    """Print a table that is uniform and aligned."""
    headings = rows[0].keys()
    # Optionally allow custom format functions for each heading
    if formatter:
        headings = map(formatter, headings)
    # This is where it shines - we make a format string template based on
    # each heading token, which can be used for both headings and rows.
    template = make_padded_chars(headings)
    # Print headings first
    print_success(template.format(*headings), prefix=False)
    divider(newline=False)
    num_rows = len(rows)
    for k, row in enumerate(rows):
        values = row.values()
        # Apply the unique values to the template
        print(template.format(*values))
        if striped and k != num_rows - 1:
            divider(atom='-', newline=False)
    divider()

def print_vars(vars, upper=False, convert=False):
    """Single line print with variable and a title, as well
    as some optional kwargs to transform the data."""
    # Allow passing in of single or multiple lists
    if isinstance(vars[0], str) and len(vars) == 2:
        is_single = True
        # Format the list appropriately
        vars = [vars]
    else:
        is_single = False
    for var in vars:
        if len(var) != 2 and not is_single:
            raise TypeError('Need both title and variable.')
        title, data = var
        if isinstance(data, list) and convert:
            if len(data) > 1:
                data = ''.join(map(str, data))
        elif upper:
            title = title.upper()
        print('{}: {}'.format(title, data))

def print_nl(title, pos='top'):
    """Print a title with a newline.
    The newline can be either on bottom (default) or top, if specified."""
    if pos == 'top':
        print('\n{}'.format(title))
    else:
        print('{}\n'.format(title))

def _heading(title, divider, desc=''):
    hyphen = '' if desc == '' else ' - '
    text = ('\n {} {} {}\n' '|{}|\n').format(
        title.upper(), hyphen, desc, divider * 80)
    print_nl(text)

def print_h1(title, desc=''):
    """Print a heading with a very bold underline"""
    _heading(title, '#', desc=desc)

def print_h2(title, desc=''):
    """Print a heading with a bold underline"""
    _heading(title, '=', desc=desc)

def print_h3(title, desc=''):
    """Print a heading with a moderate underline"""
    _heading(title, '-', desc=desc)

def print_h4(title, desc=''):
    """Print a heading with a subdued underline"""
    _heading(title, '.', desc=desc)

def print_simple(words, result, func=ppr, newline=True):
    """Print a heading with data. The content can optionally be formatted by
    a given `func`. No styling is done to the text."""
    print(words)
    _func_or_print(result, func)
    if newline:
        print('\n')

def cmd_title(msg, newlines=True):
    """Print a command type message (e.g [COMMAND]) in red."""
    if newlines:
        print('\n')
    print('{t.red}[{msg}]{t.normal}'.format(msg=msg.upper(), t=term))
    if newlines:
        print('\n')

class Section:
    """Provides a context manager for printing 'sections' of text.
    Prints a top and bottom section to visually separate different blocks
    of code/results."""

    def __init__(self, content):
        self.separator = '=' * 50
        self.content = content

    def prnt(self, prefix, newlines=True):
        nl = '\n' if newlines else ''
        print('{t.cyan}{nl}[{}]: {t.bold} {} {sep} {nl}{t.normal}'.format(
            prefix, self.content, t=term, sep=self.separator, nl=nl))

    def __enter__(self):
        self.prnt('BEGIN')

    def __exit__(self, exception_type, exception_value, traceback):
        self.prnt('END')

if __name__ == '__main__':
    print_info('Testing Info')
    print_warning('Testing Warning')
    print_success('Testing Success')
    print_error('Testing Error')

    cmd_title('DANGER WILL ROBINSON')

    print_h1('Heading 1...')
    print_h2('Heading 2...')
    print_h3('Heading 3...')
    print_h4('Heading 4...')
<commit_msg>Use constant for line width, make context manager always calculate the right line width<commit_after>from blessings import Terminal
from pprint import pprint as ppr

term = Terminal()

# Display utilities
MAX_WIDTH = 80

def _func_or_print(result, func):
    """Private function to either print or call a function on result"""
    if func is not None:
        func(result)
    else:
        print(result)

def _uncase_x(string, seperator):
    """Split by `seperator` and join the string;
    e.g. 'foo_bar' => foo bar """
    return ' '.join(string.split(str(seperator)))

def uncase_period(string):
    return _uncase_x(string, '.')

def uncase_hyphen(string):
    return _uncase_x(string, '-')

def uncase_snake(string):
    return _uncase_x(string, '_')

def uncase_snake_upper(string):
    return uncase_snake(string).upper()

def firstcaps(string):
    string = list(string.lower())
    string[0] = string[0].upper()
    return ''.join(string)

def title_case(string, seperator='_'):
    """Format a string into title case - e.g. 'some_word' => Some Word """
    return ' '.join(map(firstcaps, string.split(seperator)))

def divider(atom='_', newline=True):
    """Print a divider. Optionally override the unit of text to use
    (e.g ---, ...., ###)"""
    # Correct for longer than 1 character atoms, since it will take up
    # more than 80 chars.
    print(atom * (MAX_WIDTH // len(atom)))
    if newline:
        print('\n')

def print_subdued(msg):
    """Print info-type text in dark."""
    vals = '{t.black}{}{t.normal}'
    print(vals.format(msg, t=term))

def print_info(msg, prefix='[INFO]'):
    """Print info-type text in red with a prefix"""
    vals = '{t.blue}{} {}{t.normal}'
    print(vals.format(prefix, msg, t=term))

def print_warning(msg, prefix='[WARN]'):
    """Print warning-type text in red with a prefix"""
    vals = '{t.yellow}{} {}{t.normal}'
    print(vals.format(prefix, msg, t=term))

def print_success(msg, prefix='[YAY]'):
    """Print success-type text in red with a prefix"""
    vals = '{t.green}{} {}{t.normal}'
    print(vals.format(prefix, msg, t=term))

def print_error(msg, prefix='[ERROR]'):
    """Print error-type text in red with a prefix"""
    vals = '{t.red}{} {}{t.normal}'
    print(vals.format(prefix, msg, t=term))

def prnt(title, result, func=None, newlines=False):
    """A more useful default print function for titles and accompanying
    content. Shows stylized title, with content below, and newlines.
    The content can optionally be formatted by a given `func`."""
    if newlines:
        print('\n')
    print('{t.green}{t.underline}{}{t.normal}'.format(title, t=term))
    _func_or_print(result, func)
    if newlines:
        print('\n')

def _make_padded_char(word, padding=5):
    """Create a string format token with padding based on the length
    of the given `word` * `padding`;
    e.g. 'cat' -> {:<3}"""
    return '{:<' + str(len(str(word)) + padding) + '}'

def make_padded_chars(words, seperator=' '):
    """Call `_make_padding_char` on a list of words.
    For example, to create a new format string to pad a list of values.
    (e.g. {:<3} {<:6} {<:9}"""
    fmt_string = ''
    for word in words:
        fmt_string += _make_padded_char(word) + seperator
    return fmt_string

def print_table(rows, formatter=None, striped=True):
    """Print a table that is uniform and aligned."""
    headings = rows[0].keys()
    # Optionally allow custom format functions for each heading
    if formatter:
        headings = map(formatter, headings)
    # This is where it shines - we make a format string template based on
    # each heading token, which can be used for both headings and rows.
    template = make_padded_chars(headings)
    # Print headings first
    print_success(template.format(*headings), prefix=False)
    divider(newline=False)
    num_rows = len(rows)
    for k, row in enumerate(rows):
        values = row.values()
        # Apply the unique values to the template
        print(template.format(*values))
        if striped and k != num_rows - 1:
            divider(atom='-', newline=False)
    divider()

def print_vars(vars, upper=False, convert=False):
    """Single line print with variable and a title, as well
    as some optional kwargs to transform the data."""
    # Allow passing in of single or multiple lists
    if isinstance(vars[0], str) and len(vars) == 2:
        is_single = True
        # Format the list appropriately
        vars = [vars]
    else:
        is_single = False
    for var in vars:
        if len(var) != 2 and not is_single:
            raise TypeError('Need both title and variable.')
        title, data = var
        if isinstance(data, list) and convert:
            if len(data) > 1:
                data = ''.join(map(str, data))
        elif upper:
            title = title.upper()
        print('{}: {}'.format(title, data))

def print_nl(title, pos='top'):
    """Print a title with a newline.
    The newline can be either on bottom (default) or top, if specified."""
    if pos == 'top':
        print('\n{}'.format(title))
    else:
        print('{}\n'.format(title))

def _heading(title, divider, desc=''):
    hyphen = '' if desc == '' else ' - '
    text = ('\n {} {} {}\n' '|{}|\n').format(
        title.upper(), hyphen, desc, divider * MAX_WIDTH)
    print_nl(text)

def print_h1(title, desc=''):
    """Print a heading with a very bold underline"""
    _heading(title, '#', desc=desc)

def print_h2(title, desc=''):
    """Print a heading with a bold underline"""
    _heading(title, '=', desc=desc)

def print_h3(title, desc=''):
    """Print a heading with a moderate underline"""
    _heading(title, '-', desc=desc)

def print_h4(title, desc=''):
    """Print a heading with a subdued underline"""
    _heading(title, '.', desc=desc)

def print_simple(words, result, func=ppr, newline=True):
    """Print a heading with data. The content can optionally be formatted by
    a given `func`. No styling is done to the text."""
    print(words)
    _func_or_print(result, func)
    if newline:
        print('\n')

def cmd_title(msg, newlines=True):
    """Print a command type message (e.g [COMMAND]) in red."""
    if newlines:
        print('\n')
    print('{t.red}[{msg}]{t.normal}'.format(msg=msg.upper(), t=term))
    if newlines:
        print('\n')

class Section:
    """Provides a context manager for printing 'sections' of text.
    Prints a top and bottom section to visually separate different blocks
    of code/results."""

    def __init__(self, content):
        self.content = content

    def prnt(self, prefix, newlines=True):
        nl = '\n' if newlines else ''
        # Make alignment perfectly match the 80 character
        # mark for all display helpers.
        spaces, offset = 4, len(prefix) + len(self.content)
        seperator = '=' * (MAX_WIDTH - offset - spaces)
        print('{t.cyan}{nl}[{}]: {t.bold} {} {sep} {nl}{t.normal}'.format(
            prefix, self.content, t=term, sep=seperator, nl=nl))

    def __enter__(self):
        self.prnt('BEGIN')

    def __exit__(self, exception_type, exception_value, traceback):
        self.prnt('END')

if __name__ == '__main__':
    print_info('Testing Info')
    print_warning('Testing Warning')
    print_success('Testing Success')
    print_error('Testing Error')

    cmd_title('DANGER WILL ROBINSON')

    print_h1('Heading 1...')
    print_h2('Heading 2...')
    print_h3('Heading 3...')
    print_h4('Heading 4...')
<|endoftext|>


<commit_before>from luigi import Task, WrapperTask, Parameter, IntParameter, ListParameter
from tasks.us.census.tiger import GeoNamesTable, ShorelineClip, SUMLEVELS
from tasks.base_tasks import TempTableTask
from tasks.meta import current_session
from tasks.targets import ConstraintExistsTarget
from lib.logger import get_logger

LOGGER = get_logger(__name__)

def _union_query(tables, output):
    unions = ['SELECT * FROM {table}'.format(table=table.qualified_tablename)
              for table in tables]
    return 'CREATE TABLE {output} AS {unions}'.format(
        output=output,
        unions=' UNION ALL '.join(unions))

class USHierarchy(Task):
    year = IntParameter()

    def requires(self):
        levels = self._levels()
        LOGGER.info('Levels: {}'.format(levels))
        return {
            'info': USHierarchyInfoUnion(year=self.year, levels=levels),
            'rel': USHierarchyChildParentsUnion(year=self.year, levels=levels)
        }

    def _levels(self):
        sorted_level_infos = reversed(sorted(SUMLEVELS.items(),
                                             key=lambda level_info:
                                             level_info[1][
                                                 'weight']))
        return [level_info[0] for level_info in sorted_level_infos]

    def run(self):
        session = current_session()
        input_ = self.input()
        session.execute('ALTER TABLE {rel_table} ADD '
                        'CONSTRAINT ushierarchy_fk_child '
                        'FOREIGN KEY (child_id, child_level) '
                        'REFERENCES {info_table} (geoid, level) '.format(
            rel_table=input_['rel'].qualified_tablename,
            info_table=input_['info'].qualified_tablename))
        session.execute('ALTER TABLE {rel_table} ADD '
                        'CONSTRAINT ushierarchy_fk_parent '
                        'FOREIGN KEY (parent_id, parent_level) '
                        'REFERENCES {info_table} (geoid, level) '.format(
            rel_table=input_['rel'].qualified_tablename,
            info_table=input_['info'].qualified_tablename))
        session.commit()

    def output(self):
        table = self.input()['info']
        return ConstraintExistsTarget(table.schema, table.tablename,
                                      'ushierarchy_fk_parent')

class _YearLevelsTask:
    year = IntParameter()
    levels = ListParameter(significant=False)

class USHierarchyInfoUnion(TempTableTask, _YearLevelsTask):

    def requires(self):
        return [USLevelInfo(year=self.year, geography=level)
                for level in self.levels]

    def run(self):
        session = current_session()
        tablename = self.output().qualified_tablename
        session.execute(_union_query(self.input(), tablename))
        alter_sql = 'ALTER TABLE {tablename} ADD PRIMARY KEY (geoid, level)'
        session.execute(alter_sql.format(tablename=tablename))
        session.commit()

class USHierarchyChildParentsUnion(TempTableTask, _YearLevelsTask):

    def requires(self):
        child_parents = self._child_parents()
        LOGGER.info('Child-parents: {}'.format(child_parents))
        return {
            'hierarchy': [USLevelInclusionHierarchy(year=self.year,
                                                    current_geography=
                                                    child_parent[0],
                                                    parent_geographies=
                                                    child_parent[1])
                          for child_parent in child_parents],
            'weight': [USLevelHierarchyWeights(year=self.year,
                                               current_geography=child_parent[
                                                   0],
                                               parent_geographies=child_parent[
                                                   1])
                       for child_parent in child_parents]
        }

    def _child_parents(self):
        child_parents = []
        previous = None
        for idx, level in enumerate(self.levels):
            if previous:
                parents = self.levels[idx:]
                if parents:
                    child_parents.append([previous, parents])
            previous = level
        return child_parents

    def run(self):
        session = current_session()
        tablename = self.output().qualified_tablename
        session.execute(_union_query(self.input()['hierarchy'], tablename))
        delete_sql = 'DELETE FROM {tablename} WHERE parent_id IS NULL'
        session.execute(delete_sql.format(tablename=tablename))
        alter_sql = 'ALTER TABLE {tablename} ADD PRIMARY KEY ' \
                    '(child_id, child_level, parent_id, parent_level)'
        session.execute(alter_sql.format(tablename=tablename))
        session.commit()

GEOGRAPHIES_ABBREVIATIONS = {
    'school_district_secondary': 'sds',
    'school_district_unified': 'sdu',
    'school_district_elementary': 'sde'
}

def abbr_tablename(target, geographies, year):
    abbrs = [GEOGRAPHIES_ABBREVIATIONS.get(geo, geo) for geo in geographies]

    if [x for x in zip(abbrs, geographies) if x[0] != x[1]]:
        splits = target.split('_')
        target = '_'.join([splits[0], '_'.join(abbrs), str(year), splits[-1]])

    return target

class USLevelInclusionHierarchy(WrapperTask):
    year = IntParameter()
    current_geography = Parameter()
    parent_geographies = ListParameter()

    def requires(self):
        return {
            'level': USLevelHierarchy(year=self.year,
                                      current_geography=self.current_geography,
                                      parent_geographies=self.parent_geographies)
        }

    def output(self):
        return self.input()['level']

class USLevelHierarchyWeights(Task):
    year = IntParameter()
    current_geography = Parameter()
    parent_geographies = ListParameter()

    UNWEIGHTED_CHILD_SQL = """
        SELECT DISTINCT child_id, child_level
        FROM {table}
        WHERE weight = 1
        AND parent_id IS NOT NULL
        GROUP BY child_id, child_level
        HAVING count(1) > 1
    """

    def requires(self):
        return {
            'level': USLevelHierarchy(year=self.year,
                                      current_geography=self.current_geography,
                                      parent_geographies=self.parent_geographies),
            'current_geom': ShorelineClip(year=self.year,
                                          geography=self.current_geography),
            'parent_geoms': [ShorelineClip(year=self.year,
                                           geography=parent_geography) for
                             parent_geography in self.parent_geographies]
        }

    def run(self):
        session = current_session()
        sql = '''
            UPDATE {table}
            SET weight = ST_Area(
                ST_Intersection(cgt.the_geom, pgt.the_geom), False)
            FROM
                observatory.{current_geom_table} cgt,
                observatory.{parent_geom_table} pgt
            WHERE cgt.geoid = {table}.child_id
              AND pgt.geoid = {table}.parent_id
              AND (child_id, child_level) IN (
              {unweighted_child_sql}
        )
        '''
        table = self.input()['level'].qualified_tablename
        for parent_geom in self.input()['parent_geoms']:
            session.execute(
                sql.format(
                    table=table,
                    current_geom_table=self.input()['current_geom'].get(
                        session).tablename,
                    parent_geom_table=parent_geom.get(
                        session).tablename,
                    unweighted_child_sql=self.UNWEIGHTED_CHILD_SQL.format(
                        table=table)
                )
            )
        session.commit()

    def complete(self):
        sql = self.UNWEIGHTED_CHILD_SQL.format(
            table=self.input()['level'].qualified_tablename)
        try:
            return len(current_session().execute(sql).fetchall()) == 0
        except Exception as e:
            # Table doesn't exist yet
            LOGGER.error("ERROR running {}: {}".format(sql, e))
            return False

    def output(self):
        return self.input()['level']

class USLevelHierarchy(TempTableTask):
    year = IntParameter()
    current_geography = Parameter()
    parent_geographies = ListParameter()

    def requires(self):
        return {
            'current_info': USLevelInfo(year=self.year,
                                        geography=self.current_geography),
            'current_geom': ShorelineClip(year=self.year,
                                          geography=self.current_geography),
            'parents_infos': [USLevelInfo(year=self.year,
                                          geography=parent_geography) for
                              parent_geography in self.parent_geographies],
            'parents_geoms': [ShorelineClip(year=self.year,
                                            geography=parent_geography) for
                              parent_geography in self.parent_geographies]
        }

    def target_tablename(self):
        return abbr_tablename(super(USLevelHierarchy, self).target_tablename(),
                              [self.current_geography, 'parents'],
                              self.year)

    def run(self):
        session = current_session()
        input_ = self.input()
        current_info_tablename = input_['current_info'].qualified_tablename
        current_geom_table = input_['current_geom'].get(session)
        parent_info_tablename = input_['parents_infos'][0].qualified_tablename
        parent_geom_table = input_['parents_geoms'][0].get(session)

        create_table_sql = '''
        CREATE TABLE {output_table} AS
        {child_parent_sql}
        '''

        # First creation will link child with direct parents and leave nulls
        # for those that don't have.
        session.execute(create_table_sql.format(
            output_table=self.output().qualified_tablename,
            child_parent_sql=self._CHILD_PARENT_SQL.format(
                current_info_table=current_info_tablename,
                current_geom_table=current_geom_table.tablename,
                parent_info_table=parent_info_tablename,
                parent_geom_table=parent_geom_table.tablename)
        ))

        inputs = list(zip(input_['parents_infos'], input_['parents_geoms']))
        for parent_info_geom in inputs[1:]:
            # For those without parents, insert the next ones
            parent_info_tablename = parent_info_geom[0].qualified_tablename
            parent_geom_table = parent_info_geom[1].get(session)
            fill_parents_sql = '''
                INSERT INTO {output_table}
                (child_id, child_level, parent_id, parent_level, weight)
                {child_parent_sql}
                INNER JOIN {output_table} ot ON ot.child_id = cit.geoid
                                            AND ot.child_level = cit.level
                WHERE ot.parent_id IS NULL
            '''
            session.execute(fill_parents_sql.format(
                output_table=self.output().qualified_tablename,
                child_parent_sql=self._CHILD_PARENT_SQL.format(
                    current_info_table=current_info_tablename,
                    current_geom_table=current_geom_table.tablename,
                    parent_info_table=parent_info_tablename,
                    parent_geom_table=parent_geom_table.tablename)))

            # ... and then, delete the rows with null parents for those
            # child that have any parent
            delete_non_orphans = '''
                DELETE FROM {output_table} ot
                WHERE ot.parent_id IS NULL
                AND (child_id, child_level) IN (
                    SELECT child_id, child_level
                    FROM {output_table}
                    WHERE parent_id IS NOT NULL
            )
            '''
            session.execute(delete_non_orphans.format(
                output_table=self.output().qualified_tablename))

        session.commit()

    _CHILD_PARENT_SQL = '''
        SELECT
            cit.geoid AS child_id,
            cit.level AS child_level,
            pgt.geoid AS parent_id,
            pit.level AS parent_level,
            1.0::FLOAT AS weight
        FROM {current_info_table} cit
        INNER JOIN observatory.{current_geom_table} cgt ON cit.geoid = cgt.geoid
        LEFT JOIN observatory.{parent_geom_table} pgt
            ON ST_Within(ST_PointOnSurface(cgt.the_geom), pgt.the_geom)
        LEFT JOIN {parent_info_table} pit ON pgt.geoid = pit.geoid
    '''

class _YearGeographyTask:
    year = IntParameter()
    geography = Parameter()

class USLevelInfo(WrapperTask, _YearGeographyTask):
    GEOGRAPHIES_WITHOUT_GEONAMES = ['zcta5']

    def requires(self):
        if self.geography in self.GEOGRAPHIES_WITHOUT_GEONAMES:
            return USLevelInfoFromShorelineClip(year=self.year,
                                                geography=self.geography)
        else:
            return USLevelInfoFromGeoNames(year=self.year,
                                           geography=self.geography)

    def output(self):
        return self.input()

class USLevelInfoFromShorelineClip(TempTableTask, _YearGeographyTask):

    def requires(self):
        return ShorelineClip(year=self.year, geography=self.geography)

    def run(self):
        session = current_session()

        names_table = self.input().get(session).tablename
        output_table = self.output().qualified_tablename
        output_tablename = self.output().tablename

        sql = '''
            CREATE TABLE {output_table} AS
            SELECT n.geoid geoid, '{geography}' as level, n.geoid as name, {year} as year
            FROM observatory.{names_table} n
        '''
        session.execute(
            sql.format(output_table=output_table,
                       geography=self.geography,
                       year=self.year,
                       names_table=names_table))
        session.execute('''
            CREATE INDEX {output_tablename}_idx ON {output_table} (geoid)
        '''.format(output_table=output_table,
                   output_tablename=output_tablename))
        session.commit()

class USLevelInfoFromGeoNames(TempTableTask, _YearGeographyTask):

    def requires(self):
        return GeoNamesTable(year=self.year, geography=self.geography)

    def target_tablename(self):
        return abbr_tablename(
            super(USLevelInfoFromGeoNames, self).target_tablename(),
            [self.geography], self.year)

    def run(self):
        session = current_session()

        names_table = self.input().get(session).tablename
        output_table = self.output().qualified_tablename
        output_tablename = self.output().tablename

        session.execute(
            '''
                CREATE TABLE {output_table} AS
                SELECT n.geoidsc geoid, '{geography}' as level, n.geoname, {year} as year
                FROM observatory.{names_table} n
            '''.format(output_table=output_table,
                       geography=self.geography,
                       year=self.year,
                       names_table=names_table))
        session.execute('''
            CREATE INDEX {output_tablename}_idx ON {output_table} (geoid)
        '''.format(output_table=output_table,
                   output_tablename=output_tablename))
        session.commit()
<commit_msg>Fix duplicates at levelhierarchy<commit_after>from luigi import Task, WrapperTask, Parameter, IntParameter, ListParameter
from tasks.us.census.tiger import GeoNamesTable, ShorelineClip, SUMLEVELS
from tasks.base_tasks import TempTableTask
from tasks.meta import current_session
from tasks.targets import ConstraintExistsTarget
from lib.logger import get_logger

LOGGER = get_logger(__name__)

def _union_query(tables, output):
    unions = ['SELECT * FROM {table}'.format(table=table.qualified_tablename)
              for table in tables]
    return 'CREATE TABLE {output} AS {unions}'.format(
        output=output,
        unions=' UNION ALL '.join(unions))

class USHierarchy(Task):
    year = IntParameter()

    def requires(self):
        levels = self._levels()
        LOGGER.info('Levels: {}'.format(levels))
        return {
            'info': USHierarchyInfoUnion(year=self.year, levels=levels),
            'rel': USHierarchyChildParentsUnion(year=self.year, levels=levels)
        }

    def _levels(self):
        sorted_level_infos = reversed(sorted(SUMLEVELS.items(),
                                             key=lambda level_info:
                                             level_info[1][
                                                 'weight']))
        return [level_info[0] for level_info in sorted_level_infos]

    def run(self):
        session = current_session()
        input_ = self.input()
        session.execute('ALTER TABLE {rel_table} ADD '
                        'CONSTRAINT ushierarchy_fk_child '
                        'FOREIGN KEY (child_id, child_level) '
                        'REFERENCES {info_table} (geoid, level) '.format(
            rel_table=input_['rel'].qualified_tablename,
            info_table=input_['info'].qualified_tablename))
        session.execute('ALTER TABLE {rel_table} ADD '
                        'CONSTRAINT ushierarchy_fk_parent '
                        'FOREIGN KEY (parent_id, parent_level) '
                        'REFERENCES {info_table} (geoid, level) '.format(
            rel_table=input_['rel'].qualified_tablename,
            info_table=input_['info'].qualified_tablename))
        session.commit()

    def output(self):
        table = self.input()['info']
        return ConstraintExistsTarget(table.schema, table.tablename,
                                      'ushierarchy_fk_parent')

class _YearLevelsTask:
    year = IntParameter()
    levels = ListParameter(significant=False)

class USHierarchyInfoUnion(TempTableTask, _YearLevelsTask):

    def requires(self):
        return [USLevelInfo(year=self.year, geography=level)
                for level in self.levels]

    def run(self):
        session = current_session()
        tablename = self.output().qualified_tablename
        session.execute(_union_query(self.input(), tablename))
        alter_sql = 'ALTER TABLE {tablename} ADD PRIMARY KEY (geoid, level)'
        session.execute(alter_sql.format(tablename=tablename))
        session.commit()

class USHierarchyChildParentsUnion(TempTableTask, _YearLevelsTask):

    def requires(self):
        child_parents = self._child_parents()
        LOGGER.info('Child-parents: {}'.format(child_parents))
        return {
            'hierarchy': [USLevelInclusionHierarchy(year=self.year,
                                                    current_geography=
                                                    child_parent[0],
                                                    parent_geographies=
                                                    child_parent[1])
                          for child_parent in child_parents],
            'weight': [USLevelHierarchyWeights(year=self.year,
                                               current_geography=child_parent[
                                                   0],
                                               parent_geographies=child_parent[
                                                   1])
                       for child_parent in child_parents]
        }

    def _child_parents(self):
        child_parents = []
        previous = None
        for idx, level in enumerate(self.levels):
            if previous:
                parents = self.levels[idx:]
                if parents:
                    child_parents.append([previous, parents])
            previous = level
        return child_parents

    def run(self):
        session = current_session()
        tablename = self.output().qualified_tablename
        session.execute(_union_query(self.input()['hierarchy'], tablename))
        delete_sql = 'DELETE FROM {tablename} WHERE parent_id IS NULL'
        session.execute(delete_sql.format(tablename=tablename))
        alter_sql = 'ALTER TABLE {tablename} ADD PRIMARY KEY ' \
                    '(child_id, child_level, parent_id, parent_level)'
        session.execute(alter_sql.format(tablename=tablename))
        session.commit()

GEOGRAPHIES_ABBREVIATIONS = {
    'school_district_secondary': 'sds',
    'school_district_unified': 'sdu',
    'school_district_elementary': 'sde'
}

def abbr_tablename(target, geographies, year):
    abbrs = [GEOGRAPHIES_ABBREVIATIONS.get(geo, geo) for geo in geographies]

    if [x for x in zip(abbrs, geographies) if x[0] != x[1]]:
        splits = target.split('_')
        target = '_'.join([splits[0], '_'.join(abbrs), str(year), splits[-1]])

    return target

class USLevelInclusionHierarchy(WrapperTask):
    year = IntParameter()
    current_geography = Parameter()
    parent_geographies = ListParameter()

    def requires(self):
        return {
            'level': USLevelHierarchy(year=self.year,
                                      current_geography=self.current_geography,
                                      parent_geographies=self.parent_geographies)
        }

    def output(self):
        return self.input()['level']

class USLevelHierarchyWeights(Task):
    year = IntParameter()
    current_geography = Parameter()
    parent_geographies = ListParameter()

    UNWEIGHTED_CHILD_SQL = """
        SELECT DISTINCT child_id, child_level
        FROM {table}
        WHERE weight = 1
        AND parent_id IS NOT NULL
        GROUP BY child_id, child_level
        HAVING count(1) > 1
    """

    def requires(self):
        return {
            'level': USLevelHierarchy(year=self.year,
                                      current_geography=self.current_geography,
                                      parent_geographies=self.parent_geographies),
            'current_geom': ShorelineClip(year=self.year,
                                          geography=self.current_geography),
            'parent_geoms': [ShorelineClip(year=self.year,
                                           geography=parent_geography) for
                             parent_geography in self.parent_geographies]
        }

    def run(self):
        session = current_session()
        sql = '''
            UPDATE {table}
            SET weight = ST_Area(
                ST_Intersection(cgt.the_geom, pgt.the_geom), False)
            FROM
                observatory.{current_geom_table} cgt,
                observatory.{parent_geom_table} pgt
            WHERE cgt.geoid = {table}.child_id
              AND pgt.geoid = {table}.parent_id
              AND (child_id, child_level) IN (
              {unweighted_child_sql}
        )
        '''
        table = self.input()['level'].qualified_tablename
        for parent_geom in self.input()['parent_geoms']:
            session.execute(
                sql.format(
                    table=table,
                    current_geom_table=self.input()['current_geom'].get(
                        session).tablename,
                    parent_geom_table=parent_geom.get(
                        session).tablename,
                    unweighted_child_sql=self.UNWEIGHTED_CHILD_SQL.format(
                        table=table)
                )
            )
        session.commit()

    def complete(self):
        sql = self.UNWEIGHTED_CHILD_SQL.format(
            table=self.input()['level'].qualified_tablename)
        try:
            return len(current_session().execute(sql).fetchall()) == 0
        except Exception as e:
            # Table doesn't exist yet
            LOGGER.error("ERROR running {}: {}".format(sql, e))
            return False

    def output(self):
        return self.input()['level']

class USLevelHierarchy(TempTableTask):
    year = IntParameter()
    current_geography = Parameter()
    parent_geographies = ListParameter()

    def requires(self):
        return {
            'current_info': USLevelInfo(year=self.year,
                                        geography=self.current_geography),
            'current_geom': ShorelineClip(year=self.year,
                                          geography=self.current_geography),
            'parents_infos': [USLevelInfo(year=self.year,
                                          geography=parent_geography) for
                              parent_geography in self.parent_geographies],
            'parents_geoms': [ShorelineClip(year=self.year,
                                            geography=parent_geography) for
                              parent_geography in self.parent_geographies]
        }

    def target_tablename(self):
        return abbr_tablename(super(USLevelHierarchy, self).target_tablename(),
                              [self.current_geography, 'parents'],
                              self.year)

    def run(self):
        session = current_session()
        input_ = self.input()
        current_info_tablename = input_['current_info'].qualified_tablename
        current_geom_table = input_['current_geom'].get(session)
        parent_info_tablename = input_['parents_infos'][0].qualified_tablename
        parent_geom_table = input_['parents_geoms'][0].get(session)

        create_table_sql = '''
        CREATE TABLE {output_table} AS
        {child_parent_sql}
        '''

        # First creation will link child with direct parents and leave nulls
        # for those that don't have.
        session.execute(create_table_sql.format(
            output_table=self.output().qualified_tablename,
            child_parent_sql=self._CHILD_PARENT_SQL.format(
                current_info_table=current_info_tablename,
                current_geom_table=current_geom_table.tablename,
                parent_info_table=parent_info_tablename,
                parent_geom_table=parent_geom_table.tablename,
                inner_or_left='LEFT')
        ))

        inputs = list(zip(input_['parents_infos'], input_['parents_geoms']))
        for parent_info_geom in inputs[1:]:
            # For those without parents, insert the next ones
            parent_info_tablename = parent_info_geom[0].qualified_tablename
            parent_geom_table = parent_info_geom[1].get(session)
            fill_parents_sql = '''
                INSERT INTO {output_table}
                (child_id, child_level, parent_id, parent_level, weight)
                {child_parent_sql}
                INNER JOIN {output_table} ot ON ot.child_id = cit.geoid
                                            AND ot.child_level = cit.level
                WHERE ot.parent_id IS NULL
            '''
            session.execute(fill_parents_sql.format(
                output_table=self.output().qualified_tablename,
                child_parent_sql=self._CHILD_PARENT_SQL.format(
                    current_info_table=current_info_tablename,
                    current_geom_table=current_geom_table.tablename,
                    parent_info_table=parent_info_tablename,
                    parent_geom_table=parent_geom_table.tablename,
                    inner_or_left='INNER')))

            # ... and then, delete the rows with null parents for those
            # child that have any parent
            delete_non_orphans = '''
                DELETE FROM {output_table} ot
                WHERE ot.parent_id IS NULL
                AND (child_id, child_level) IN (
                    SELECT child_id, child_level
                    FROM {output_table}
                    WHERE parent_id IS NOT NULL
            )
            '''
            session.execute(delete_non_orphans.format(
                output_table=self.output().qualified_tablename))

        session.commit()

    _CHILD_PARENT_SQL = '''
        SELECT
            cit.geoid AS child_id,
            cit.level AS child_level,
            pgt.geoid AS parent_id,
            pit.level AS parent_level,
            1.0::FLOAT AS weight
        FROM {current_info_table} cit
        INNER JOIN observatory.{current_geom_table} cgt ON cit.geoid = cgt.geoid
        {inner_or_left} JOIN observatory.{parent_geom_table} pgt
            ON ST_Within(ST_PointOnSurface(cgt.the_geom), pgt.the_geom)
        {inner_or_left} JOIN {parent_info_table} pit ON pgt.geoid = pit.geoid
    '''

class _YearGeographyTask:
    year = IntParameter()
    geography = Parameter()

class USLevelInfo(WrapperTask, _YearGeographyTask):
    GEOGRAPHIES_WITHOUT_GEONAMES = ['zcta5']

    def requires(self):
        if self.geography in self.GEOGRAPHIES_WITHOUT_GEONAMES:
            return USLevelInfoFromShorelineClip(year=self.year,
                                                geography=self.geography)
        else:
            return USLevelInfoFromGeoNames(year=self.year,
                                           geography=self.geography)

    def output(self):
        return self.input()

class USLevelInfoFromShorelineClip(TempTableTask, _YearGeographyTask):

    def requires(self):
        return ShorelineClip(year=self.year, geography=self.geography)

    def run(self):
        session = current_session()

        names_table = self.input().get(session).tablename
        output_table = self.output().qualified_tablename
        output_tablename = self.output().tablename

        sql = '''
            CREATE TABLE {output_table} AS
            SELECT n.geoid geoid, '{geography}' as level, n.geoid as name, {year} as year
            FROM observatory.{names_table} n
        '''
        session.execute(
            sql.format(output_table=output_table,
                       geography=self.geography,
                       year=self.year,
                       names_table=names_table))
        session.execute('''
            CREATE INDEX {output_tablename}_idx ON {output_table} (geoid)
        '''.format(output_table=output_table,
                   output_tablename=output_tablename))
        session.commit()

class USLevelInfoFromGeoNames(TempTableTask, _YearGeographyTask):

    def requires(self):
        return GeoNamesTable(year=self.year, geography=self.geography)

    def target_tablename(self):
        return abbr_tablename(
            super(USLevelInfoFromGeoNames, self).target_tablename(),
            [self.geography], self.year)

    def run(self):
        session = current_session()

        names_table = self.input().get(session).tablename
        output_table = self.output().qualified_tablename
        output_tablename = self.output().tablename

        session.execute(
            '''
                CREATE TABLE {output_table} AS
                SELECT n.geoidsc geoid, '{geography}' as level, n.geoname, {year} as year
                FROM observatory.{names_table} n
            '''.format(output_table=output_table,
                       geography=self.geography,
                       year=self.year,
                       names_table=names_table))
        session.execute('''
            CREATE INDEX {output_tablename}_idx ON {output_table} (geoid)
        '''.format(output_table=output_table,
                   output_tablename=output_tablename))
        session.commit()
<|endoftext|>


<commit_before>#!/usr/bin/env python
def swap(data_list, i1, i2):
    temp = data_list[i1]
    data_list[i1] = data_list[i2]
    data_list[i2] = temp

def bubble_sort(data_list):
    flag = True
    for i in range( 1, len(data_list)):
        flag = True
        for j in range( 1, len(data_list)):
            if data_list[j-1] > data_list[j]:
                flag = False
                swap(data_list, j, j-1)
                print data_list

        if flag:
            break

def selection_sort(data_list):
    for i in range( len(data_list)-1):
        for j in range( i+1, len(data_list)):
            if data_list[j] < data_list[i]:
                swap(data_list, i, j)
                print data_list

def insertion_sort(data_list):
    for i in range( len(data_list) ):
        temp = data_list[i]
        k = i
        while data_list[k-1] > temp and k > 0:
            data_list[k] = data_list[k-1]
            k -= 1
        data_list[k] = temp
        print data_list

def merge_sort(data_list, low, high):
    if low < high:
        middle = (low + high) / 2
        merge_sort(data_list, low, middle)
        merge_sort(data_list, middle+1, high)
        merge(data_list, low, middle, high)

def merge(data_list, low, middle, high):
    helper = [None] * len(data_list)
    for i in range(low, high+1):
        helper[i] = data_list[i]

    helper_left = low
    helper_right = middle + 1
    current = low

    while helper_left <= middle and helper_right <= high:
        if helper[helper_left] <= helper[helper_right]:
            data_list[current] = helper[helper_left]
            helper_left += 1
        else:
            data_list[current] = helper[helper_right]
            helper_right += 1
        current += 1

    remaining = middle - helper_left
    for i in range(remaining+1):
        data_list[current+i] = helper[helper_left+i]

# These are all for quick sort
def quick_sort(data_list):
    pass

def partition():
    pass

#def swap():
#   pass

if __name__ == '__main__':
    import random
    data_list = []
    while True:
        rand = random.randint(0,20)
        if rand not in data_list:
            data_list.append(rand)
            if len(data_list) == 21:
                break
    print data_list
    #selection_sort(data_list)
    #bubble_sort(data_list)
    #insertion_sort(data_list)
    merge_sort(data_list, 0, len(data_list)-1)
    print data_list
<commit_msg>Done with quicksort<commit_after>#!/usr/bin/env python

def swap(data_list, i1, i2):
    temp = data_list[i1]
    data_list[i1] = data_list[i2]
    data_list[i2] = temp

def bubble_sort(data_list):
    flag = True
    for i in range( 1, len(data_list)):
        flag = True
        for j in range( 1, len(data_list)):
            if data_list[j-1] > data_list[j]:
                flag = False
                swap(data_list, j, j-1)
                print data_list

        if flag:
            break

def selection_sort(data_list):
    for i in range( len(data_list)-1):
        for j in range( i+1, len(data_list)):
            if data_list[j] < data_list[i]:
                swap(data_list, i, j)
                print data_list

def insertion_sort(data_list):
    for i in range( len(data_list) ):
        temp = data_list[i]
        k = i
        while data_list[k-1] > temp and k > 0:
            data_list[k] = data_list[k-1]
            k -= 1
        data_list[k] = temp
        print data_list

def merge_sort(data_list, low, high):
    if low < high:
        middle = (low + high) / 2
        merge_sort(data_list, low, middle)
        merge_sort(data_list, middle+1, high)
        merge(data_list, low, middle, high)

def merge(data_list, low, middle, high):
    helper = [None] * len(data_list)
    for i in range(low, high+1):
        helper[i] = data_list[i]

    helper_left = low
    helper_right = middle + 1
    current = low

    while helper_left <= middle and helper_right <= high:
        if helper[helper_left] <= helper[helper_right]:
            data_list[current] = helper[helper_left]
            helper_left += 1
        else:
            data_list[current] = helper[helper_right]
            helper_right += 1
        current += 1

    remaining = middle - helper_left
    for i in range(remaining+1):
        data_list[current+i] = helper[helper_left+i]

# These are all for quick sort
def quick_sort(data_list, left, right):
    index = partition(data_list, left, right)

    if left < index - 1:
        quick_sort(data_list, left, index - 1)

    if index < right:
        quick_sort(data_list, index, right)

def partition(data_list, left, right):
    pivot = data_list[ (left + right ) / 2]

    while left < right:
        while data_list[left] < pivot:
            left += 1
        while data_list[right] > pivot:
            right -= 1
        if left <= right:
            swap(data_list, left, right)
            left += 1
            right -= 1

    return left

if __name__ == '__main__':
    import random
    data_list = []
    while True:
        rand = random.randint(0,20)
        if rand not in data_list:
            data_list.append(rand)
            if len(data_list) == 21:
                break
    print data_list
    #selection_sort(data_list)
    #bubble_sort(data_list)
    #insertion_sort(data_list)
    #merge_sort(data_list, 0, len(data_list)-1)
    quick_sort(data_list, 0, len(data_list)-1)
    print data_list
<|endoftext|>