<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at http://mozilla.org/MPL/2.0/.
"""
Django settings for atmo project.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""
from datetime import timedelta
import os
import raven

import dj_database_url
from django.core.urlresolvers import reverse_lazy
from decouple import Csv, config
import atmo

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = config('SECRET_KEY')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = config('DEBUG', cast=bool)

ALLOWED_HOSTS = config('ALLOWED_HOSTS', cast=Csv())

SITE_ID = 1

# The URL under which this instance is running
SITE_URL = config('SITE_URL', default='http://localhost:8000')

# Whether or not this runs in Heroku
IS_HEROKU = 'HEROKU_APP_NAME' in os.environ

# Application definition

INSTALLED_APPS = [
    # Project specific apps
    'atmo',
    'atmo.clusters',
    'atmo.jobs',
    'atmo.users',

    # Third party apps
    'whitenoise.runserver_nostatic',
    'django_rq',
    'allauth',
    'allauth.account',
    'allauth.socialaccount',

    # Django apps
    'django.contrib.sites',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE_CLASSES = (
    'django.middleware.security.SecurityMiddleware',
    'atmo.middleware.NewRelicPapertrailMiddleware',
    'whitenoise.middleware.WhiteNoiseMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'session_csrf.CsrfMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'csp.middleware.CSPMiddleware',
)

ROOT_URLCONF = 'atmo.urls'

WSGI_APPLICATION = 'atmo.wsgi.application'

# AWS configuration
AWS_CONFIG = {
    # AWS EC2 configuration
    'AWS_REGION': 'us-west-2',

    # EMR configuration
    # Master and slave instance types should be the same as the telemetry
    # setup bootstrap action depends on it to autotune the cluster.
    'MASTER_INSTANCE_TYPE': 'c3.4xlarge',
    'WORKER_INSTANCE_TYPE': 'c3.4xlarge',
    'USE_SPOT_INSTANCES': config('USE_SPOT_INSTANCES', default=True, cast=bool),
    'CORE_SPOT_BID': config('CORE_SPOT_BID', default='0.84'),
    # available EMR releases, to be used as choices for Spark jobs and clusters
    # forms. Please keep the latest (newest) as the first item
    'EMR_RELEASES': (
        '5.0.0',
        '4.5.0',
    ),
    'SPARK_INSTANCE_PROFILE': 'telemetry-spark-cloudformation-'
                              'TelemetrySparkInstanceProfile-1SATUBVEXG7E3',
    'SPARK_EMR_BUCKET': 'telemetry-spark-emr-2',
    'INSTANCE_APP_TAG': 'telemetry-analysis-worker-instance',
    'EMAIL_SOURCE': 'telemetry-alerts@mozilla.com',
    'MAX_CLUSTER_SIZE': 30,

    # Tags for accounting purposes
    'ACCOUNTING_APP_TAG': 'telemetry-analysis',
    'ACCOUNTING_TYPE_TAG': 'worker',

    # Buckets for storing S3 data
    'CODE_BUCKET': 'telemetry-analysis-code-2',
    'PUBLIC_DATA_BUCKET': 'telemetry-public-analysis-2',
    'PRIVATE_DATA_BUCKET': 'telemetry-private-analysis-2',
}

for aws_cred in ('AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_DEFAULT_REGION'):
    if aws_cred not in os.environ:
        os.environ[aws_cred] = config(aws_cred, default='')

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases
DATABASES = {
    'default': config(
        'DATABASE_URL',
        cast=dj_database_url.parse
    )
}
# require encrypted connections to Postgres
if IS_HEROKU:
    DATABASES['default'].setdefault('OPTIONS', {})['sslmode'] = 'require'

REDIS_URL = config('REDIS_URL')

RQ_QUEUES = {
    'default': {
        'URL': REDIS_URL,
        'DEFAULT_TIMEOUT': 600,
    }
}

CACHES = {
    'default': {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': REDIS_URL,
    },
    'forms': {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': REDIS_URL + '?db=1',  # use a different db for form files
        'TIMEOUT': 60 * 60,  # an hour ought to be enough to edit a form
    },
}

# Add the django-allauth authentication backend.
AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
)

LOGIN_URL = reverse_lazy('account_login')
LOGOUT_URL = reverse_lazy('account_logout')
LOGIN_REDIRECT_URL = reverse_lazy('dashboard')

# django-allauth configuration
ACCOUNT_LOGOUT_REDIRECT_URL = LOGIN_REDIRECT_URL
ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = 7
ACCOUNT_EMAIL_SUBJECT_PREFIX = '[Telemetry Analysis Service] '
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = 'optional'
ACCOUNT_LOGOUT_ON_GET = True
ACCOUNT_ADAPTER = 'atmo.users.adapters.AtmoAccountAdapter'
ACCOUNT_USERNAME_REQUIRED = False

SOCIALACCOUNT_ADAPTER = 'atmo.users.adapters.AtmoSocialAccountAdapter'
SOCIALACCOUNT_EMAIL_VERIFICATION = 'none'  # no extra verification needed
SOCIALACCOUNT_QUERY_EMAIL = True  # needed by the Google provider

SOCIALACCOUNT_PROVIDERS = {
    'google': {
        'HOSTED_DOMAIN': 'mozilla.com',
        'AUTH_PARAMS': {
            'prompt': 'select_account',
        }
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/
LANGUAGE_CODE = config('LANGUAGE_CODE', default='en-us')
TIME_ZONE = config('TIME_ZONE', default='UTC')
USE_I18N = False
USE_L10N = False
USE_TZ = config('USE_TZ', default=True, cast=bool)
DATETIME_FORMAT = 'Y-m-d H:i'  # simplified ISO format since we assume UTC

STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'
STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

MEDIA_ROOT = config('MEDIA_ROOT', default=os.path.join(BASE_DIR, 'media'))
MEDIA_URL = config('MEDIA_URL', '/media/')

SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
SESSION_CACHE_ALIAS = 'default'

SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
X_FRAME_OPTIONS = 'DENY'

if SITE_URL.startswith('https://'):
    ACCOUNT_DEFAULT_HTTP_PROTOCOL = 'https'
    SECURE_SSL_REDIRECT = True
    SECURE_HSTS_SECONDS = int(timedelta(days=365).total_seconds())
    # Mark session and CSRF cookies as being HTTPS-only.
    CSRF_COOKIE_SECURE = True
    SESSION_COOKIE_SECURE = True

SILENCED_SYSTEM_CHECKS = [
    'security.W003',  # We're using django-session-csrf
    # We can't set SECURE_HSTS_INCLUDE_SUBDOMAINS since this runs under a
    # mozilla.org subdomain
    'security.W005',
    'security.W009',  # we know the SECRET_KEY is strong
]

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'OPTIONS': {
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.template.context_processors.request',
                'django.contrib.messages.context_processors.messages',
                'session_csrf.context_processor',
                'atmo.context_processors.settings',
                'atmo.context_processors.revision',
            ],
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            ],
        }
    },
]

# Django-CSP
CSP_DEFAULT_SRC = (
    "'self'",
)
CSP_FONT_SRC = (
    "'self'",
    "'unsafe-inline'",
    'http://*.mozilla.net',
    'https://*.mozilla.net',
    'http://*.mozilla.org',
    'https://*.mozilla.org',
)
CSP_IMG_SRC = (
    "'self'",
    "data:",
    'http://*.mozilla.net',
    'https://*.mozilla.net',
    'http://*.mozilla.org',
    'https://*.mozilla.org',
)
CSP_SCRIPT_SRC = (
    "'self'",
    "'unsafe-inline'",
    'http://*.mozilla.org',
    'https://*.mozilla.org',
    'http://*.mozilla.net',
    'https://*.mozilla.net',
)
CSP_STYLE_SRC = (
    "'self'",
    "'unsafe-inline'",
    'http://*.mozilla.org',
    'https://*.mozilla.org',
    'http://*.mozilla.net',
    'https://*.mozilla.net',
)

# This is needed to get a CRSF token in /admin
ANON_ALWAYS = True

# Sentry setup
SENTRY_DSN = config('SENTRY_DSN', default='')
if SENTRY_DSN:
    MIDDLEWARE_CLASSES = (
        'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware',
    ) + MIDDLEWARE_CLASSES

    INSTALLED_APPS = INSTALLED_APPS + [
        'raven.contrib.django.raven_compat',
    ]

    RAVEN_CONFIG = {
        'dsn': SENTRY_DSN,
        # If you are using git, you can also automatically configure the
        # release based on the git info.
        'release': raven.fetch_git_sha(os.path.dirname(os.pardir)),
    }
<commit_msg>Use revision from revision.txt for Sentry<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at http://mozilla.org/MPL/2.0/.
"""
Django settings for atmo project.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""
from datetime import timedelta
import os

import dj_database_url
from django.core.urlresolvers import reverse_lazy
from decouple import Csv, config
import atmo

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = config('SECRET_KEY')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = config('DEBUG', cast=bool)

ALLOWED_HOSTS = config('ALLOWED_HOSTS', cast=Csv())

SITE_ID = 1

# The URL under which this instance is running
SITE_URL = config('SITE_URL', default='http://localhost:8000')

# Whether or not this runs in Heroku
IS_HEROKU = 'HEROKU_APP_NAME' in os.environ

# Application definition

INSTALLED_APPS = [
    # Project specific apps
    'atmo',
    'atmo.clusters',
    'atmo.jobs',
    'atmo.users',

    # Third party apps
    'whitenoise.runserver_nostatic',
    'django_rq',
    'allauth',
    'allauth.account',
    'allauth.socialaccount',

    # Django apps
    'django.contrib.sites',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE_CLASSES = (
    'django.middleware.security.SecurityMiddleware',
    'atmo.middleware.NewRelicPapertrailMiddleware',
    'whitenoise.middleware.WhiteNoiseMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'session_csrf.CsrfMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'csp.middleware.CSPMiddleware',
)

ROOT_URLCONF = 'atmo.urls'

WSGI_APPLICATION = 'atmo.wsgi.application'

# AWS configuration
AWS_CONFIG = {
    # AWS EC2 configuration
    'AWS_REGION': 'us-west-2',

    # EMR configuration
    # Master and slave instance types should be the same as the telemetry
    # setup bootstrap action depends on it to autotune the cluster.
    'MASTER_INSTANCE_TYPE': 'c3.4xlarge',
    'WORKER_INSTANCE_TYPE': 'c3.4xlarge',
    'USE_SPOT_INSTANCES': config('USE_SPOT_INSTANCES', default=True, cast=bool),
    'CORE_SPOT_BID': config('CORE_SPOT_BID', default='0.84'),
    # available EMR releases, to be used as choices for Spark jobs and clusters
    # forms. Please keep the latest (newest) as the first item
    'EMR_RELEASES': (
        '5.0.0',
        '4.5.0',
    ),
    'SPARK_INSTANCE_PROFILE': 'telemetry-spark-cloudformation-'
                              'TelemetrySparkInstanceProfile-1SATUBVEXG7E3',
    'SPARK_EMR_BUCKET': 'telemetry-spark-emr-2',
    'INSTANCE_APP_TAG': 'telemetry-analysis-worker-instance',
    'EMAIL_SOURCE': 'telemetry-alerts@mozilla.com',
    'MAX_CLUSTER_SIZE': 30,

    # Tags for accounting purposes
    'ACCOUNTING_APP_TAG': 'telemetry-analysis',
    'ACCOUNTING_TYPE_TAG': 'worker',

    # Buckets for storing S3 data
    'CODE_BUCKET': 'telemetry-analysis-code-2',
    'PUBLIC_DATA_BUCKET': 'telemetry-public-analysis-2',
    'PRIVATE_DATA_BUCKET': 'telemetry-private-analysis-2',
}

for aws_cred in ('AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_DEFAULT_REGION'):
    if aws_cred not in os.environ:
        os.environ[aws_cred] = config(aws_cred, default='')

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases
DATABASES = {
    'default': config(
        'DATABASE_URL',
        cast=dj_database_url.parse
    )
}
# require encrypted connections to Postgres
if IS_HEROKU:
    DATABASES['default'].setdefault('OPTIONS', {})['sslmode'] = 'require'

REDIS_URL = config('REDIS_URL')

RQ_QUEUES = {
    'default': {
        'URL': REDIS_URL,
        'DEFAULT_TIMEOUT': 600,
    }
}

CACHES = {
    'default': {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': REDIS_URL,
    },
    'forms': {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': REDIS_URL + '?db=1',  # use a different db for form files
        'TIMEOUT': 60 * 60,  # an hour ought to be enough to edit a form
    },
}

# Add the django-allauth authentication backend.
AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
)

LOGIN_URL = reverse_lazy('account_login')
LOGOUT_URL = reverse_lazy('account_logout')
LOGIN_REDIRECT_URL = reverse_lazy('dashboard')

# django-allauth configuration
ACCOUNT_LOGOUT_REDIRECT_URL = LOGIN_REDIRECT_URL
ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = 7
ACCOUNT_EMAIL_SUBJECT_PREFIX = '[Telemetry Analysis Service] '
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = 'optional'
ACCOUNT_LOGOUT_ON_GET = True
ACCOUNT_ADAPTER = 'atmo.users.adapters.AtmoAccountAdapter'
ACCOUNT_USERNAME_REQUIRED = False

SOCIALACCOUNT_ADAPTER = 'atmo.users.adapters.AtmoSocialAccountAdapter'
SOCIALACCOUNT_EMAIL_VERIFICATION = 'none'  # no extra verification needed
SOCIALACCOUNT_QUERY_EMAIL = True  # needed by the Google provider

SOCIALACCOUNT_PROVIDERS = {
    'google': {
        'HOSTED_DOMAIN': 'mozilla.com',
        'AUTH_PARAMS': {
            'prompt': 'select_account',
        }
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/
LANGUAGE_CODE = config('LANGUAGE_CODE', default='en-us')
TIME_ZONE = config('TIME_ZONE', default='UTC')
USE_I18N = False
USE_L10N = False
USE_TZ = config('USE_TZ', default=True, cast=bool)
DATETIME_FORMAT = 'Y-m-d H:i'  # simplified ISO format since we assume UTC

STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'
STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

MEDIA_ROOT = config('MEDIA_ROOT', default=os.path.join(BASE_DIR, 'media'))
MEDIA_URL = config('MEDIA_URL', '/media/')

SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
SESSION_CACHE_ALIAS = 'default'

SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
X_FRAME_OPTIONS = 'DENY'

if SITE_URL.startswith('https://'):
    ACCOUNT_DEFAULT_HTTP_PROTOCOL = 'https'
    SECURE_SSL_REDIRECT = True
    SECURE_HSTS_SECONDS = int(timedelta(days=365).total_seconds())
    # Mark session and CSRF cookies as being HTTPS-only.
    CSRF_COOKIE_SECURE = True
    SESSION_COOKIE_SECURE = True

SILENCED_SYSTEM_CHECKS = [
    'security.W003',  # We're using django-session-csrf
    # We can't set SECURE_HSTS_INCLUDE_SUBDOMAINS since this runs under a
    # mozilla.org subdomain
    'security.W005',
    'security.W009',  # we know the SECRET_KEY is strong
]

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'OPTIONS': {
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.template.context_processors.request',
                'django.contrib.messages.context_processors.messages',
                'session_csrf.context_processor',
                'atmo.context_processors.settings',
                'atmo.context_processors.revision',
            ],
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            ],
        }
    },
]

# Django-CSP
CSP_DEFAULT_SRC = (
    "'self'",
)
CSP_FONT_SRC = (
    "'self'",
    "'unsafe-inline'",
    'http://*.mozilla.net',
    'https://*.mozilla.net',
    'http://*.mozilla.org',
    'https://*.mozilla.org',
)
CSP_IMG_SRC = (
    "'self'",
    "data:",
    'http://*.mozilla.net',
    'https://*.mozilla.net',
    'http://*.mozilla.org',
    'https://*.mozilla.org',
)
CSP_SCRIPT_SRC = (
    "'self'",
    "'unsafe-inline'",
    'http://*.mozilla.org',
    'https://*.mozilla.org',
    'http://*.mozilla.net',
    'https://*.mozilla.net',
)
CSP_STYLE_SRC = (
    "'self'",
    "'unsafe-inline'",
    'http://*.mozilla.org',
    'https://*.mozilla.org',
    'http://*.mozilla.net',
    'https://*.mozilla.net',
)

# This is needed to get a CRSF token in /admin
ANON_ALWAYS = True

# Sentry setup
SENTRY_DSN = config('SENTRY_DSN', default='')
if SENTRY_DSN:
    MIDDLEWARE_CLASSES = (
        'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware',
    ) + MIDDLEWARE_CLASSES

    INSTALLED_APPS = INSTALLED_APPS + [
        'raven.contrib.django.raven_compat',
    ]

    RAVEN_CONFIG = {
        'dsn': SENTRY_DSN,
    revision = atmo.get_revision()
    if revision:
        RAVEN_CONFIG['release'] = revision

    }
<|endoftext|>


<commit_before>"""Available Hebrew corpora available for download."""

MARATHI_CORPORA = [
    {'name': 'marathi_text_wikisource',
     'origin': 'https://github.com/cltk/marathi_text_wikisource.git',
     'location': 'remote',
     'type': 'text'},
]
<commit_msg>Update Corpora.py (#580)<commit_after>"""Available Marathi corpora available for download."""

MARATHI_CORPORA = [
    {'name': 'marathi_text_wikisource',
     'origin': 'https://github.com/cltk/marathi_text_wikisource.git',
     'location': 'remote',
     'type': 'text'},
]
<|endoftext|>


<commit_before>"""
#lista de repetio

#Antonio Rodrigues de Souza Neto --- matrcula ----1615310028
#Gabriel Machado Moreira --- matrcula ----1615310034
#Luiz Gustavo de Rocha Melo --- matrcula ---- 1615310015
#Lucas Ferreira Soares --- 1615310014
#Calebe Roberto Chaves da Silva Rebello --- matrcula---1615310043

"""
entrada = int(input("Digite um nmero positivo inteiro: "))

primo = 1
contador = 0

entrada1 = entrada/2

while primo <= entrada:
    if entrada % primo==0:
        primo = primo+1
        contador = contador+1
    if primo >= entrada1:
        primo = entrada
        primo = primo + 1
        contador = contador + 1
    else:
        primo = primo + 1
if contador==2:
    print("O nmero requisitado  primo!")
else:
    print("O nmero requisitado no  primo!")
<commit_msg>Delete ipc_lista3.21.py<commit_after><|endoftext|>


<commit_before># coding: utf-8
"""namu.wiki contrib grabber"""
from datetime import datetime, timedelta
import urllib.parse

from bs4 import BeautifulSoup
import requests

__all__ = 'contrib',

URL = 'https://namu.wiki/contribution/author/{author}/document'
DATE_FORMAT = r'%Y-%m-%d %H:%M:%S'

class NamuContrib(object):
    """contribution object"""

    def __init__(self, document, revision, changes, when, desc='', revert=None):
        self.document = document
        self.revision = revision
        self.changes = changes
        self.when = when
        self.desc = desc
        self.revert = revert

    @property
    def when(self):
        return self._when

    @when.setter
    def when(self, when):
        if isinstance(when, datetime):
            self._when = when
        else:
            self._when = datetime.strptime(when, DATE_FORMAT) - timedelta(hours=9)

    def as_dict(self):
        obj = {
            'document': self.document,
            'revision': self.revision,
            'changes': self.changes,
            'when': self.when,
            'desc': self.desc,
            'revert': self.revert,
        }
        return obj

    def __repr__(self):
        return '<NamuContrib doc=%s rev=%s changes=%s when=%s desc=%s>' % (
            self.document, self.revision, self.changes, self.when, self.desc)

def extint(input):
    """Extract numbers from a string"""
    num = None
    try:
        num = int(''.join([s for s in input if s.isdigit()]))
    except ValueError:
        pass
    return num

def contrib(username):
    """contributions"""
    result = []

    url = URL.format(author=username)
    req = requests.get(url)
    soup = BeautifulSoup(req.text, 'lxml')
    soup.encoding = 'utf-8'

    rows = soup.select('article table tbody tr')
    item = None
    hasdetail = False
    for i, row in enumerate(rows):
        if not item and not hasdetail:
            info = row.select('td')[0]
            document = info.select('a')[0].string
            try:
                qs = {k: ''.join(v) for k, v in
                      urllib.parse.parse_qs(info.select('a')[2].attrs['href'].split('?')[-1]).items()}
            except IndexError:
                revision = 1
            else:
                revision = int(qs['rev'])
            changes = int(info.select('span')[-1].string)
            when = row.select('td')[2].string.strip()
            item = NamuContrib(document=document, revision=revision, changes=changes, when=when)

            # Find reverts
            revert = info.select('a + i')
            if revert:
                revert = extint(revert[0].string)
                item.revert = revert
        elif item and hasdetail:
            desc = row.select('td')[0].string
            item.desc = desc
            hasdetail = False
        if 'no-line' in row.attrs.get('class', []):
            hasdetail = True

        if item and not hasdetail:
            result.append(item)
            item = None

    return result
<commit_msg>Add a URL for IP-based contributions<commit_after># coding: utf-8
"""namu.wiki contrib grabber"""
import re
import urllib.parse
from datetime import datetime, timedelta

from bs4 import BeautifulSoup
import requests

__all__ = 'contrib',

URL = 'https://namu.wiki/contribution/author/{author}/document'
URL_IP = 'https://namu.wiki/contribution/ip/{author}/document'
DATE_FORMAT = r'%Y-%m-%d %H:%M:%S'
RE_IP = re.compile('^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$')

class NamuContrib(object):
    """contribution object"""

    def __init__(self, document, revision, changes, when, desc='', revert=None):
        self.document = document
        self.revision = revision
        self.changes = changes
        self.when = when
        self.desc = desc
        self.revert = revert

    @property
    def when(self):
        return self._when

    @when.setter
    def when(self, when):
        if isinstance(when, datetime):
            self._when = when
        else:
            self._when = datetime.strptime(when, DATE_FORMAT) - timedelta(hours=9)

    def as_dict(self):
        obj = {
            'document': self.document,
            'revision': self.revision,
            'changes': self.changes,
            'when': self.when,
            'desc': self.desc,
            'revert': self.revert,
        }
        return obj

    def __repr__(self):
        return '<NamuContrib doc=%s rev=%s changes=%s when=%s desc=%s>' % (
            self.document, self.revision, self.changes, self.when, self.desc)

def extint(input):
    """Extract numbers from a string"""
    num = None
    try:
        num = int(''.join([s for s in input if s.isdigit()]))
    except ValueError:
        pass
    return num

def contrib(username):
    """contributions"""
    result = []

    url = URL.format(author=username)
    if '.' in username and RE_IP.match(username):
        url = URL_IP.format(author=username)

    req = requests.get(url)
    soup = BeautifulSoup(req.text, 'lxml')
    soup.encoding = 'utf-8'

    rows = soup.select('article table tbody tr')
    item = None
    hasdetail = False
    for i, row in enumerate(rows):
        if not item and not hasdetail:
            info = row.select('td')[0]
            document = info.select('a')[0].string
            try:
                qs = {k: ''.join(v) for k, v in
                      urllib.parse.parse_qs(info.select('a')[2].attrs['href'].split('?')[-1]).items()}
            except IndexError:
                revision = 1
            else:
                revision = int(qs['rev'])
            changes = int(info.select('span')[-1].string)
            when = row.select('td')[2].string.strip()
            item = NamuContrib(document=document, revision=revision, changes=changes, when=when)

            # Find reverts
            revert = info.select('a + i')
            if revert:
                revert = extint(revert[0].string)
                item.revert = revert
        elif item and hasdetail:
            desc = row.select('td')[0].string
            item.desc = desc
            hasdetail = False
        if 'no-line' in row.attrs.get('class', []):
            hasdetail = True

        if item and not hasdetail:
            result.append(item)
            item = None

    return result
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sqlite3

from django.conf import settings

import cluster_labeling.optemo_django_models as optemo

import cluster_count_table as cct
import cluster_totalcount_table as ctct

# At the leaf clusters,
# - wordcounts stores the number of a particular word found in all
#   reviews associated with the cluster.
# - prodcounts stores the number of a particular product associated
#   with the cluster that contain the word. This should be equal to 1,
#   since only one product is associated with each leaf cluster.
# - reviewcounts stores the number of reviews of a particular product
#   associated with the cluster that contain the word.
count_tables = [cct.ClusterProdCount, cct.ClusterWordCount,
                cct.ClusterReviewCount]
totalcount_tables = [ctct.ClusterProdTotalCount,
                     ctct.ClusterWordTotalCount,
                     ctct.ClusterReviewTotalCount]

import nltk.tokenize.punkt as punkt
import nltk.tokenize.treebank as treebank
sentence_tokenizer = punkt.PunktSentenceTokenizer()
word_tokenizer = treebank.TreebankWordTokenizer()

# From: http://www.daniweb.com/code/snippet216879.html#
def flatten(lst):
    for elem in lst:
        if type(elem) == list:
            for i in flatten(elem):
                yield i
        else:
            yield elem

spurious_pos = \
    ('TO', 'CC', '.', ':', 'DT', 'IN', 'PRP', ',',
     'EX', 'WRB', 'RP', 'CD', 'MD', '``', '$')

spurious_words = \
    ('br', '<')

def is_spurious_pos(postag):
    return postag in spurious_pos

import xml.sax.saxutils
def remove_html_escaping(content):
    return xml.sax.saxutils.unescape(content,
                                     {"'" : "'", """ : '"'})

from nltk.corpus import stopwords
stopword_set = set(stopwords.words('english'))
stopword_set |= set(['br', '<', '"', '(', ')'])
def is_stopword(word):
    return word in stopword_set

import nltk.stem.porter
stemmer = nltk.stem.porter.PorterStemmer()

import re
punct_re = re.compile('\W+')
number_re = re.compile('^\d+$')

def get_words_from_review(content):
    # Use the Punkt sentence tokenizer and the Treebank word tokenizer
    # to get the words in the review.
    words = map(word_tokenizer.tokenize,
                sentence_tokenizer.tokenize(remove_html_escaping(content)))

    # (Don't) perform part-of-speech tagging to get rid of spurious tokens.
    do_pos_based_elimination_of_spurious_tokens = False
    if do_pos_based_elimination_of_spurious_tokens:
        words = map(nltk.pos_tag, words)
        words = flatten(words)
        words = map(lambda (word, pos): word,
                    filter(lambda (word, pos): not is_spurious_pos(pos),
                           words))
    else:
        words = flatten(words)

    words = filter(lambda x: not is_stopword(x), words)
    words = map(lambda x: x.lower(), words)

    # Get rid of punctuation
    words_punct_split = []
    for word in words:
        splitwords = filter(lambda x: len(x) > 0, punct_re.split(word))
        words_punct_split.extend(splitwords)
    words = words_punct_split

    # Filter out everything is only 1 character long
    words = filter(lambda x: len(x) > 1, words)

    # Filter out everything that only consists of numbers
    words = filter(lambda x: not number_re.match(x), words)

    # Filter out everything that has as many or more numbers than
    # letters. This gets rid of a lot of model numbers.
    words = filter(lambda word:
                   reduce(lambda n, l: n + 1 if l.isalpha() else 0,
                          word, 0)
                   >
                   (len(word)/2),
                   words)
    return words

import cluster_labeling.pn_spellcheck as pnsc

default_spellchecker_fn = '/optemo/site/cluster_labeling/spellchecker.pkl'

def train_spellchecker_on_reviews\
        (spellchecker_fn = default_spellchecker_fn):
    spellchecker = pnsc.PNSpellChecker()

    i = 0

    content = ""
    for review in optemo.Review.get_manager().all():
        i += 1
        content += " " + review.content

        print i, ": ", len(content)

        if len(content) > 2**20:
            words = get_words_from_review(content)
            spellchecker.train(words)
            content = ""

    words = get_words_from_review(content)
    spellchecker.train(words)

    pnsc.save_spellchecker(spellchecker, spellchecker_fn)

def compute_wordcounts_for_review(content):
    wcs = {}

    words = get_words_from_review(content)

    # Populate word counts
    for word in words:
        wcs[word] = wcs.get(word, 0) + 1

    # Create a map of stemmings and labels for stemmings
    stemmings = {}
    stemming_labels = {}
    for key in wcs.iterkeys():
        stemmed_key = stemmer.stem(key)

        if stemmed_key not in stemmings:
            stemmings[stemmed_key] = set()

        stemmings[stemmed_key].update([key])

        if (stemmed_key not in stemming_labels or
            len(key) < stemming_labels[stemmed_key]):
            stemming_labels[stemmed_key] = key

    # Create a stemmed wordcount
    wcs_stemmed = {}
    for (k,v) in stemmings.iteritems():
        wcs_stemmed[stemming_labels[k]] = sum(map(lambda x: wcs[x], v))

    return wcs_stemmed

def compute_counts_for_product(product):
    reviews = product.get_reviews()
    wordcounts = \
        map(lambda r: compute_wordcounts_for_review(r.content),
            reviews)

    merged_wordcount = {}
    reviewcount = {}

    for wc in wordcounts:
        for word, count in wc.iteritems():
            merged_wordcount[word] = \
                merged_wordcount.get(word, 0) + count
            reviewcount[word] = \
                reviewcount.get(word, 0) + 1

    prodcount = dict(map(lambda k: (k, 1), reviewcount.keys()))

    return merged_wordcount, reviewcount, prodcount

def compute_counts_for_cluster(cluster):
    children = cluster.get_children()
    numchildren = children.count()

    if numchildren == 0:
        nodes = cluster.get_nodes()
        assert(nodes.count() == 1)

        product = nodes[0].get_product()
        wordcount, reviewcount, prodcount = \
            compute_counts_for_product(product)

        map(lambda table, counts:
            table.add_values_from(cluster.id, cluster.parent_id,
                                  numchildren, counts),
            count_tables, [prodcount, wordcount, reviewcount])

        totalcounts_to_mod_values = [1]
        totalcounts_to_mod = [ctct.ClusterProdTotalCount]

        # Total counts for reviews and words only need to be added if
        # the product actually contains reviews.
        if wordcount != {}:
            totalcounts_to_mod_values.extend\
            ([sum(wordcount.itervalues()),
              product.get_reviews().count()])
            totalcounts_to_mod.extend([ctct.ClusterWordTotalCount,
                                       ctct.ClusterReviewTotalCount])

        map(lambda table, totalcount:
            table\
            (cluster_id = cluster.id,
             parent_cluster_id = cluster.parent_id,
             totalcount = totalcount, numchildren = 0).save(),
            totalcounts_to_mod, totalcounts_to_mod_values)
    else:
        map(lambda child: compute_counts_for_cluster(child),
            children)

        map(lambda table:
            table.sum_child_cluster_counts\
            (cluster.id, cluster.parent_id, numchildren),
            count_tables)
        map(lambda table:
            table.sum_child_cluster_totalcounts\
            (cluster.id, cluster.parent_id, numchildren),
            totalcount_tables)

def compute_all_counts\
        (version=optemo.CameraCluster.get_latest_version()):
    # All tables should be recreated, otherwise the resulting counts
    # will not be valid.
    map(lambda table: table.drop_table_if_exists(), count_tables)
    map(lambda table: table.drop_table_if_exists(), totalcount_tables)

    map(lambda table: table.create_table(), count_tables)
    map(lambda table: table.create_table(), totalcount_tables)

    # Get clusters just below the root.
    root_children = optemo.CameraCluster.get_root_children(version)

    map(lambda child: compute_counts_for_cluster(child),
        root_children)

    map(lambda table:
        table.sum_child_cluster_counts(0, -1, root_children.count()),
        count_tables)
    map(lambda table:
        table.sum_child_cluster_totalcounts\
        (0, -1, root_children.count()),
        totalcount_tables)
<commit_msg>Adding code to use spellchecker<commit_after>#!/usr/bin/env python
import os
import sqlite3

from django.conf import settings

import cluster_labeling.optemo_django_models as optemo

import cluster_count_table as cct
import cluster_totalcount_table as ctct

# At the leaf clusters,
# - wordcounts stores the number of a particular word found in all
#   reviews associated with the cluster.
# - prodcounts stores the number of a particular product associated
#   with the cluster that contain the word. This should be equal to 1,
#   since only one product is associated with each leaf cluster.
# - reviewcounts stores the number of reviews of a particular product
#   associated with the cluster that contain the word.
count_tables = [cct.ClusterProdCount, cct.ClusterWordCount,
                cct.ClusterReviewCount]
totalcount_tables = [ctct.ClusterProdTotalCount,
                     ctct.ClusterWordTotalCount,
                     ctct.ClusterReviewTotalCount]

import nltk.tokenize.punkt as punkt
import nltk.tokenize.treebank as treebank
sentence_tokenizer = punkt.PunktSentenceTokenizer()
word_tokenizer = treebank.TreebankWordTokenizer()

# From: http://www.daniweb.com/code/snippet216879.html#
def flatten(lst):
    for elem in lst:
        if type(elem) == list:
            for i in flatten(elem):
                yield i
        else:
            yield elem

spurious_pos = \
    ('TO', 'CC', '.', ':', 'DT', 'IN', 'PRP', ',',
     'EX', 'WRB', 'RP', 'CD', 'MD', '``', '$')

spurious_words = \
    ('br', '<')

def is_spurious_pos(postag):
    return postag in spurious_pos

import xml.sax.saxutils
def remove_html_escaping(content):
    return xml.sax.saxutils.unescape(content,
                                     {"'" : "'", """ : '"'})

from nltk.corpus import stopwords
stopword_set = set(stopwords.words('english'))
stopword_set |= set(['br', '<', '"', '(', ')'])
def is_stopword(word):
    return word in stopword_set

import nltk.stem.porter
stemmer = nltk.stem.porter.PorterStemmer()

import re
punct_re = re.compile('\W+')
number_re = re.compile('^\d+$')

def get_words_from_review(content):
    # Use the Punkt sentence tokenizer and the Treebank word tokenizer
    # to get the words in the review.
    words = map(word_tokenizer.tokenize,
                sentence_tokenizer.tokenize(remove_html_escaping(content)))

    # (Don't) perform part-of-speech tagging to get rid of spurious tokens.
    do_pos_based_elimination_of_spurious_tokens = False
    if do_pos_based_elimination_of_spurious_tokens:
        words = map(nltk.pos_tag, words)
        words = flatten(words)
        words = map(lambda (word, pos): word,
                    filter(lambda (word, pos): not is_spurious_pos(pos),
                           words))
    else:
        words = flatten(words)

    words = filter(lambda x: not is_stopword(x), words)
    words = map(lambda x: x.lower(), words)

    # Get rid of punctuation
    words_punct_split = []
    for word in words:
        splitwords = filter(lambda x: len(x) > 0, punct_re.split(word))
        words_punct_split.extend(splitwords)
    words = words_punct_split

    # Filter out everything is only 1 character long
    words = filter(lambda x: len(x) > 1, words)

    # Filter out everything that only consists of numbers
    words = filter(lambda x: not number_re.match(x), words)

    # Filter out everything that has as many or more numbers than
    # letters. This gets rid of a lot of model numbers.
    words = filter(lambda word:
                   reduce(lambda n, l: n + 1 if l.isalpha() else 0,
                          word, 0)
                   >
                   (len(word)/2),
                   words)
    return words

import cluster_labeling.pn_spellcheck as pnsc

default_spellchecker_fn = '/optemo/site/cluster_labeling/spellchecker.pkl'

def train_spellchecker_on_reviews\
        (spellchecker_fn = default_spellchecker_fn):
    spellchecker = pnsc.PNSpellChecker()

    i = 0

    content = ""
    for review in optemo.Review.get_manager().all():
        i += 1
        content += " " + review.content

        print i, ": ", len(content)

        if len(content) > 2**20:
            words = get_words_from_review(content)
            spellchecker.train(words)
            content = ""

    words = get_words_from_review(content)
    spellchecker.train(words)

    pnsc.save_spellchecker(spellchecker, spellchecker_fn)

def compute_wordcounts_for_review(content, spellchecker):
    wcs = {}

    words = get_words_from_review(content)

    # Perform spell-checking
    words = map(spellchecker.correct, words)

    # Populate word counts
    for word in words:
        wcs[word] = wcs.get(word, 0) + 1

    # Create a map of stemmings and labels for stemmings
    stemmings = {}
    stemming_labels = {}
    for key in wcs.iterkeys():
        stemmed_key = stemmer.stem(key)

        if stemmed_key not in stemmings:
            stemmings[stemmed_key] = set()

        stemmings[stemmed_key].update([key])

        if (stemmed_key not in stemming_labels or
            len(key) < stemming_labels[stemmed_key]):
            stemming_labels[stemmed_key] = key

    # Create a stemmed wordcount
    wcs_stemmed = {}
    for (k,v) in stemmings.iteritems():
        wcs_stemmed[stemming_labels[k]] = sum(map(lambda x: wcs[x], v))

    return wcs_stemmed

def compute_counts_for_product(product, spellchecker):
    reviews = product.get_reviews()
    wordcounts = \
        map(lambda r: compute_wordcounts_for_review(r.content, spellchecker),
            reviews)

    merged_wordcount = {}
    reviewcount = {}

    for wc in wordcounts:
        for word, count in wc.iteritems():
            merged_wordcount[word] = \
                merged_wordcount.get(word, 0) + count
            reviewcount[word] = \
                reviewcount.get(word, 0) + 1

    prodcount = dict(map(lambda k: (k, 1), reviewcount.keys()))

    return merged_wordcount, reviewcount, prodcount

def compute_counts_for_cluster(cluster, spellchecker):
    children = cluster.get_children()
    numchildren = children.count()

    if numchildren == 0:
        nodes = cluster.get_nodes()
        assert(nodes.count() == 1)

        product = nodes[0].get_product()
        wordcount, reviewcount, prodcount = \
            compute_counts_for_product(product, spellchecker)

        map(lambda table, counts:
            table.add_values_from(cluster.id, cluster.parent_id,
                                  numchildren, counts),
            count_tables, [prodcount, wordcount, reviewcount])

        totalcounts_to_mod_values = [1]
        totalcounts_to_mod = [ctct.ClusterProdTotalCount]

        # Total counts for reviews and words only need to be added if
        # the product actually contains reviews.
        if wordcount != {}:
            totalcounts_to_mod_values.extend\
            ([sum(wordcount.itervalues()),
              product.get_reviews().count()])
            totalcounts_to_mod.extend([ctct.ClusterWordTotalCount,
                                       ctct.ClusterReviewTotalCount])

        map(lambda table, totalcount:
            table\
            (cluster_id = cluster.id,
             parent_cluster_id = cluster.parent_id,
             totalcount = totalcount, numchildren = 0).save(),
            totalcounts_to_mod, totalcounts_to_mod_values)
    else:
        map(lambda child:
            compute_counts_for_cluster(child, spellchecker),
            children)

        map(lambda table:
            table.sum_child_cluster_counts\
            (cluster.id, cluster.parent_id, numchildren),
            count_tables)
        map(lambda table:
            table.sum_child_cluster_totalcounts\
            (cluster.id, cluster.parent_id, numchildren),
            totalcount_tables)

def compute_all_counts\
        (spellchecker = None,
        version=optemo.CameraCluster.get_latest_version()):
    if spellchecker == None:
        spellchecker = pnsc.load_spellchecker(default_spellchecker_fn)

    # All tables should be recreated, otherwise the resulting counts
    # will not be valid.
    map(lambda table: table.drop_table_if_exists(), count_tables)
    map(lambda table: table.drop_table_if_exists(), totalcount_tables)

    map(lambda table: table.create_table(), count_tables)
    map(lambda table: table.create_table(), totalcount_tables)

    # Get clusters just below the root.
    root_children = optemo.CameraCluster.get_root_children(version)

    map(lambda child: compute_counts_for_cluster(child, spellchecker),
        root_children)

    map(lambda table:
        table.sum_child_cluster_counts(0, -1, root_children.count()),
        count_tables)
    map(lambda table:
        table.sum_child_cluster_totalcounts\
        (0, -1, root_children.count()),
        totalcount_tables)
<|endoftext|>


<commit_before>import sys, socket, time, logging
import shlex, subprocess
import netifaces as ni
from hdfs import *
from kazoo.client import KazooClient

def zkConnect(conn_str):
  zk = KazooClient(hosts=conn_str)
  zk.start()
  return zk

def zkCreateJobDir(zk, job_name):
  zk.ensure_path("/napper/naiad/%s" % (job_name))

def zkRemoveJobDir(zk, job_name):
  zk.delete("/napper/naiad/%s" % (job_name), recursive=True)

def zkRegisterWorker(zk, job_name, worker_id, hostname, port):
  while zk.exists("/napper/naiad/%s:%d" % (hostname, port)):
    port += 1
  print "Registering myself as %s:%d on %s:%d" % (job_name, worker_id, hostname, port)
  zk.create("/napper/naiad/%s:%d" % (hostname, port), "%d" % (port), ephemeral=True)
  zk.create("/napper/naiad/%s/%d" % (job_name, worker_id), "%s:%d" % (hostname, port), ephemeral=True)
  return port

def zkDeregisterWorker(zk, job_name, worker_id, hostname, port):
  print "Finished; unregistering myself from %s" % (job_name)
  zk.delete("/napper/naiad/%s/%d" % (job_name, worker_id))
  zk.delete("/napper/naiad/%s:%d" % (hostname, port))

logging.basicConfig()

if len(sys.argv) < 6:
  print "usage: napper_naiad <Zookeeper hostname:port> <job name> <worker ID> <num workers> <executable>"
  sys.exit(1)

hostport = sys.argv[1]
job_name = sys.argv[2]
worker_id = int(sys.argv[3])
num_workers = int(sys.argv[4])
naiad_path = " ".join(sys.argv[5:])

client = zkConnect(hostport)
zkCreateJobDir(client, job_name)

done = False
hostname = ni.ifaddresses('p1p1')[2][0]['addr']
while not done:
  try:
    actual_port = zkRegisterWorker(client, job_name, worker_id, hostname, 2100 + worker_id)
    done = True
  except NodeExistsError:
    pass

done = False
hosts = []
while not done:
  children = client.get_children("/napper/naiad/%s" % (job_name))
  if len(children) == num_workers:
    print "All workers are here!"
    for c in sorted(children,
                    key=lambda item: (int(item.partition(' ')[0])
                                      if item[0].isdigit()
                                      else float('inf'), item)):
      data, stat = client.get("/napper/naiad/%s/%s" % (job_name, c))
      print "%s @ %s" % (c, data)
      hosts.append("%s" % (data))
    done = True
  time.sleep(1)

if not os.path.exists(os.environ['FLAGS_task_data_dir']):
  os.makedirs(os.environ['FLAGS_task_data_dir'])

# fetch inputs from HDFS if necessary
if "tpch" in job_name:
  hdfs_fetch_file("/input/part_splits%d/part%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  hdfs_fetch_file("/input/lineitem_splits%d/lineitem%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  naiad_path += " %s" % (os.environ['FLAGS_task_data_dir'])
elif "netflix" in job_name:
  hdfs_fetch_file("/input/netflix_movies_splits%d/netflix_movies%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  hdfs_fetch_file("/input/netflix_ratings_splits%d/netflix_ratings%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  naiad_path += " netflix_ratings%d.in netflix_movies%d.in %s" % (worker_id, worker_id, os.environ['FLAGS_task_data_dir'])
elif "pagerank" in job_name:
  hdfs_fetch_file("/input/pagerank_livejournal_edges_splits%d/pagerank_livejournal_edges%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  hdfs_fetch_file("/input/pagerank_livejournal_vertices_splits%d/pagerank_livejournal_vertices%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  naiad_path += " livejournal %s" % (os.environ['FLAGS_task_data_dir'])
elif "sssp" in job_name:
  hdfs_fetch_file("/input/sssp_tw_edges_splits%d/sssp_tw_edges%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  hdfs_fetch_file("/input/sssp_tw_vertices_splits%d/sssp_tw_vertices%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  naiad_path += " tw %s" % (os.environ['FLAGS_task_data_dir'])
else:
  print "WARNING: unknown Naiad job type; won't fetch any input data from HDFS."

# execute program
command = "mono-sgen %s -p %d -n %d -t 1 -h %s --inlineserializer" % (naiad_path, worker_id, num_workers, " ".join(hosts))
print "RUNNING: %s" % (command)
ret = subprocess.call(shlex.split(command))

if ret != 0:
  print "ERROR: Naiad run failed!"
  print "Not cleaning up any state."
  sys.exit(ret)

zkDeregisterWorker(client, job_name, worker_id, hostname, actual_port)
#if worker_id == 0:
#  zkRemoveJobDir(client, job_name)
client.stop()

hdfs_mkdir("/output/%s" % (job_name))
if "tpch" in job_name:
  push_ret = hdfs_push_file("%s/avg_yearly%d.out" % (os.environ['FLAGS_task_data_dir'], worker_id), "/output/%s/" % (job_name))
elif "netflix" in job_name:
  push_ret = hdfs_push_file("%s/prediction%d.out" % (os.environ['FLAGS_task_data_dir'], worker_id), "/output/%s/" % (job_name))
elif "pagerank" in job_name:
  push_ret = hdfs_push_file("%s/pagerank_livejournal%d.out" % (os.environ['FLAGS_task_data_dir'], worker_id), "/output/%s/" % (job_name))
elif "sssp" in job_name:
  push_ret = hdfs_push_file("%s/dij_vertices%d.out" % (os.environ['FLAGS_task_data_dir'], worker_id), "/output/%s/" % (job_name))
else:
  print "WARNING: unknown Naiad job type; won't fetch any input data from HDFS."

if push_ret != 0:
  print "ERROR: failed to push result to HDFS! Leaving state around for inspection."
else:
  print "Deleting scratch data..."
  del_command = "rm -rf %s" % (os.environ['FLAGS_task_data_dir'])
  ret = subprocess.call(shlex.split(del_command))

  if ret != 0:
    print "Failed to delete local scratch directory"

print "All done -- goodbye from Napper!"

sys.exit(0)
<commit_msg>Always initialize push_ret (even for SCC job, which doesn't need HDFS).<commit_after>import sys, socket, time, logging
import shlex, subprocess
import netifaces as ni
from hdfs import *
from kazoo.client import KazooClient

def zkConnect(conn_str):
  zk = KazooClient(hosts=conn_str)
  zk.start()
  return zk

def zkCreateJobDir(zk, job_name):
  zk.ensure_path("/napper/naiad/%s" % (job_name))

def zkRemoveJobDir(zk, job_name):
  zk.delete("/napper/naiad/%s" % (job_name), recursive=True)

def zkRegisterWorker(zk, job_name, worker_id, hostname, port):
  while zk.exists("/napper/naiad/%s:%d" % (hostname, port)):
    port += 1
  print "Registering myself as %s:%d on %s:%d" % (job_name, worker_id, hostname, port)
  zk.create("/napper/naiad/%s:%d" % (hostname, port), "%d" % (port), ephemeral=True)
  zk.create("/napper/naiad/%s/%d" % (job_name, worker_id), "%s:%d" % (hostname, port), ephemeral=True)
  return port

def zkDeregisterWorker(zk, job_name, worker_id, hostname, port):
  print "Finished; unregistering myself from %s" % (job_name)
  zk.delete("/napper/naiad/%s/%d" % (job_name, worker_id))
  zk.delete("/napper/naiad/%s:%d" % (hostname, port))

logging.basicConfig()

if len(sys.argv) < 6:
  print "usage: napper_naiad <Zookeeper hostname:port> <job name> <worker ID> <num workers> <executable>"
  sys.exit(1)

hostport = sys.argv[1]
job_name = sys.argv[2]
worker_id = int(sys.argv[3])
num_workers = int(sys.argv[4])
naiad_path = " ".join(sys.argv[5:])

client = zkConnect(hostport)
zkCreateJobDir(client, job_name)

done = False
hostname = ni.ifaddresses('p1p1')[2][0]['addr']
while not done:
  try:
    actual_port = zkRegisterWorker(client, job_name, worker_id, hostname, 2100 + worker_id)
    done = True
  except NodeExistsError:
    pass

done = False
hosts = []
while not done:
  children = client.get_children("/napper/naiad/%s" % (job_name))
  if len(children) == num_workers:
    print "All workers are here!"
    for c in sorted(children,
                    key=lambda item: (int(item.partition(' ')[0])
                                      if item[0].isdigit()
                                      else float('inf'), item)):
      data, stat = client.get("/napper/naiad/%s/%s" % (job_name, c))
      print "%s @ %s" % (c, data)
      hosts.append("%s" % (data))
    done = True
  time.sleep(1)

if not os.path.exists(os.environ['FLAGS_task_data_dir']):
  os.makedirs(os.environ['FLAGS_task_data_dir'])

# fetch inputs from HDFS if necessary
if "tpch" in job_name:
  hdfs_fetch_file("/input/part_splits%d/part%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  hdfs_fetch_file("/input/lineitem_splits%d/lineitem%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  naiad_path += " %s" % (os.environ['FLAGS_task_data_dir'])
elif "netflix" in job_name:
  hdfs_fetch_file("/input/netflix_movies_splits%d/netflix_movies%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  hdfs_fetch_file("/input/netflix_ratings_splits%d/netflix_ratings%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  naiad_path += " netflix_ratings%d.in netflix_movies%d.in %s" % (worker_id, worker_id, os.environ['FLAGS_task_data_dir'])
elif "pagerank" in job_name:
  hdfs_fetch_file("/input/pagerank_livejournal_edges_splits%d/pagerank_livejournal_edges%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  hdfs_fetch_file("/input/pagerank_livejournal_vertices_splits%d/pagerank_livejournal_vertices%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  naiad_path += " livejournal %s" % (os.environ['FLAGS_task_data_dir'])
elif "sssp" in job_name:
  hdfs_fetch_file("/input/sssp_tw_edges_splits%d/sssp_tw_edges%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  hdfs_fetch_file("/input/sssp_tw_vertices_splits%d/sssp_tw_vertices%d.in" % (num_workers, worker_id), os.environ['FLAGS_task_data_dir'])
  naiad_path += " tw %s" % (os.environ['FLAGS_task_data_dir'])
else:
  print "WARNING: unknown Naiad job type; won't fetch any input data from HDFS."

# execute program
command = "mono-sgen %s -p %d -n %d -t 1 -h %s --inlineserializer" % (naiad_path, worker_id, num_workers, " ".join(hosts))
print "RUNNING: %s" % (command)
ret = subprocess.call(shlex.split(command))

if ret != 0:
  print "ERROR: Naiad run failed!"
  print "Not cleaning up any state."
  sys.exit(ret)

zkDeregisterWorker(client, job_name, worker_id, hostname, actual_port)
#if worker_id == 0:
#  zkRemoveJobDir(client, job_name)
client.stop()

hdfs_mkdir("/output/%s" % (job_name))
push_ret = 0
if "tpch" in job_name:
  push_ret = hdfs_push_file("%s/avg_yearly%d.out" % (os.environ['FLAGS_task_data_dir'], worker_id), "/output/%s/" % (job_name))
elif "netflix" in job_name:
  push_ret = hdfs_push_file("%s/prediction%d.out" % (os.environ['FLAGS_task_data_dir'], worker_id), "/output/%s/" % (job_name))
elif "pagerank" in job_name:
  push_ret = hdfs_push_file("%s/pagerank_livejournal%d.out" % (os.environ['FLAGS_task_data_dir'], worker_id), "/output/%s/" % (job_name))
elif "sssp" in job_name:
  push_ret = hdfs_push_file("%s/dij_vertices%d.out" % (os.environ['FLAGS_task_data_dir'], worker_id), "/output/%s/" % (job_name))
else:
  print "WARNING: unknown Naiad job type; won't fetch any input data from HDFS."

if push_ret != 0:
  print "ERROR: failed to push result to HDFS! Leaving state around for inspection."
else:
  print "Deleting scratch data..."
  del_command = "rm -rf %s" % (os.environ['FLAGS_task_data_dir'])
  ret = subprocess.call(shlex.split(del_command))

  if ret != 0:
    print "Failed to delete local scratch directory"

print "All done -- goodbye from Napper!"

sys.exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import fcntl
import getopt
import os
import random
import select
import socket
import sys

def usage(f = sys.stdout):
    print >> f, """\
Usage: %s [-a|-b] [OPTIONS] REMOTE\
""" % sys.argv[0]

MAGIC = "MAGIC_STREAM_START"

SIDE = None
PORT = 2000
N = 500
remote = None
DELAY = 0.1

def unblock(fd):
    flags = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)

def main_a(remote):
    ports = set()
    rset = []

    while True:
        for i in range(N):
            while True:
                r = random.randrange(1024, 65536)
                if r not in ports:
                    break
            ports.add(r)
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
            s.bind(("0.0.0.0", r))
            print r
            addr = (remote, PORT)
            s.sendto("hello", 0, addr)
            rset.append(s)
        print "waiting 20 seconds..."
        readable, _, _ = select.select(rset, [], [], 20)
        for fd in readable:
            data, peer = fd.recvfrom(512)
            if data:
                return fd, peer
        while rset:
            rset.pop().close()

def main_b(remote):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
    s.bind(("0.0.0.0", PORT))

    i = 0
    while True:
        if i >= N:
            readable, _, _ = select.select([s], [], [], DELAY)
            for fd in readable:
                data, peer = fd.recvfrom(512)
                if data:
                    return fd, peer
        else:
            i += 1
        port = random.randrange(1024, 65536)
        addr = (remote, port)
        print addr
        s.sendto("hello", 0, addr)

opts, args = getopt.gnu_getopt(sys.argv[1:], "abn:p:")
for o, a in opts:
    if o == "-a":
        SIDE = 0
    elif o == "-b":
        SIDE = 1
    elif o == "-n":
        N = int(a)
    elif o == "-p":
        PORT = int(a)

try:
    remote_hostname, = args
except ValueError:
    usage(sys.stderr)
    sys.exit(1)

addrinfo = socket.getaddrinfo(remote_hostname, 0, socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
print addrinfo

if SIDE is None:
    usage(sys.stderr)
    sys.exit(1)
elif SIDE == 0:
    s, peer = main_a(addrinfo[0][4][0])
else:
    s, peer = main_b(addrinfo[0][4][0])

print "got connection", peer

s.sendto(MAGIC, 0, peer)

unblock(sys.stdin)

print "start typing"

while True:
    readable, _, _ = select.select([sys.stdin, s], [], [])
    if sys.stdin in readable:
        data = sys.stdin.read(1024)
        if not data:
            break
        s.sendto(data, 0, peer)
    elif s in readable:
        data = s.recv(1024)
        if not data:
            break
        if data == MAGIC:
            MAGIC = None
        else:
            sys.stdout.write(data)
<commit_msg>Add usage to nat-birthday.py.<commit_after>#!/usr/bin/env python

import fcntl
import getopt
import os
import random
import select
import socket
import sys

def usage(f = sys.stdout):
    print >> f, """\
Usage: %s [-a|-b] [OPTIONS] REMOTE
UDP NAT traversal based on randomly generating port numbers. Run with
-a on one host, then with -b on the other.
  -a       bind to random local ports, connect to static remote port.
  -b       bind to static local port, connect to random remote ports.
  -h       show this help.
  -n N     let N packet be outstanding at once (default %d).
  -p PORT  use PORT as the static port (default %d).\
""" % (sys.argv[0], N, PORT)

MAGIC = "MAGIC_STREAM_START"

SIDE = None
PORT = 2000
N = 500
remote = None
DELAY = 0.1

def unblock(fd):
    flags = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)

def main_a(remote):
    ports = set()
    rset = []

    while True:
        for i in range(N):
            while True:
                r = random.randrange(1024, 65536)
                if r not in ports:
                    break
            ports.add(r)
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
            s.bind(("0.0.0.0", r))
            print r
            addr = (remote, PORT)
            s.sendto("hello", 0, addr)
            rset.append(s)
        print "waiting 20 seconds..."
        readable, _, _ = select.select(rset, [], [], 20)
        for fd in readable:
            data, peer = fd.recvfrom(512)
            if data:
                return fd, peer
        while rset:
            rset.pop().close()

def main_b(remote):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
    s.bind(("0.0.0.0", PORT))

    i = 0
    while True:
        if i >= N:
            readable, _, _ = select.select([s], [], [], DELAY)
            for fd in readable:
                data, peer = fd.recvfrom(512)
                if data:
                    return fd, peer
        else:
            i += 1
        port = random.randrange(1024, 65536)
        addr = (remote, port)
        print addr
        s.sendto("hello", 0, addr)

opts, args = getopt.gnu_getopt(sys.argv[1:], "abhn:p:")
for o, a in opts:
    if o == "-a":
        SIDE = 0
    elif o == "-b":
        SIDE = 1
    elif o == "-h":
        usage()
        sys.exit()
    elif o == "-n":
        N = int(a)
    elif o == "-p":
        PORT = int(a)

try:
    remote_hostname, = args
except ValueError:
    usage(sys.stderr)
    sys.exit(1)

addrinfo = socket.getaddrinfo(remote_hostname, 0, socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
print addrinfo

if SIDE is None:
    usage(sys.stderr)
    sys.exit(1)
elif SIDE == 0:
    s, peer = main_a(addrinfo[0][4][0])
else:
    s, peer = main_b(addrinfo[0][4][0])

print "got connection", peer

s.sendto(MAGIC, 0, peer)

unblock(sys.stdin)

print "start typing"

while True:
    readable, _, _ = select.select([sys.stdin, s], [], [])
    if sys.stdin in readable:
        data = sys.stdin.read(1024)
        if not data:
            break
        s.sendto(data, 0, peer)
    elif s in readable:
        data = s.recv(1024)
        if not data:
            break
        if data == MAGIC:
            MAGIC = None
        else:
            sys.stdout.write(data)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

from django.core.exceptions import ImproperlyConfigured
from django.utils.translation import ugettext_lazy as _

from cms.toolbar_base import CMSToolbar
from cms.toolbar_pool import toolbar_pool
from cms.utils.urlutils import admin_reverse

from aldryn_apphooks_config.utils import get_app_instance
from .models import Article
from .cms_appconfig import NewsBlogConfig

@toolbar_pool.register
class NewsBlogToolbar(CMSToolbar):
    watch_models = (Article, )
    supported_apps = ('aldryn_newsblog',)

    def __get_newsblog_config(self):
        try:
            __, config = get_app_instance(self.request)
            if not isinstance(config, NewsBlogConfig):
                # This is not the app_hook you are looking for.
                return None
        except ImproperlyConfigured:
            # There is no app_hook at all.
            return None

        return config

    def populate(self):
        config = self.__get_newsblog_config()
        if not config:
            # Do nothing if there is no NewsBlog app_config to work with
            return

        user = getattr(self.request, 'user', None)

        menu = self.toolbar.get_or_create_menu('newsblog-app', config.get_app_title())

        if self.request.user.has_perm('aldryn_newsblog.change_newsblogconfig'):
            menu.add_modal_item(
                _('Configure application'),
                url=admin_reverse(
                    'aldryn_newsblog_newsblogconfig_change',
                    args=(config.pk, )
                ),
            )

        if self.request.user.has_perm('aldryn_newsblog.change_article'):
            menu.add_sideframe_item(
                _('Article list'),
                url="{base}?app_config__id__exact={config}".format(
                    base=admin_reverse('aldryn_newsblog_article_changelist'),
                    config=config.pk,
                ),
            )

        if self.request.user.has_perm('aldryn_newsblog.add_article'):
            menu.add_modal_item(
                _('Add new article'),
                url="{base}?app_config={config}&owner={owner}".format(
                    base=admin_reverse('aldryn_newsblog_article_add'),
                    config=config.pk,
                    owner=user.pk,
                ),
            )

        view_name = self.request.resolver_match.view_name
        if (view_name == '{0}:article-detail'.format(config.namespace) and
                self.request.user.has_perm('aldryn_newsblog.change_article')):
            kwargs = self.request.resolver_match.kwargs
            articles = Article.objects
            if hasattr(kwargs, 'slug'):
                slug = kwargs['slug']
                articles = articles.translated(slug=slug)
            elif hasattr(kwargs, 'pk'):
                pk = kwargs['pk']
                articles = articles.filter(pk=pk)
            if articles.count() == 1:
                menu.add_modal_item(_('Edit article'), admin_reverse(
                    'aldryn_newsblog_article_change', args=(
                        articles[0].pk,)), active=True,)
<commit_msg>Minor refactoring<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

from django.core.exceptions import ImproperlyConfigured
from django.utils.translation import ugettext_lazy as _

from cms.toolbar_base import CMSToolbar
from cms.toolbar_pool import toolbar_pool
from cms.utils.urlutils import admin_reverse

from aldryn_apphooks_config.utils import get_app_instance
from .models import Article
from .cms_appconfig import NewsBlogConfig

@toolbar_pool.register
class NewsBlogToolbar(CMSToolbar):
    watch_models = (Article, )
    supported_apps = ('aldryn_newsblog',)

    def __get_newsblog_config(self):
        try:
            __, config = get_app_instance(self.request)
            if not isinstance(config, NewsBlogConfig):
                # This is not the app_hook you are looking for.
                return None
        except ImproperlyConfigured:
            # There is no app_hook at all.
            return None

        return config

    def populate(self):
        config = self.__get_newsblog_config()
        if not config:
            # Do nothing if there is no NewsBlog app_config to work with
            return

        user = getattr(self.request, 'user', None)

        menu = self.toolbar.get_or_create_menu('newsblog-app', config.get_app_title())

        if self.request.user.has_perm('aldryn_newsblog.change_newsblogconfig'):
            menu.add_modal_item(
                _('Configure application'),
                url=admin_reverse(
                    'aldryn_newsblog_newsblogconfig_change',
                    args=(config.pk, )
                ),
            )

        if self.request.user.has_perm('aldryn_newsblog.change_article'):
            menu.add_sideframe_item(
                _('Article list'),
                url="{base}?app_config__id__exact={config}".format(
                    base=admin_reverse('aldryn_newsblog_article_changelist'),
                    config=config.pk,
                ),
            )

        if self.request.user.has_perm('aldryn_newsblog.add_article'):
            menu.add_modal_item(
                _('Add new article'),
                url="{base}?app_config={config}&owner={owner}".format(
                    base=admin_reverse('aldryn_newsblog_article_add'),
                    config=config.pk,
                    owner=user.pk,
                ),
            )

        view_name = self.request.resolver_match.view_name
        if (view_name == '{0}:article-detail'.format(config.namespace) and
                self.request.user.has_perm('aldryn_newsblog.change_article')):
            kwargs = self.request.resolver_match.kwargs
            articles = Article.objects
            if hasattr(kwargs, 'slug'):
                articles = articles.translated(slug=kwargs['slug'])
            elif hasattr(kwargs, 'pk'):
                articles = articles.filter(pk=kwargs['pk'])
            articles = articles.all()
            if articles.count():
                menu.add_modal_item(_('Edit article'), admin_reverse(
                    'aldryn_newsblog_article_change', args=(
                        articles[0].pk,)), active=True,)
<|endoftext|>


<commit_before>"""
Copyright (c) 2013 Algolia
http://www.algolia.com/
Thanks to Arthur Lenoir <arthur (at) lenoir.me> for the initial version.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""
import json
import random
import urllib
import urllib3
import time

POOL_MANAGER = urllib3.PoolManager()

# Exception launched by Algolia Client when an error occured
class AlgoliaException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

"""
Entry point in the Python API.
You should instanciate a Client object with your ApplicationID, ApiKey and Hosts
to start using Algolia Search API
"""
class Client:
    """
    Algolia Search library initialization
    @param applicationID the application ID you have in your admin interface
    @param apiKey a valid API key for the service
    @param hostsArray the list of hosts that you have received for the service
    """
    def __init__(self, applicationID, apiKey, hostsArray):
        self.hosts = hostsArray
        random.shuffle(self.hosts)
        self.applicationID = applicationID
        self.apiKey = apiKey
        self.headers = {
            'Content-Type': 'application/json; charset=utf-8',
            'X-Algolia-API-Key': self.apiKey,
            'X-Algolia-Application-Id': self.applicationID
        }

    """
    List all existing indexes
    return an Answer object with answer in the form
       {"items": [{ "name": "contacts", "createdAt": "2013-01-18T15:33:13.556Z"},
                  {"name": "notes", "createdAt": "2013-01-18T15:33:13.556Z"}]}
    """
    def listIndexes(self):
        return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/")

    """
    Delete an index

    @param indexName the name of index to delete
    Return an Answer object whith answer in the form {"deletedAt": "2013-01-18T15:33:13.556Z"}
    """
    def deleteIndex(self, indexName):
        return AlgoliaUtils_request(self.headers, self.hosts, "DELETE", "/1/indexes/%s" % urllib.quote(indexName))

    """
    Get the index object initialized (no server call needed for initialization)

    @param indexName the name of index
    """
    def initIndex(self, indexName):
        return Index(self.headers, self.hosts, indexName)

    """
    List all existing user keys with their associated ACLs
    """
    def listUserKeys(self):
        return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/keys")

    # Get ACL of a user key
    def getUserKeyACL(self, key):
        return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/keys/%s" % key)

    # Delete an existing user key
    def deleteUserKey(self, key):
        return AlgoliaUtils_request(self.headers, self.hosts, "DELETE", "/1/keys/%s" % key)

    """
    Create a new user key

    @param acls the list of ACL for this key. Defined by an array of strings that
           can contains the following values:
             - search: allow to search (https and http)
             - addObject: allows to add a new object in the index (https only)
             - updateObject : allows to change content of an existing object (https only)
             - deleteObject : allows to delete an existing object (https only)
             - deleteIndex : allows to delete index content (https only)
             - settings : allows to get index settings (https only)
             - editSettings : allows to change index settings (https only)
    """
    def addUserKey(self, acls):
        return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/keys", {"acl": acls} )

"""
Contains all the functions related to one index
You should use Client.initIndex(indexName) to retrieve this object
"""
class Index:
    def __init__(self, headers, hosts, indexName):
        self.hosts = hosts
        self.headers = headers
        self.indexName = indexName
        self.urlIndexName = urllib.quote(self.indexName)

    """
    Add an object in this index

    @param content contains the object to add inside the index.
      The object is represented by an associative array
    @param objectID (optional) an objectID you want to attribute to this object
      (if the attribute already exist the old object will be overwrite)
    """
    def addObject(self, content, objectID = None):
        if objectID is None:
            return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/indexes/%s" % self.urlIndexName, content)
        else:
            return AlgoliaUtils_request(self.headers, self.hosts, "PUT", "/1/indexes/%s/%s" % (self.urlIndexName, urllib.quote(objectID)), content)

    """
    Add several objects

    @param objects contains an array of objects to add
    """
    def addObjects(self, objects):
        requests = []
        for obj in objects:
            requests.append({"action": "addObject", "body": obj})
        request = {"requests": requests}
        return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/indexes/%s/batch" % self.urlIndexName, request)

    """
    Get an object from this index

    @param objectID the unique identifier of the object to retrieve
    @param attributesToRetrieve (optional) if set, contains the list of attributes to retrieve as a string separated by a comma
    """
    def getObject(self, objectID, attributesToRetrieve = None):
        objID = urllib.quote(objectID)
        if (attributesToRetrieve == None):
            return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s/%s" % (self.urlIndexName, objID))
        else:
            return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s/%s?attributes=%s" % (self.urlIndexName, objID, attributesToRetrieve))

    """
    Update partially an object (only update attributes passed in argument)

    @param partialObject contains the javascript attributes to override, the
           object must contains an objectID attribute
    """
    def partialUpdateObject(self, partialObject):
        return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/indexes/%s/%s/partial" % (self.urlIndexName, urllib.quote(partialObject["objectID"])), partialObject)

    """
    Override the content of object

    @param object contains the javascript object to save, the object must contains an objectID attribute
    """
    def saveObject(self, obj):
        return AlgoliaUtils_request(self.headers, self.hosts, "PUT", "/1/indexes/%s/%s" % (self.urlIndexName, urllib.quote(obj["objectID"])), obj)

    """
    Override the content of several objects

    @param objects contains an array of objects to update (each object must contains a objectID attribute)
    """
    def saveObjects(self, objects):
        requests = []
        for obj in objects:
            requests.append({"action": "updateObject", "objetID": obj["objectID"], "body": obj})
        request = {"requests": requests}
        return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/indexes/%s/batch" % self.urlIndexName, request)

    """
    Delete an object from the index

    @param objectID the unique identifier of object to delete
    """
    def deleteObject(self, objectID):
        return AlgoliaUtils_request(self.headers, self.hosts, "DELETE", "/1/indexes/%s/%s" % (self.urlIndexName, urllib.quote(objectID)))

    """
    Search inside the index

    @param query the full text query
    @param args (optional) if set, contains an associative array with query parameters:
      - attributes: a string that contains attribute names to retrieve separated by a comma.
        By default all attributes are retrieved.
      - attributesToHighlight: a string that contains attribute names to highlight separated by a comma.
        By default indexed attributes are highlighted.
      - minWordSizeForApprox1: the minimum number of characters in a query word to accept one typo in this word.
        Defaults to 3.
      - minWordSizeForApprox2: the minimum number of characters in a query word to accept two typos in this word.
         Defaults to 7.
      - getRankingInfo: if set to 1, the result hits will contain ranking information in
         _rankingInfo attribute
      - page: (pagination parameter) page to retrieve (zero base). Defaults to 0.
      - hitsPerPage: (pagination parameter) number of hits per page. Defaults to 10.
      - aroundLatLng let you search for entries around a given latitude/longitude (two float separated
        by a ',' for example aroundLatLng=47.316669,5.016670).
        You can specify the maximum distance in meters with aroundRadius parameter (in meters).
        At indexing, geoloc of an object should be set with _geoloc attribute containing lat and lng attributes (for example {"_geoloc":{"lat":48.853409, "lng":2.348800}})
      - insideBoundingBox let you search entries inside a given area defined by the two extreme points of
        a rectangle (defined by 4 floats: p1Lat,p1Lng,p2Lat, p2Lng.
        For example insideBoundingBox=47.3165,4.9665,47.3424,5.0201).
        At indexing, geoloc of an object should be set with _geoloc attribute containing lat and lng attributes (for example {"_geoloc":{"lat":48.853409, "lng":2.348800}})
      - tags let you filter the query by a set of tags (contains a list of tags separated by ',').
        At indexing, tags should be added in _tags attribute of objects (for example {"_tags":["tag1","tag2"]} )
    """
    def search(self, query, args = None):
        if args == None:
            return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s?query=%s" % (self.urlIndexName, urllib.quote(query)))
        else:
            return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s?query=%s&%s" % (self.urlIndexName, urllib.quote(query), urllib.urlencode(args)))

    """
    Wait the publication of a task on the server.
    All server task are asynchronous and you can check with this method that the task is published.

    @param taskID the id of the task returned by server
    @param timeBeforeRetry the time in milliseconds before retry (default = 100ms)
    """
    def waitTask(self, taskID, timeBeforeRetry = 100):
        while True:
            res = AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s/task/%d/" % (self.urlIndexName, taskID))
            if res.hasError():
                return res;
            content = res.getContent()
            if (content["status"] == "published"):
                return res
            time.sleep(timeBeforeRetry / 1000)

    # Get settings of this index
    def getSettings(self):
        return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s/settings" % self.urlIndexName)

    """
      Set settings for this index

      @param settigns the settings object that can contains :
       - minWordSizeForApprox1 (integer) the minimum number of characters to accept one typo (default = 3)
       - minWordSizeForApprox2: (integer) the minimum number of characters to accept two typos (default = 7)
       - hitsPerPage: (integer) the number of hits per page (default = 10)
       - attributesToRetrieve: (array of strings) default list of attributes to retrieve for objects
       - attributesToHighlight: (array of strings) default list of attributes to highlight
       - attributesToIndex: (array of strings) the list of fields you want to index.
         By default all textual attributes of your objects are indexed, but you should update it to get optimal
         results. This parameter has two important uses:
            - Limit the attributes to index.
              For example if you store a binary image in base64, you want to store it in the index but you
              don't want to use the base64 string for search.
            - Control part of the ranking (see the ranking parameter for full explanation).
              Matches in attributes at the beginning of the list will be considered more important than matches
              in attributes further down the list.
       - ranking: (array of strings) controls the way results are sorted.
          We have four available criteria:
            - typo (sort according to number of typos),
            - geo: (sort according to decreassing distance when performing a geo-location based search),
            - position (sort according to the matching attribute),
            - custom which is user defined
          (the standard order is ["typo", "geo", position", "custom"])
       - customRanking: (array of strings) lets you specify part of the ranking.
         The syntax of this condition is an array of strings containing attributes prefixed
         by asc (ascending order) or desc (descending order) operator.
     """
    def setSettings(self, settings):
        return AlgoliaUtils_request(self.headers, self.hosts, "PUT", "/1/indexes/%s/settings" % self.urlIndexName, settings)

# Util function used to send request
def AlgoliaUtils_request(headers, hosts, method, request, body = None):
    for host in hosts:
        try:
            obj = None
            if body != None:
                obj = json.dumps(body)
            conn = POOL_MANAGER.connection_from_host(host, scheme = 'https')
            response = conn.urlopen(method, request, headers = headers, body = obj)
            content = json.loads(response.data)
            if response.status == 400:
                raise AlgoliaException(content["message"])
            elif response.status == 403:
                raise AlgoliaException("Invalid Application-ID or API-Key")
            elif response.status == 404:
                raise AlgoliaException("Resource does not exist")
            elif response.status == 200:
                return json.loads(response.data)
        except AlgoliaException, e:
            raise e
        except:
            pass
    raise AlgoliaException("Unreachable host")

<commit_msg>Fixed typos<commit_after>"""
Copyright (c) 2013 Algolia
http://www.algolia.com/
Thanks to Arthur Lenoir <arthur (at) lenoir.me> for the initial version.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""
import json
import random
import urllib
import urllib3
import time

POOL_MANAGER = urllib3.PoolManager()

# Exception launched by Algolia Client when an error occured
class AlgoliaException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

"""
Entry point in the Python API.
You should instanciate a Client object with your ApplicationID, ApiKey and Hosts
to start using Algolia Search API
"""
class Client:
    """
    Algolia Search library initialization
    @param applicationID the application ID you have in your admin interface
    @param apiKey a valid API key for the service
    @param hostsArray the list of hosts that you have received for the service
    """
    def __init__(self, applicationID, apiKey, hostsArray):
        self.hosts = hostsArray
        random.shuffle(self.hosts)
        self.applicationID = applicationID
        self.apiKey = apiKey
        self.headers = {
            'Content-Type': 'application/json; charset=utf-8',
            'X-Algolia-API-Key': self.apiKey,
            'X-Algolia-Application-Id': self.applicationID
        }

    """
    List all existing indexes
    return an object of the form
       {"items": [{ "name": "contacts", "createdAt": "2013-01-18T15:33:13.556Z"},
                  {"name": "notes", "createdAt": "2013-01-18T15:33:13.556Z"}]}
    """
    def listIndexes(self):
        return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/")

    """
    Delete an index

    @param indexName the name of index to delete
    Return an object of the form {"deletedAt": "2013-01-18T15:33:13.556Z"}
    """
    def deleteIndex(self, indexName):
        return AlgoliaUtils_request(self.headers, self.hosts, "DELETE", "/1/indexes/%s" % urllib.quote(indexName))

    """
    Get the index object initialized (no server call needed for initialization)

    @param indexName the name of index
    """
    def initIndex(self, indexName):
        return Index(self.headers, self.hosts, indexName)

    """
    List all existing user keys with their associated ACLs
    """
    def listUserKeys(self):
        return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/keys")

    # Get ACL of a user key
    def getUserKeyACL(self, key):
        return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/keys/%s" % key)

    # Delete an existing user key
    def deleteUserKey(self, key):
        return AlgoliaUtils_request(self.headers, self.hosts, "DELETE", "/1/keys/%s" % key)

    """
    Create a new user key

    @param acls the list of ACL for this key. Defined by an array of strings that
           can contains the following values:
             - search: allow to search (https and http)
             - addObject: allows to add a new object in the index (https only)
             - updateObject : allows to change content of an existing object (https only)
             - deleteObject : allows to delete an existing object (https only)
             - deleteIndex : allows to delete index content (https only)
             - settings : allows to get index settings (https only)
             - editSettings : allows to change index settings (https only)
    """
    def addUserKey(self, acls):
        return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/keys", {"acl": acls} )

"""
Contains all the functions related to one index
You should use Client.initIndex(indexName) to retrieve this object
"""
class Index:
    def __init__(self, headers, hosts, indexName):
        self.hosts = hosts
        self.headers = headers
        self.indexName = indexName
        self.urlIndexName = urllib.quote(self.indexName)

    """
    Add an object in this index

    @param content contains the object to add inside the index.
      The object is represented by an associative array
    @param objectID (optional) an objectID you want to attribute to this object
      (if the attribute already exist the old object will be overwrite)
    """
    def addObject(self, content, objectID = None):
        if objectID is None:
            return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/indexes/%s" % self.urlIndexName, content)
        else:
            return AlgoliaUtils_request(self.headers, self.hosts, "PUT", "/1/indexes/%s/%s" % (self.urlIndexName, urllib.quote(objectID)), content)

    """
    Add several objects

    @param objects contains an array of objects to add
    """
    def addObjects(self, objects):
        requests = []
        for obj in objects:
            requests.append({"action": "addObject", "body": obj})
        request = {"requests": requests}
        return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/indexes/%s/batch" % self.urlIndexName, request)

    """
    Get an object from this index

    @param objectID the unique identifier of the object to retrieve
    @param attributesToRetrieve (optional) if set, contains the list of attributes to retrieve as a string separated by a comma
    """
    def getObject(self, objectID, attributesToRetrieve = None):
        objID = urllib.quote(objectID)
        if (attributesToRetrieve == None):
            return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s/%s" % (self.urlIndexName, objID))
        else:
            return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s/%s?attributes=%s" % (self.urlIndexName, objID, attributesToRetrieve))

    """
    Update partially an object (only update attributes passed in argument)

    @param partialObject contains the object attributes to override, the
           object must contains an objectID attribute
    """
    def partialUpdateObject(self, partialObject):
        return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/indexes/%s/%s/partial" % (self.urlIndexName, urllib.quote(partialObject["objectID"])), partialObject)

    """
    Override the content of object

    @param object contains the object to save, the object must contains an objectID attribute
    """
    def saveObject(self, obj):
        return AlgoliaUtils_request(self.headers, self.hosts, "PUT", "/1/indexes/%s/%s" % (self.urlIndexName, urllib.quote(obj["objectID"])), obj)

    """
    Override the content of several objects

    @param objects contains an array of objects to update (each object must contains a objectID attribute)
    """
    def saveObjects(self, objects):
        requests = []
        for obj in objects:
            requests.append({"action": "updateObject", "objetID": obj["objectID"], "body": obj})
        request = {"requests": requests}
        return AlgoliaUtils_request(self.headers, self.hosts, "POST", "/1/indexes/%s/batch" % self.urlIndexName, request)

    """
    Delete an object from the index

    @param objectID the unique identifier of object to delete
    """
    def deleteObject(self, objectID):
        return AlgoliaUtils_request(self.headers, self.hosts, "DELETE", "/1/indexes/%s/%s" % (self.urlIndexName, urllib.quote(objectID)))

    """
    Search inside the index

    @param query the full text query
    @param args (optional) if set, contains an associative array with query parameters:
      - attributes: a string that contains attribute names to retrieve separated by a comma.
        By default all attributes are retrieved.
      - attributesToHighlight: a string that contains attribute names to highlight separated by a comma.
        By default indexed attributes are highlighted.
      - minWordSizeForApprox1: the minimum number of characters in a query word to accept one typo in this word.
        Defaults to 3.
      - minWordSizeForApprox2: the minimum number of characters in a query word to accept two typos in this word.
         Defaults to 7.
      - getRankingInfo: if set to 1, the result hits will contain ranking information in
         _rankingInfo attribute
      - page: (pagination parameter) page to retrieve (zero base). Defaults to 0.
      - hitsPerPage: (pagination parameter) number of hits per page. Defaults to 10.
      - aroundLatLng let you search for entries around a given latitude/longitude (two float separated
        by a ',' for example aroundLatLng=47.316669,5.016670).
        You can specify the maximum distance in meters with aroundRadius parameter (in meters).
        At indexing, geoloc of an object should be set with _geoloc attribute containing lat and lng attributes (for example {"_geoloc":{"lat":48.853409, "lng":2.348800}})
      - insideBoundingBox let you search entries inside a given area defined by the two extreme points of
        a rectangle (defined by 4 floats: p1Lat,p1Lng,p2Lat, p2Lng.
        For example insideBoundingBox=47.3165,4.9665,47.3424,5.0201).
        At indexing, geoloc of an object should be set with _geoloc attribute containing lat and lng attributes (for example {"_geoloc":{"lat":48.853409, "lng":2.348800}})
      - tags filter the query by a set of tags. You can AND tags by separating them by commas. To OR tags, you must add parentheses. For example, tags=tag1,(tag2,tag3) means tag1 AND (tag2 OR tag3).
        At indexing, tags should be added in the _tags attribute of objects (for example {"_tags":["tag1","tag2"]} )

    """
    def search(self, query, args = None):
        if args == None:
            return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s?query=%s" % (self.urlIndexName, urllib.quote(query)))
        else:
            return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s?query=%s&%s" % (self.urlIndexName, urllib.quote(query), urllib.urlencode(args)))

    """
    Wait the publication of a task on the server.
    All server task are asynchronous and you can check with this method that the task is published.

    @param taskID the id of the task returned by server
    @param timeBeforeRetry the time in milliseconds before retry (default = 100ms)
    """
    def waitTask(self, taskID, timeBeforeRetry = 100):
        while True:
            res = AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s/task/%d/" % (self.urlIndexName, taskID))
            if res.hasError():
                return res;
            content = res.getContent()
            if (content["status"] == "published"):
                return res
            time.sleep(timeBeforeRetry / 1000)

    # Get settings of this index
    def getSettings(self):
        return AlgoliaUtils_request(self.headers, self.hosts, "GET", "/1/indexes/%s/settings" % self.urlIndexName)

    """
      Set settings for this index

      @param settigns the settings object that can contains :
       - minWordSizeForApprox1 (integer) the minimum number of characters to accept one typo (default = 3)
       - minWordSizeForApprox2: (integer) the minimum number of characters to accept two typos (default = 7)
       - hitsPerPage: (integer) the number of hits per page (default = 10)
       - attributesToRetrieve: (array of strings) default list of attributes to retrieve for objects
       - attributesToHighlight: (array of strings) default list of attributes to highlight
       - attributesToIndex: (array of strings) the list of fields you want to index.
         By default all textual attributes of your objects are indexed, but you should update it to get optimal
         results. This parameter has two important uses:
            - Limit the attributes to index.
              For example if you store a binary image in base64, you want to store it in the index but you
              don't want to use the base64 string for search.
            - Control part of the ranking (see the ranking parameter for full explanation).
              Matches in attributes at the beginning of the list will be considered more important than matches
              in attributes further down the list.
       - ranking: (array of strings) controls the way results are sorted.
          We have four available criteria:
            - typo (sort according to number of typos),
            - geo: (sort according to decreassing distance when performing a geo-location based search),
            - position (sort according to the matching attribute),
            - custom which is user defined
          (the standard order is ["typo", "geo", position", "custom"])
       - customRanking: (array of strings) lets you specify part of the ranking.
         The syntax of this condition is an array of strings containing attributes prefixed
         by asc (ascending order) or desc (descending order) operator.
     """
    def setSettings(self, settings):
        return AlgoliaUtils_request(self.headers, self.hosts, "PUT", "/1/indexes/%s/settings" % self.urlIndexName, settings)

# Util function used to send request
def AlgoliaUtils_request(headers, hosts, method, request, body = None):
    for host in hosts:
        try:
            obj = None
            if body != None:
                obj = json.dumps(body)
            conn = POOL_MANAGER.connection_from_host(host, scheme = 'https')
            response = conn.urlopen(method, request, headers = headers, body = obj)
            content = json.loads(response.data)
            if response.status == 400:
                raise AlgoliaException(content["message"])
            elif response.status == 403:
                raise AlgoliaException("Invalid Application-ID or API-Key")
            elif response.status == 404:
                raise AlgoliaException("Resource does not exist")
            elif response.status == 200:
                return json.loads(response.data)
        except AlgoliaException, e:
            raise e
        except:
            pass
    raise AlgoliaException("Unreachable host")

<|endoftext|>