<commit_before>from django.db import models
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
from django.utils.translation import get_language

def get_dataset_upload_path(instance, filename):
    username = 'anonymous'
    if instance.user:
        username = instance.user.username
    return  '%s/datasets/%s' % (username, filename)

def type_validator(value):
    pass

class Dataset(models.Model):
    title = models.CharField(_('Title'), max_length=255)
    file = models.FileField(_('File'), upload_to=get_dataset_upload_path, validators=[type_validator])
    description = models.TextField(_('Description'), blank=True, null=True)
    user = models.ForeignKey(User, blank=True, null=True, verbose_name=_('User'))
    updated = models.DateTimeField(_('Updated'), auto_now=True, blank=True, null=True)
    created = models.DateTimeField(_('Created'), auto_now_add=True, blank=True, null=True)

    def get_absolute_url(self):
        return reverse('dataset-list')

    def __unicode__(self):
        return self.title

def get_algorithm_file_upload_path(instance, filename):
    username = 'anonymous'
    if instance.user:
        username = instance.user.username
    return  '%s/algorithms/%s' % (username, filename)

class Cluster(models.Model):
    title = models.CharField(_('Title EN'), max_length=255, null=True)
    title_lt = models.CharField(_('Title LT'), max_length=255, null=True)
    url = models.URLField(_('Cluster home page'), max_length=255, null=True, blank=True)
    workload_url = models.URLField(_('Workload page'), max_length=255, null=True)
    description = models.TextField(_('Description EN'), null=True, blank=True)
    description_lt = models.TextField(_('Description LT'), null=True, blank=True)

    def __unicode__(self):
        return self.title

    def get_title_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.title_lt
        else:
            return self.title

    def get_description_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.description_lt
        else:
            return self.description

    def get_tooltip_html(self):
        return _('
{0}

Cluster home:

{1}

Cluster workload:

{2}
').format(
                    self.get_description_display(),
                    self.url,
                    self.workload_url
                )

class Component(models.Model):
    CATEGORIES= (
        ('DATA', _('Upload data')),
        ('PREPROCESSING', _('Preprocessing')),
        ('STAT_PRIMITIVES', _('Statistical primitives')),
        ('DIMENSION_REDUCTION', _('Dimension reduction')),
        ('DATA_MINING', _('Classification, grouping')),
        ('RESULTS', _('View results')),
    )
    title = models.CharField(_('Title'), max_length=255, null=True, blank=True)
    user = models.ForeignKey(User, blank=True, null=True, verbose_name=_('User'))
    category = models.CharField(_('Category'), max_length=255, null=True, blank=True, choices=CATEGORIES)
    wsdl = models.URLField(_('WSDL link'), null=True, blank=True)
    function = models.CharField(_('WSDL function name'), max_length=255, null=True, blank=True)
    updated = models.DateTimeField(_('Updated'), auto_now=True, blank=True, null=True)
    created = models.DateTimeField(_('Created'), auto_now_add=True, blank=True, null=True)
    icon = models.ImageField(_('Icon'), upload_to='icons', blank=True, null=True)
    cluster = models.ForeignKey('Cluster', null=True, verbose_name=_('Cluster'))
    label = models.CharField(_('Label EN'), max_length=255, null=True, blank=True)
    label_lt = models.CharField(_('Label LT'), max_length=255, null=True, blank=True)
    description = models.TextField(_('Description EN'), null=True, blank=True)
    description_lt = models.TextField(_('Description LT'), null=True, blank=True)

    def get_absolute_url(self):
        return reverse('algorithm-list')

    def __unicode__(self):
        return str(self.title)

    def get_label_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.label_lt
        else:
            return self.label

    def get_description_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.description_lt
        else:
            return self.description

class Parameter(models.Model):
    CONNECTION_TYPES = (
        ('INPUT_VALUE', 'Input value'),
        ('INPUT_CONNECTION', 'Input connection'),
        ('INPUT_COMMON', 'Input common'),
        ('OUTPUT_VALUE', 'Output value'),
        ('OUTPUT_CONNECTION', 'Output connection'),
    )
    VALIDATORS = (
        ('dataset', _('Dataset')),
        ('int', _('Integer')),
        ('float', _('Float')),
        ('double', _('Double')),
        ('string', _('String')),
        ('text', _('Text')),
        ('boolean', _('Boolean')),
        ('date', _('Date')),
        ('datetime', _('Date and time')),
        ('time', _('Time')),
    )
    algorithm = models.ForeignKey(Component, related_name='parameters',
                                  null=True, blank=True,
                                  verbose_name=_('Component'))
    name = models.CharField(_('Option'), max_length=255, null=True)
    type = models.CharField(_('Type'), choices=VALIDATORS , max_length=255, null=True, blank=True)
    required = models.BooleanField(_('Required'), blank=True)
    default = models.CharField(_('Default'), max_length=255, null=True, blank=True)
    connection_type = models.CharField(_('Connection type'), max_length=255, null=True, blank=True,
                              choices=CONNECTION_TYPES, default='INPUT_VALUE')
    label = models.CharField(_('Label EN'), max_length=255, null=True, blank=True)
    label_lt = models.CharField(_('Label LT'), max_length=255, null=True, blank=True)
    description = models.TextField(_('Description EN'), null=True, blank=True)
    description_lt = models.TextField(_('Description LT'), null=True, blank=True)

    def __unicode__(self):
        return '%s (%s)' % (self.name, self.type)

    def get_label_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.label_LT
        else:
            return self.label

    def get_description_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.description_LT
        else:
            return self.description

class Experiment(models.Model):
    STATUSES = (
        ('SAVED', _('Saved')),
        ('RUNNING', _('Running')),
        ('ERROR', _('Error')),
        ('FINISHED', _('Finished')),
    )
    title = models.CharField(_('Experiment title'), max_length=255, null=True)
    start = models.DateTimeField(_('Start'), blank=True, null=True)
    finish = models.DateTimeField(_('Finish'), blank=True, null=True)
    status = models.CharField(_('Status'), max_length=255, null=True, blank=True,
                              choices=STATUSES, default='SAVED')
    user = models.ForeignKey(User, blank=True, null=True, verbose_name=_('User'), related_name='experiments')
    workflow_state = models.TextField(_('Workflow state'), blank=True, null=True)

    max_calc_time = models.TimeField(_('Maximum calculation time'),
                                     default="2:00", null=True)
    p = models.IntegerField(_('Processor number'), default=1, null=True)

    updated = models.DateTimeField(_('Updated'), auto_now=True, blank=True, null=True)
    created = models.DateTimeField(_('Created'), auto_now_add=True, blank=True, null=True)

    def get_absolute_url(self):
        return reverse('experiment-list')

    def __unicode__(self):
        return str(self.title)

def get_result_file_upload_path(instance, filename):
    username = 'anonymous'
    if instance.algorithm.user:
        username = instance.algorithm.user.username
    return  '%s/result/%s' % (username, filename)

class WorkflowTask(models.Model):
    STATUSES = (
        ('SAVED', _('Saved')),
        ('RUNNING', _('Running')),
        ('ERROR', _('Error')),
        ('FINISHED', _('Finished')),
    )
    experiment = models.ForeignKey('Experiment', related_name='tasks',
                                   null=True, verbose_name=_('Experiment'))
    algorithm = models.ForeignKey('Component', verbose_name=_('Component'))
    status = models.CharField(_('Status'), max_length=255, null=True, blank=True,
                              choices=STATUSES, default='SAVED')

    def __unicode__(self):
        return '%s' % (self.algorithm.title,)

class ParameterValue(models.Model):
    parameter = models.ForeignKey('Parameter', verbose_name=_('Parameter'))
    value = models.CharField(max_length=255, blank=True, null=True)
    task = models.ForeignKey('WorkflowTask', related_name='parameter_values',
                                             verbose_name=_('Workflow Task'))
    related = models.ManyToManyField('self', symmetrical=False, null=True,
                                     blank=True, through='Connection')

    def __unicode__(self):
        return '%s: %s' % (self.task.algorithm, self.value)

class Connection(models.Model):
    source = models.ForeignKey('ParameterValue', null=True, blank=True,
                             verbose_name=_('Source'), related_name='source')
    target = models.ForeignKey('ParameterValue', null=True, blank=True,
                             verbose_name=_('Target'), related_name='target')
<commit_msg>Fix Parameter get_description_dispaly and get_label_dispaly methods for Lithuanian language.<commit_after>from django.db import models
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
from django.utils.translation import get_language

def get_dataset_upload_path(instance, filename):
    username = 'anonymous'
    if instance.user:
        username = instance.user.username
    return  '%s/datasets/%s' % (username, filename)

def type_validator(value):
    pass

class Dataset(models.Model):
    title = models.CharField(_('Title'), max_length=255)
    file = models.FileField(_('File'), upload_to=get_dataset_upload_path, validators=[type_validator])
    description = models.TextField(_('Description'), blank=True, null=True)
    user = models.ForeignKey(User, blank=True, null=True, verbose_name=_('User'))
    updated = models.DateTimeField(_('Updated'), auto_now=True, blank=True, null=True)
    created = models.DateTimeField(_('Created'), auto_now_add=True, blank=True, null=True)

    def get_absolute_url(self):
        return reverse('dataset-list')

    def __unicode__(self):
        return self.title

def get_algorithm_file_upload_path(instance, filename):
    username = 'anonymous'
    if instance.user:
        username = instance.user.username
    return  '%s/algorithms/%s' % (username, filename)

class Cluster(models.Model):
    title = models.CharField(_('Title EN'), max_length=255, null=True)
    title_lt = models.CharField(_('Title LT'), max_length=255, null=True)
    url = models.URLField(_('Cluster home page'), max_length=255, null=True, blank=True)
    workload_url = models.URLField(_('Workload page'), max_length=255, null=True)
    description = models.TextField(_('Description EN'), null=True, blank=True)
    description_lt = models.TextField(_('Description LT'), null=True, blank=True)

    def __unicode__(self):
        return self.title

    def get_title_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.title_lt
        else:
            return self.title

    def get_description_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.description_lt
        else:
            return self.description

    def get_tooltip_html(self):
        return _('
{0}

Cluster home:

{1}

Cluster workload:

{2}
').format(
                    self.get_description_display(),
                    self.url,
                    self.workload_url
                )

class Component(models.Model):
    CATEGORIES= (
        ('DATA', _('Upload data')),
        ('PREPROCESSING', _('Preprocessing')),
        ('STAT_PRIMITIVES', _('Statistical primitives')),
        ('DIMENSION_REDUCTION', _('Dimension reduction')),
        ('DATA_MINING', _('Classification, grouping')),
        ('RESULTS', _('View results')),
    )
    title = models.CharField(_('Title'), max_length=255, null=True, blank=True)
    user = models.ForeignKey(User, blank=True, null=True, verbose_name=_('User'))
    category = models.CharField(_('Category'), max_length=255, null=True, blank=True, choices=CATEGORIES)
    wsdl = models.URLField(_('WSDL link'), null=True, blank=True)
    function = models.CharField(_('WSDL function name'), max_length=255, null=True, blank=True)
    updated = models.DateTimeField(_('Updated'), auto_now=True, blank=True, null=True)
    created = models.DateTimeField(_('Created'), auto_now_add=True, blank=True, null=True)
    icon = models.ImageField(_('Icon'), upload_to='icons', blank=True, null=True)
    cluster = models.ForeignKey('Cluster', null=True, verbose_name=_('Cluster'))
    label = models.CharField(_('Label EN'), max_length=255, null=True, blank=True)
    label_lt = models.CharField(_('Label LT'), max_length=255, null=True, blank=True)
    description = models.TextField(_('Description EN'), null=True, blank=True)
    description_lt = models.TextField(_('Description LT'), null=True, blank=True)

    def get_absolute_url(self):
        return reverse('algorithm-list')

    def __unicode__(self):
        return str(self.title)

    def get_label_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.label_lt
        else:
            return self.label

    def get_description_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.description_lt
        else:
            return self.description

class Parameter(models.Model):
    CONNECTION_TYPES = (
        ('INPUT_VALUE', 'Input value'),
        ('INPUT_CONNECTION', 'Input connection'),
        ('INPUT_COMMON', 'Input common'),
        ('OUTPUT_VALUE', 'Output value'),
        ('OUTPUT_CONNECTION', 'Output connection'),
    )
    VALIDATORS = (
        ('dataset', _('Dataset')),
        ('int', _('Integer')),
        ('float', _('Float')),
        ('double', _('Double')),
        ('string', _('String')),
        ('text', _('Text')),
        ('boolean', _('Boolean')),
        ('date', _('Date')),
        ('datetime', _('Date and time')),
        ('time', _('Time')),
    )
    algorithm = models.ForeignKey(Component, related_name='parameters',
                                  null=True, blank=True,
                                  verbose_name=_('Component'))
    name = models.CharField(_('Option'), max_length=255, null=True)
    type = models.CharField(_('Type'), choices=VALIDATORS , max_length=255, null=True, blank=True)
    required = models.BooleanField(_('Required'), blank=True)
    default = models.CharField(_('Default'), max_length=255, null=True, blank=True)
    connection_type = models.CharField(_('Connection type'), max_length=255, null=True, blank=True,
                              choices=CONNECTION_TYPES, default='INPUT_VALUE')
    label = models.CharField(_('Label EN'), max_length=255, null=True, blank=True)
    label_lt = models.CharField(_('Label LT'), max_length=255, null=True, blank=True)
    description = models.TextField(_('Description EN'), null=True, blank=True)
    description_lt = models.TextField(_('Description LT'), null=True, blank=True)

    def __unicode__(self):
        return '%s (%s)' % (self.name, self.type)

    def get_label_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.label_lt
        else:
            return self.label

    def get_description_display(self):
        language_code = get_language()
        if language_code == 'lt':
            return self.description_lt
        else:
            return self.description

class Experiment(models.Model):
    STATUSES = (
        ('SAVED', _('Saved')),
        ('RUNNING', _('Running')),
        ('ERROR', _('Error')),
        ('FINISHED', _('Finished')),
    )
    title = models.CharField(_('Experiment title'), max_length=255, null=True)
    start = models.DateTimeField(_('Start'), blank=True, null=True)
    finish = models.DateTimeField(_('Finish'), blank=True, null=True)
    status = models.CharField(_('Status'), max_length=255, null=True, blank=True,
                              choices=STATUSES, default='SAVED')
    user = models.ForeignKey(User, blank=True, null=True, verbose_name=_('User'), related_name='experiments')
    workflow_state = models.TextField(_('Workflow state'), blank=True, null=True)

    max_calc_time = models.TimeField(_('Maximum calculation time'),
                                     default="2:00", null=True)
    p = models.IntegerField(_('Processor number'), default=1, null=True)

    updated = models.DateTimeField(_('Updated'), auto_now=True, blank=True, null=True)
    created = models.DateTimeField(_('Created'), auto_now_add=True, blank=True, null=True)

    def get_absolute_url(self):
        return reverse('experiment-list')

    def __unicode__(self):
        return str(self.title)

def get_result_file_upload_path(instance, filename):
    username = 'anonymous'
    if instance.algorithm.user:
        username = instance.algorithm.user.username
    return  '%s/result/%s' % (username, filename)

class WorkflowTask(models.Model):
    STATUSES = (
        ('SAVED', _('Saved')),
        ('RUNNING', _('Running')),
        ('ERROR', _('Error')),
        ('FINISHED', _('Finished')),
    )
    experiment = models.ForeignKey('Experiment', related_name='tasks',
                                   null=True, verbose_name=_('Experiment'))
    algorithm = models.ForeignKey('Component', verbose_name=_('Component'))
    status = models.CharField(_('Status'), max_length=255, null=True, blank=True,
                              choices=STATUSES, default='SAVED')

    def __unicode__(self):
        return '%s' % (self.algorithm.title,)

class ParameterValue(models.Model):
    parameter = models.ForeignKey('Parameter', verbose_name=_('Parameter'))
    value = models.CharField(max_length=255, blank=True, null=True)
    task = models.ForeignKey('WorkflowTask', related_name='parameter_values',
                                             verbose_name=_('Workflow Task'))
    related = models.ManyToManyField('self', symmetrical=False, null=True,
                                     blank=True, through='Connection')

    def __unicode__(self):
        return '%s: %s' % (self.task.algorithm, self.value)

class Connection(models.Model):
    source = models.ForeignKey('ParameterValue', null=True, blank=True,
                             verbose_name=_('Source'), related_name='source')
    target = models.ForeignKey('ParameterValue', null=True, blank=True,
                             verbose_name=_('Target'), related_name='target')
<|endoftext|>


<commit_before>import inference

def get_args():
    from argparse import ArgumentParser
    parser = ArgumentParser()

    parser.add_argument("filename", help="Module to infer types from")

    args = parser.parse_args()
    return args

def main():
    args = get_args()

    with open(args.filename, "r") as f:
        env = inference.ModuleEnv()
        env.parse_code(f.read())

    return 0

if __name__ == "__main__":
    main()
<commit_msg>removed infer_module.py<commit_after><|endoftext|>


<commit_before>import os
import dj_database_url

### Basic config

BASE = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
DEBUG = TEMPLATE_DEBUG = True
SITE_ID = 1
SECRET_KEY = 'its-a-secret-to-everybody'

# Until Sentry works on Py3, do errors the old-fashioned way.
ADMINS = [
    ("JKM", "jacob@jacobian.org"),
    ('Jeff', 'jeff.triplett+pythondotorg@gmail.com'),
    ('Frank', 'frank.wiles@gmail.com'),
    ("Flavio Curella", "flavio.curella@gmail.com"),
]

# General project information
# These are available in the template as SITE_INFO.<title>
SITE_VARIABLES = {
    'site_name': 'Python.org',
    'site_descript': 'The official home of the Python Programming Language',
}

### Databases

DATABASES = {
    'default': dj_database_url.config(default='postgres:///python.org')
}

### Locale settings

TIME_ZONE = 'UTC'
LANGUAGE_CODE = 'en-us'
USE_I18N = True
USE_L10N = True
USE_TZ = True

### Files (media and static)

MEDIA_ROOT = os.path.join(BASE, 'media')
MEDIA_URL = '/m/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = os.path.join(BASE, 'static-root')
STATIC_URL = '/static/'

STATICFILES_DIRS = (os.path.join(BASE, 'static'), MEDIA_ROOT)
STATICFILES_STORAGE = 'pipeline.storage.PipelineStorage'

### Authentication

AUTHENTICATION_BACKENDS = (
    # Needed to login by username in Django admin, regardless of `allauth`
    "django.contrib.auth.backends.ModelBackend",

    # `allauth` specific authentication methods, such as login by e-mail
    "allauth.account.auth_backends.AuthenticationBackend",
)

LOGIN_REDIRECT_URL = 'home'
ACCOUNT_LOGOUT_REDIRECT_URL = 'home'
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_UNIQUE_EMAIL = True
SOCIALACCOUNT_EMAIL_REQUIRED = True
SOCIALACCOUNT_EMAIL_VERIFICATION = True
SOCIALACCOUNT_QUERY_EMAIL = True

### Templates

TEMPLATE_DIRS = [
    os.path.join(BASE, 'templates')
]

TEMPLATE_CONTEXT_PROCESSORS = [
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.static",
    "django.core.context_processors.tz",
    "django.core.context_processors.request",
    "allauth.account.context_processors.account",
    "allauth.socialaccount.context_processors.socialaccount",
    "django.contrib.messages.context_processors.messages",
    "pydotorg.context_processors.site_info",
    "pydotorg.context_processors.url_name",
]

### URLs, WSGI, middleware, etc.

ROOT_URLCONF = 'pydotorg.urls'

MIDDLEWARE_CLASSES = (
    'pydotorg.middleware.AdminNoCaching',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'pages.middleware.PageFallbackMiddleware',
    'django.contrib.redirects.middleware.RedirectFallbackMiddleware',
)

AUTH_USER_MODEL = 'users.User'

WSGI_APPLICATION = 'pydotorg.wsgi.application'

### Apps

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.redirects',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.comments',
    'django.contrib.admin',
    'django.contrib.admindocs',

    'django_comments_xtd',
    'jsonfield',
    'pipeline',
    'sitetree',
    'south',
    'timedelta',
    'imagekit',
    'haystack',
    'honeypot',

    'users',
    'boxes',
    'cms',
    'companies',
    'feedbacks',
    'community',
    'jobs',
    'pages',
    'sponsors',
    'successstories',
    'events',
    'minutes',
    'peps',
    'blogs',
    'downloads',
    'codesamples',

    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    #'allauth.socialaccount.providers.facebook',
    #'allauth.socialaccount.providers.github',
    #'allauth.socialaccount.providers.openid',
    #'allauth.socialaccount.providers.twitter',

    # Tastypie needs the `users` app to be already loaded.
    'tastypie',

]

### Testing

SOUTH_TESTS_MIGRATE = False
TEST_RUNNER = 'discover_runner.DiscoverRunner'
TEST_DISCOVER_TOP_LEVEL = BASE
SKIP_NETWORK_TESTS = True

### Logging

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

### Comments

COMMENTS_APP = 'django_comments_xtd'
COMMENTS_XTD_MAX_THREAD_LEVEL = 0
COMMENTS_XTD_FORM_CLASS = "jobs.forms.JobCommentForm"

# Honeypot
HONEYPOT_FIELD_NAME = 'email_body_text'
HONEYPOT_VALUE = 'write your message'
### Blog Feed URL
PYTHON_BLOG_FEED_URL = "http://feeds.feedburner.com/PythonInsider"
PYTHON_BLOG_URL = "http://blog.python.org"

### Pipeline

from .pipeline import (
    PIPELINE_CSS, PIPELINE_JS,
    PIPELINE_COMPILERS,
    PIPELINE_SASS_BINARY, PIPELINE_SASS_ARGUMENTS,
    PIPELINE_CSS_COMPRESSOR, PIPELINE_JS_COMPRESSOR,
)
<commit_msg>Making email validation mandatory<commit_after>import os
import dj_database_url

### Basic config

BASE = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
DEBUG = TEMPLATE_DEBUG = True
SITE_ID = 1
SECRET_KEY = 'its-a-secret-to-everybody'

# Until Sentry works on Py3, do errors the old-fashioned way.
ADMINS = [
    ("JKM", "jacob@jacobian.org"),
    ('Jeff', 'jeff.triplett+pythondotorg@gmail.com'),
    ('Frank', 'frank.wiles@gmail.com'),
    ("Flavio Curella", "flavio.curella@gmail.com"),
]

# General project information
# These are available in the template as SITE_INFO.<title>
SITE_VARIABLES = {
    'site_name': 'Python.org',
    'site_descript': 'The official home of the Python Programming Language',
}

### Databases

DATABASES = {
    'default': dj_database_url.config(default='postgres:///python.org')
}

### Locale settings

TIME_ZONE = 'UTC'
LANGUAGE_CODE = 'en-us'
USE_I18N = True
USE_L10N = True
USE_TZ = True

### Files (media and static)

MEDIA_ROOT = os.path.join(BASE, 'media')
MEDIA_URL = '/m/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = os.path.join(BASE, 'static-root')
STATIC_URL = '/static/'

STATICFILES_DIRS = (os.path.join(BASE, 'static'), MEDIA_ROOT)
STATICFILES_STORAGE = 'pipeline.storage.PipelineStorage'

### Authentication

AUTHENTICATION_BACKENDS = (
    # Needed to login by username in Django admin, regardless of `allauth`
    "django.contrib.auth.backends.ModelBackend",

    # `allauth` specific authentication methods, such as login by e-mail
    "allauth.account.auth_backends.AuthenticationBackend",
)

LOGIN_REDIRECT_URL = 'home'
ACCOUNT_LOGOUT_REDIRECT_URL = 'home'
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_UNIQUE_EMAIL = True
ACCOUNT_EMAIL_VERIFICATION = 'mandatory'
SOCIALACCOUNT_EMAIL_REQUIRED = True
SOCIALACCOUNT_EMAIL_VERIFICATION = True
SOCIALACCOUNT_QUERY_EMAIL = True

### Templates

TEMPLATE_DIRS = [
    os.path.join(BASE, 'templates')
]

TEMPLATE_CONTEXT_PROCESSORS = [
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.static",
    "django.core.context_processors.tz",
    "django.core.context_processors.request",
    "allauth.account.context_processors.account",
    "allauth.socialaccount.context_processors.socialaccount",
    "django.contrib.messages.context_processors.messages",
    "pydotorg.context_processors.site_info",
    "pydotorg.context_processors.url_name",
]

### URLs, WSGI, middleware, etc.

ROOT_URLCONF = 'pydotorg.urls'

MIDDLEWARE_CLASSES = (
    'pydotorg.middleware.AdminNoCaching',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'pages.middleware.PageFallbackMiddleware',
    'django.contrib.redirects.middleware.RedirectFallbackMiddleware',
)

AUTH_USER_MODEL = 'users.User'

WSGI_APPLICATION = 'pydotorg.wsgi.application'

### Apps

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.redirects',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.comments',
    'django.contrib.admin',
    'django.contrib.admindocs',

    'django_comments_xtd',
    'jsonfield',
    'pipeline',
    'sitetree',
    'south',
    'timedelta',
    'imagekit',
    'haystack',
    'honeypot',

    'users',
    'boxes',
    'cms',
    'companies',
    'feedbacks',
    'community',
    'jobs',
    'pages',
    'sponsors',
    'successstories',
    'events',
    'minutes',
    'peps',
    'blogs',
    'downloads',
    'codesamples',

    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    #'allauth.socialaccount.providers.facebook',
    #'allauth.socialaccount.providers.github',
    #'allauth.socialaccount.providers.openid',
    #'allauth.socialaccount.providers.twitter',

    # Tastypie needs the `users` app to be already loaded.
    'tastypie',

]

### Testing

SOUTH_TESTS_MIGRATE = False
TEST_RUNNER = 'discover_runner.DiscoverRunner'
TEST_DISCOVER_TOP_LEVEL = BASE
SKIP_NETWORK_TESTS = True

### Logging

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

### Comments

COMMENTS_APP = 'django_comments_xtd'
COMMENTS_XTD_MAX_THREAD_LEVEL = 0
COMMENTS_XTD_FORM_CLASS = "jobs.forms.JobCommentForm"

# Honeypot
HONEYPOT_FIELD_NAME = 'email_body_text'
HONEYPOT_VALUE = 'write your message'
### Blog Feed URL
PYTHON_BLOG_FEED_URL = "http://feeds.feedburner.com/PythonInsider"
PYTHON_BLOG_URL = "http://blog.python.org"

### Pipeline

from .pipeline import (
    PIPELINE_CSS, PIPELINE_JS,
    PIPELINE_COMPILERS,
    PIPELINE_SASS_BINARY, PIPELINE_SASS_ARGUMENTS,
    PIPELINE_CSS_COMPRESSOR, PIPELINE_JS_COMPRESSOR,
)
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2018 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for Transformer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np

from tensor2tensor.data_generators import problem_hparams
from tensor2tensor.models import transformer

import tensorflow as tf

BATCH_SIZE = 3
INPUT_LENGTH = 5
TARGET_LENGTH = 7
VOCAB_SIZE = 10

def get_model(hparams=None, mode=tf.estimator.ModeKeys.TRAIN,
              has_input=True, model_cls=transformer.Transformer):
  if hparams is None:
    hparams = transformer.transformer_tiny()
  hparams.hidden_size = 8
  hparams.filter_size = 32
  hparams.num_heads = 1
  hparams.layer_prepostprocess_dropout = 0.0

  p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE, VOCAB_SIZE)
  if not has_input:
    p_hparams.input_modality = {}
  hparams.problem_hparams = p_hparams

  inputs = -1 + np.random.random_integers(
      VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
  targets = -1 + np.random.random_integers(
      VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
  features = {
      "targets": tf.constant(targets, dtype=tf.int32, name="targets"),
      "target_space_id": tf.constant(1, dtype=tf.int32)
  }
  if has_input:
    features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")

  return model_cls(hparams, mode, p_hparams), features

class TransformerTest(tf.test.TestCase):

  def testTransformer(self):
    model, features = get_model(transformer.transformer_small())
    logits, _ = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(logits)
    self.assertEqual(res.shape, (BATCH_SIZE, TARGET_LENGTH, 1, 1, VOCAB_SIZE))

  def testTransformerRelative(self):
    model, features = get_model(transformer.transformer_relative_tiny())
    logits, _ = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(logits)
    self.assertEqual(res.shape, (BATCH_SIZE, TARGET_LENGTH, 1, 1, VOCAB_SIZE))

  def testSlowVsFast(self):
    model, features = get_model(transformer.transformer_small())

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      greedy_result = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      greedy_result = tf.squeeze(greedy_result, axis=[2, 3])

      fast_result = model._greedy_infer(features, decode_length)["outputs"]

    with self.test_session():
      greedy_res = greedy_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(fast_res.shape, (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(greedy_res, fast_res)

  def testSlowVsFastNoInput(self):
    model, features = get_model(
        transformer.transformer_small(), has_input=False)

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      slow_result = tf.squeeze(slow_result, axis=[2, 3])

      fast_result = model._greedy_infer(features, decode_length)["outputs"]

    with self.test_session():
      slow_res = slow_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(slow_res.shape, (BATCH_SIZE, decode_length))
    self.assertAllClose(slow_res, fast_res)

  def testBeamDecodeWithRelativeAttention(self):
    decode_length = 2
    model, features = get_model(transformer.transformer_relative_tiny())
    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    beam_result = model._beam_decode(
        features, decode_length, beam_size=4, top_beams=1,
        alpha=1.0)["outputs"]

    with self.test_session():
      tf.global_variables_initializer().run()
      beam_result.eval()

    # TODO(petershaw): This test is flaky because the decode may hit EOS before
    # getting to the expected length.
    # self.assertEqual(beam_res.shape,
    #                  (BATCH_SIZE, INPUT_LENGTH + decode_length))

  def testBeamVsFast(self):
    model, features = get_model(transformer.transformer_small())

    decode_length = 2

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      beam_result = model._beam_decode_slow(
          features,
          decode_length,
          beam_size=4,
          top_beams=1,
          alpha=1.0)["outputs"]

      fast_result = model._beam_decode(
          features,
          decode_length,
          beam_size=4,
          top_beams=1,
          alpha=1.0)["outputs"]

    with self.test_session():
      beam_res = beam_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(fast_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(beam_res, fast_res)

  def testTransformerWithoutProblem(self):
    hparams = transformer.transformer_test()

    embedded_inputs = np.random.random_sample(
        (BATCH_SIZE, INPUT_LENGTH, 1, hparams.hidden_size))
    embedded_targets = np.random.random_sample(
        (BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size))

    transformed_features = {
        "inputs": tf.constant(embedded_inputs, dtype=tf.float32),
        "targets": tf.constant(embedded_targets, dtype=tf.float32)
    }

    model = transformer.Transformer(hparams)
    body_out, _ = model(transformed_features)

    self.assertAllEqual(
        body_out.get_shape().as_list(),
        [BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size])

  def testTransformerWithEncoderDecoderAttentionLoss(self):
    model, features = get_model(
        transformer.transformer_supervised_attention())
    expected_attention_weights = np.random.random_sample(
        size=(BATCH_SIZE, TARGET_LENGTH, INPUT_LENGTH))
    features["expected_attentions"] = tf.constant(
        expected_attention_weights, dtype=tf.float32)
    _, extra_loss = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(extra_loss["attention_loss"])
    self.assertEqual(res.shape, ())

  def testGreedySlowTPUVsNonTPU(self):
    # Only works with TF 1.8+
    major, minor, _ = [int(el) for el in tf.__version__.split(".")]
    if major < 1 or (major == 1 and minor < 8):
      return
    model, features = get_model(transformer.transformer_small())

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result_non_tpu = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      slow_result_non_tpu = tf.squeeze(slow_result_non_tpu, axis=[2, 3])

      slow_result_tpu = model._slow_greedy_infer_tpu(
          features, decode_length)["outputs"]
      slow_result_tpu = tf.squeeze(slow_result_tpu, axis=[2, 3])

    with self.test_session():
      slow_non_tpu_res = slow_result_non_tpu.eval()
      slow_tpu_res = slow_result_tpu.eval()

    self.assertEqual(slow_tpu_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(slow_tpu_res, slow_non_tpu_res)

class TransformerScorerTest(tf.test.TestCase):

  def testReturnsScores(self):
    model, features = get_model(
        mode=tf.estimator.ModeKeys.PREDICT,
        model_cls=transformer.TransformerScorer)
    infer_out = model.infer(features)
    self.assertTrue("outputs" in infer_out)
    self.assertTrue("scores" in infer_out)

    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      infer_out = session.run(infer_out)
      self.assertEqual((BATCH_SIZE,), infer_out["scores"].shape)
      self.assertEqual((BATCH_SIZE, TARGET_LENGTH), infer_out["outputs"].shape)

  def testVarNames(self):
    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.PREDICT,
          model_cls=transformer.TransformerScorer)
      _ = model.infer(features)
      scorer_vars = [v.name for v in tf.global_variables()]

    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.EVAL,
          model_cls=transformer.TransformerScorer)
      _ = model(features)
      scorer_eval_vars = [v.name for v in tf.global_variables()]

    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.EVAL,
          model_cls=transformer.Transformer)
      _ = model(features)
      transformer_vars = [v.name for v in tf.global_variables()]

    self.assertEqual(sorted(scorer_vars), sorted(transformer_vars))
    self.assertEqual(sorted(scorer_eval_vars), sorted(transformer_vars))

if __name__ == "__main__":
  tf.test.main()
<commit_msg>Internal change<commit_after># coding=utf-8
# Copyright 2018 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for Transformer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np

from tensor2tensor.data_generators import problem_hparams
from tensor2tensor.models import transformer

import tensorflow as tf

BATCH_SIZE = 3
INPUT_LENGTH = 5
TARGET_LENGTH = 7
VOCAB_SIZE = 10

def get_model(hparams=None, mode=tf.estimator.ModeKeys.TRAIN,
              has_input=True, model_cls=transformer.Transformer):
  if hparams is None:
    hparams = transformer.transformer_tiny()
  hparams.hidden_size = 8
  hparams.filter_size = 32
  hparams.num_heads = 1
  hparams.layer_prepostprocess_dropout = 0.0

  p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE, VOCAB_SIZE)
  if not has_input:
    p_hparams.input_modality = {}
  hparams.problem_hparams = p_hparams

  inputs = -1 + np.random.random_integers(
      VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
  targets = -1 + np.random.random_integers(
      VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
  features = {
      "targets": tf.constant(targets, dtype=tf.int32, name="targets"),
      "target_space_id": tf.constant(1, dtype=tf.int32)
  }
  if has_input:
    features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")

  return model_cls(hparams, mode, p_hparams), features

class TransformerTest(tf.test.TestCase):

  def testTransformer(self):
    model, features = get_model(transformer.transformer_small())
    logits, _ = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(logits)
    self.assertEqual(res.shape, (BATCH_SIZE, TARGET_LENGTH, 1, 1, VOCAB_SIZE))

  def testTransformerRelative(self):
    model, features = get_model(transformer.transformer_relative_tiny())
    logits, _ = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(logits)
    self.assertEqual(res.shape, (BATCH_SIZE, TARGET_LENGTH, 1, 1, VOCAB_SIZE))

  def testSlowVsFast(self):
    model, features = get_model(transformer.transformer_small())

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      greedy_result = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      greedy_result = tf.squeeze(greedy_result, axis=[2, 3])

      fast_result = model._greedy_infer(features, decode_length)["outputs"]

    with self.test_session():
      greedy_res = greedy_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(fast_res.shape, (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(greedy_res, fast_res)

  def testSlowVsFastNoInput(self):
    model, features = get_model(
        transformer.transformer_small(), has_input=False)

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      slow_result = tf.squeeze(slow_result, axis=[2, 3])

      fast_result = model._greedy_infer(features, decode_length)["outputs"]

    with self.test_session():
      slow_res = slow_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(slow_res.shape, (BATCH_SIZE, decode_length))
    self.assertAllClose(slow_res, fast_res)

  def testBeamDecodeWithRelativeAttention(self):
    decode_length = 2
    model, features = get_model(transformer.transformer_relative_tiny())
    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    beam_result = model._beam_decode(
        features, decode_length, beam_size=4, top_beams=1,
        alpha=1.0)["outputs"]

    with self.test_session():
      tf.global_variables_initializer().run()
      beam_result.eval()

    # TODO(petershaw): This test is flaky because the decode may hit EOS before
    # getting to the expected length.
    # self.assertEqual(beam_res.shape,
    #                  (BATCH_SIZE, INPUT_LENGTH + decode_length))

  def testBeamVsFast(self):
    model, features = get_model(transformer.transformer_small())

    decode_length = 2

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      beam_result = model._beam_decode_slow(
          features,
          decode_length,
          beam_size=4,
          top_beams=1,
          alpha=1.0)["outputs"]

      fast_result = model._beam_decode(
          features,
          decode_length,
          beam_size=4,
          top_beams=1,
          alpha=1.0)["outputs"]

    with self.test_session():
      beam_res = beam_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(fast_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(beam_res, fast_res)

  def testTransformerWithoutProblem(self):
    hparams = transformer.transformer_test()

    embedded_inputs = np.random.random_sample(
        (BATCH_SIZE, INPUT_LENGTH, 1, hparams.hidden_size))
    embedded_targets = np.random.random_sample(
        (BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size))

    transformed_features = {
        "inputs": tf.constant(embedded_inputs, dtype=tf.float32),
        "targets": tf.constant(embedded_targets, dtype=tf.float32)
    }

    model = transformer.Transformer(hparams)
    body_out, _ = model(transformed_features)

    self.assertAllEqual(
        body_out.get_shape().as_list(),
        [BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size])

  def testTransformerWithEncoderDecoderAttentionLoss(self):
    model, features = get_model(
        transformer.transformer_supervised_attention())
    expected_attention_weights = np.random.random_sample(
        size=(BATCH_SIZE, TARGET_LENGTH, INPUT_LENGTH))
    features["expected_attentions"] = tf.constant(
        expected_attention_weights, dtype=tf.float32)
    _, extra_loss = model(features)
    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      res = session.run(extra_loss["attention_loss"])
    self.assertEqual(res.shape, ())

  def testGreedySlowTPUVsNonTPU(self):
    # Only works with TF 1.8+
    # Version string can take the following form: "1.9.0-rc0"
    major_str, minor_str, unused_rest = tf.__version__.split(".", 3)
    major, minor = int(major_str), int(minor_str)
    if major < 1 or (major == 1 and minor < 8):
      return
    model, features = get_model(transformer.transformer_small())

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      slow_result_non_tpu = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      slow_result_non_tpu = tf.squeeze(slow_result_non_tpu, axis=[2, 3])

      slow_result_tpu = model._slow_greedy_infer_tpu(
          features, decode_length)["outputs"]
      slow_result_tpu = tf.squeeze(slow_result_tpu, axis=[2, 3])

    with self.test_session():
      slow_non_tpu_res = slow_result_non_tpu.eval()
      slow_tpu_res = slow_result_tpu.eval()

    self.assertEqual(slow_tpu_res.shape,
                     (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(slow_tpu_res, slow_non_tpu_res)

class TransformerScorerTest(tf.test.TestCase):

  def testReturnsScores(self):
    model, features = get_model(
        mode=tf.estimator.ModeKeys.PREDICT,
        model_cls=transformer.TransformerScorer)
    infer_out = model.infer(features)
    self.assertTrue("outputs" in infer_out)
    self.assertTrue("scores" in infer_out)

    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      infer_out = session.run(infer_out)
      self.assertEqual((BATCH_SIZE,), infer_out["scores"].shape)
      self.assertEqual((BATCH_SIZE, TARGET_LENGTH), infer_out["outputs"].shape)

  def testVarNames(self):
    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.PREDICT,
          model_cls=transformer.TransformerScorer)
      _ = model.infer(features)
      scorer_vars = [v.name for v in tf.global_variables()]

    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.EVAL,
          model_cls=transformer.TransformerScorer)
      _ = model(features)
      scorer_eval_vars = [v.name for v in tf.global_variables()]

    with tf.Graph().as_default():
      model, features = get_model(
          mode=tf.estimator.ModeKeys.EVAL,
          model_cls=transformer.Transformer)
      _ = model(features)
      transformer_vars = [v.name for v in tf.global_variables()]

    self.assertEqual(sorted(scorer_vars), sorted(transformer_vars))
    self.assertEqual(sorted(scorer_eval_vars), sorted(transformer_vars))

if __name__ == "__main__":
  tf.test.main()
<|endoftext|>


<commit_before>import yaml

class Parser(object):

    def __init__(self):
        pass

    def parse_file(self, path):
        with open(path, 'rb') as f:
            yaml_data = f.read()
        return self.parse(yaml_data)

    def parse(self, string):
        return yaml.load(string)

    def dump_to_file(self, configuration, path):
        yaml_data = self.dump(configuration)
        with open(path, 'wb') as f:
            f.write(yaml_data)

    def dump(self, configuration):
        yaml_string = ''
        for key in ['name', 'author', 'version', 'repositories', 'dependencies']:
            if configuration.has_key(key):
                yaml_string += yaml.dump({key: configuration[key]}, default_flow_style=False)
        return yaml_string
<commit_msg>Fixed a bug in the parser<commit_after>import yaml

class Parser(object):

    def __init__(self):
        pass

    def parse_file(self, path):
        with open(path, 'rb') as f:
            yaml_data = f.read()
        return self.parse(yaml_data)

    def parse(self, string):
        return yaml.load(string)

    def dump_to_file(self, configuration, path):
        yaml_data = self.dump(configuration)
        with open(path, 'wb') as f:
            f.write(yaml_data)

    def dump(self, configuration):
        yaml_string = ''
        for key in ['name', 'author', 'version', 'repository', 'dependencies']:
            if configuration.has_key(key):
                yaml_string += yaml.dump({key: configuration[key]}, default_flow_style=False)
        return yaml_string
<|endoftext|>


<commit_before>"""distutils.command.bdist_wininst

Implements the Distutils 'bdist_wininst' command: create a windows installer
exe-program."""

# This module should be kept compatible with Python 1.5.2.

__revision__ = "$Id$"

import sys, os, string
from distutils.core import Command
from distutils.util import get_platform
from distutils.dir_util import create_tree, remove_tree
from distutils.errors import *
from distutils.sysconfig import get_python_version
from distutils import log

class bdist_wininst (Command):

    description = "create an executable installer for MS Windows"

    user_options = [('bdist-dir=', None,
                     "temporary directory for creating the distribution"),
                    ('keep-temp', 'k',
                     "keep the pseudo-installation tree around after " +
                     "creating the distribution archive"),
                    ('target-version=', 'v',
                     "require a specific python version" +
                     " on the target system"),
                    ('no-target-compile', 'c',
                     "do not compile .py to .pyc on the target system"),
                    ('no-target-optimize', 'o',
                     "do not compile .py to .pyo (optimized)"
                     "on the target system"),
                    ('dist-dir=', 'd',
                     "directory to put final built distributions in"),
                    ('bitmap=', 'b',
                     "bitmap to use for the installer instead of python-powered logo"),
                    ('title=', 't',
                     "title to display on the installer background instead of default"),
                    ('skip-build', None,
                     "skip rebuilding everything (for testing/debugging)"),
                    ('install-script=', None,
                     "basename of installation script to be run after"
                     "installation or before deinstallation"),
                    ('pre-install-script=', None,
                     "Fully qualified filename of a script to be run before "
                     "any files are installed.  This script need not be in the "
                     "distribution"),
                   ]

    boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize',
                       'skip-build']

    def initialize_options (self):
        self.bdist_dir = None
        self.keep_temp = 0
        self.no_target_compile = 0
        self.no_target_optimize = 0
        self.target_version = None
        self.dist_dir = None
        self.bitmap = None
        self.title = None
        self.skip_build = 0
        self.install_script = None
        self.pre_install_script = None

    # initialize_options()

    def finalize_options (self):
        if self.bdist_dir is None:
            bdist_base = self.get_finalized_command('bdist').bdist_base
            self.bdist_dir = os.path.join(bdist_base, 'wininst')
        if not self.target_version:
            self.target_version = ""
        if not self.skip_build and self.distribution.has_ext_modules():
            short_version = get_python_version()
            if self.target_version and self.target_version != short_version:
                raise DistutilsOptionError, \
                      "target version can only be %s, or the '--skip_build'" \
                      " option must be specified" % (short_version,)
            self.target_version = short_version

        self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))

        if self.install_script:
            for script in self.distribution.scripts:
                if self.install_script == os.path.basename(script):
                    break
            else:
                raise DistutilsOptionError, \
                      "install_script '%s' not found in scripts" % \
                      self.install_script
    # finalize_options()

    def run (self):
        if (sys.platform != "win32" and
            (self.distribution.has_ext_modules() or
             self.distribution.has_c_libraries())):
            raise DistutilsPlatformError \
                  ("distribution contains extensions and/or C libraries; "
                   "must be compiled on a Windows 32 platform")

        if not self.skip_build:
            self.run_command('build')

        install = self.reinitialize_command('install', reinit_subcommands=1)
        install.root = self.bdist_dir
        install.skip_build = self.skip_build
        install.warn_dir = 0

        install_lib = self.reinitialize_command('install_lib')
        # we do not want to include pyc or pyo files
        install_lib.compile = 0
        install_lib.optimize = 0

        # If we are building an installer for a Python version other
        # than the one we are currently running, then we need to ensure
        # our build_lib reflects the other Python version rather than ours.
        # Note that for target_version!=sys.version, we must have skipped the
        # build step, so there is no issue with enforcing the build of this
        # version.
        target_version = self.target_version
        if not target_version:
            assert self.skip_build, "Should have already checked this"
            target_version = sys.version[0:3]
        plat_specifier = ".%s-%s" % (get_platform(), target_version)
        build = self.get_finalized_command('build')
        build.build_lib = os.path.join(build.build_base,
                                       'lib' + plat_specifier)

        # Use a custom scheme for the zip-file, because we have to decide
        # at installation time which scheme to use.
        for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'):
            value = string.upper(key)
            if key == 'headers':
                value = value + '/Include/$dist_name'
            setattr(install,
                    'install_' + key,
                    value)

        log.info("installing to %s", self.bdist_dir)
        install.ensure_finalized()

        # avoid warning of 'install_lib' about installing
        # into a directory not in sys.path
        sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB'))

        install.run()

        del sys.path[0]

        # And make an archive relative to the root of the
        # pseudo-installation tree.
        from tempfile import mktemp
        archive_basename = mktemp()
        fullname = self.distribution.get_fullname()
        arcname = self.make_archive(archive_basename, "zip",
                                    root_dir=self.bdist_dir)
        # create an exe containing the zip-file
        self.create_exe(arcname, fullname, self.bitmap)
        # remove the zip-file again
        log.debug("removing temporary file '%s'", arcname)
        os.remove(arcname)

        if not self.keep_temp:
            remove_tree(self.bdist_dir, dry_run=self.dry_run)

    # run()

    def get_inidata (self):
        # Return data describing the installation.

        lines = []
        metadata = self.distribution.metadata

        # Write the [metadata] section.  Values are written with
        # repr()[1:-1], so they do not contain unprintable characters, and
        # are not surrounded by quote chars.
        lines.append("[metadata]")

        # 'info' will be displayed in the installer's dialog box,
        # describing the items to be installed.
        info = (metadata.long_description or '') + '\n'

        for name in ["author", "author_email", "description", "maintainer",
                     "maintainer_email", "name", "url", "version"]:
            data = getattr(metadata, name, "")
            if data:
                info = info + ("\n    %s: %s" % \
                               (string.capitalize(name), data))
                lines.append("%s=%s" % (name, repr(data)[1:-1]))

        # The [setup] section contains entries controlling
        # the installer runtime.
        lines.append("\n[Setup]")
        if self.install_script:
            lines.append("install_script=%s" % self.install_script)
        lines.append("info=%s" % repr(info)[1:-1])
        lines.append("target_compile=%d" % (not self.no_target_compile))
        lines.append("target_optimize=%d" % (not self.no_target_optimize))
        if self.target_version:
            lines.append("target_version=%s" % self.target_version)

        title = self.title or self.distribution.get_fullname()
        lines.append("title=%s" % repr(title)[1:-1])
        import time
        import distutils
        build_info = "Built %s with distutils-%s" % \
                     (time.ctime(time.time()), distutils.__version__)
        lines.append("build_info=%s" % build_info)
        return string.join(lines, "\n")

    # get_inidata()

    def create_exe (self, arcname, fullname, bitmap=None):
        import struct

        self.mkpath(self.dist_dir)

        cfgdata = self.get_inidata()

        if self.target_version:
            # if we create an installer for a specific python version,
            # it's better to include this in the name
            installer_name = os.path.join(self.dist_dir,
                                          "%s.win32-py%s.exe" %
                                           (fullname, self.target_version))
        else:
            installer_name = os.path.join(self.dist_dir,
                                          "%s.win32.exe" % fullname)
        self.announce("creating %s" % installer_name)

        if bitmap:
            bitmapdata = open(bitmap, "rb").read()
            bitmaplen = len(bitmapdata)
        else:
            bitmaplen = 0

        file = open(installer_name, "wb")
        file.write(self.get_exe_bytes())
        if bitmap:
            file.write(bitmapdata)

        # Append the pre-install script
        cfgdata = cfgdata + "\0"
        if self.pre_install_script:
            script_data = open(self.pre_install_script, "r").read()
            cfgdata = cfgdata + script_data + "\n\0"
        file.write(cfgdata)
        header = struct.pack("<iii",
                             0x1234567A,       # tag
                             len(cfgdata),     # length
                             bitmaplen,        # number of bytes in bitmap
                             )
        file.write(header)
        file.write(open(arcname, "rb").read())

    # create_exe()

    def get_exe_bytes (self):
        # wininst.exe is in the same directory as this file
        directory = os.path.dirname(__file__)
        filename = os.path.join(directory, "wininst.exe")
        return open(filename, "rb").read()
# class bdist_wininst
<commit_msg>Use the right wininstXX.exe, depending on msvccompiler.get_build_version().<commit_after>"""distutils.command.bdist_wininst

Implements the Distutils 'bdist_wininst' command: create a windows installer
exe-program."""

# This module should be kept compatible with Python 1.5.2.

__revision__ = "$Id$"

import sys, os, string
from distutils.core import Command
from distutils.util import get_platform
from distutils.dir_util import create_tree, remove_tree
from distutils.errors import *
from distutils.sysconfig import get_python_version
from distutils import log

class bdist_wininst (Command):

    description = "create an executable installer for MS Windows"

    user_options = [('bdist-dir=', None,
                     "temporary directory for creating the distribution"),
                    ('keep-temp', 'k',
                     "keep the pseudo-installation tree around after " +
                     "creating the distribution archive"),
                    ('target-version=', 'v',
                     "require a specific python version" +
                     " on the target system"),
                    ('no-target-compile', 'c',
                     "do not compile .py to .pyc on the target system"),
                    ('no-target-optimize', 'o',
                     "do not compile .py to .pyo (optimized)"
                     "on the target system"),
                    ('dist-dir=', 'd',
                     "directory to put final built distributions in"),
                    ('bitmap=', 'b',
                     "bitmap to use for the installer instead of python-powered logo"),
                    ('title=', 't',
                     "title to display on the installer background instead of default"),
                    ('skip-build', None,
                     "skip rebuilding everything (for testing/debugging)"),
                    ('install-script=', None,
                     "basename of installation script to be run after"
                     "installation or before deinstallation"),
                    ('pre-install-script=', None,
                     "Fully qualified filename of a script to be run before "
                     "any files are installed.  This script need not be in the "
                     "distribution"),
                   ]

    boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize',
                       'skip-build']

    def initialize_options (self):
        self.bdist_dir = None
        self.keep_temp = 0
        self.no_target_compile = 0
        self.no_target_optimize = 0
        self.target_version = None
        self.dist_dir = None
        self.bitmap = None
        self.title = None
        self.skip_build = 0
        self.install_script = None
        self.pre_install_script = None

    # initialize_options()

    def finalize_options (self):
        if self.bdist_dir is None:
            bdist_base = self.get_finalized_command('bdist').bdist_base
            self.bdist_dir = os.path.join(bdist_base, 'wininst')
        if not self.target_version:
            self.target_version = ""
        if not self.skip_build and self.distribution.has_ext_modules():
            short_version = get_python_version()
            if self.target_version and self.target_version != short_version:
                raise DistutilsOptionError, \
                      "target version can only be %s, or the '--skip_build'" \
                      " option must be specified" % (short_version,)
            self.target_version = short_version

        self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))

        if self.install_script:
            for script in self.distribution.scripts:
                if self.install_script == os.path.basename(script):
                    break
            else:
                raise DistutilsOptionError, \
                      "install_script '%s' not found in scripts" % \
                      self.install_script
    # finalize_options()

    def run (self):
        if (sys.platform != "win32" and
            (self.distribution.has_ext_modules() or
             self.distribution.has_c_libraries())):
            raise DistutilsPlatformError \
                  ("distribution contains extensions and/or C libraries; "
                   "must be compiled on a Windows 32 platform")

        if not self.skip_build:
            self.run_command('build')

        install = self.reinitialize_command('install', reinit_subcommands=1)
        install.root = self.bdist_dir
        install.skip_build = self.skip_build
        install.warn_dir = 0

        install_lib = self.reinitialize_command('install_lib')
        # we do not want to include pyc or pyo files
        install_lib.compile = 0
        install_lib.optimize = 0

        # If we are building an installer for a Python version other
        # than the one we are currently running, then we need to ensure
        # our build_lib reflects the other Python version rather than ours.
        # Note that for target_version!=sys.version, we must have skipped the
        # build step, so there is no issue with enforcing the build of this
        # version.
        target_version = self.target_version
        if not target_version:
            assert self.skip_build, "Should have already checked this"
            target_version = sys.version[0:3]
        plat_specifier = ".%s-%s" % (get_platform(), target_version)
        build = self.get_finalized_command('build')
        build.build_lib = os.path.join(build.build_base,
                                       'lib' + plat_specifier)

        # Use a custom scheme for the zip-file, because we have to decide
        # at installation time which scheme to use.
        for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'):
            value = string.upper(key)
            if key == 'headers':
                value = value + '/Include/$dist_name'
            setattr(install,
                    'install_' + key,
                    value)

        log.info("installing to %s", self.bdist_dir)
        install.ensure_finalized()

        # avoid warning of 'install_lib' about installing
        # into a directory not in sys.path
        sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB'))

        install.run()

        del sys.path[0]

        # And make an archive relative to the root of the
        # pseudo-installation tree.
        from tempfile import mktemp
        archive_basename = mktemp()
        fullname = self.distribution.get_fullname()
        arcname = self.make_archive(archive_basename, "zip",
                                    root_dir=self.bdist_dir)
        # create an exe containing the zip-file
        self.create_exe(arcname, fullname, self.bitmap)
        # remove the zip-file again
        log.debug("removing temporary file '%s'", arcname)
        os.remove(arcname)

        if not self.keep_temp:
            remove_tree(self.bdist_dir, dry_run=self.dry_run)

    # run()

    def get_inidata (self):
        # Return data describing the installation.

        lines = []
        metadata = self.distribution.metadata

        # Write the [metadata] section.  Values are written with
        # repr()[1:-1], so they do not contain unprintable characters, and
        # are not surrounded by quote chars.
        lines.append("[metadata]")

        # 'info' will be displayed in the installer's dialog box,
        # describing the items to be installed.
        info = (metadata.long_description or '') + '\n'

        for name in ["author", "author_email", "description", "maintainer",
                     "maintainer_email", "name", "url", "version"]:
            data = getattr(metadata, name, "")
            if data:
                info = info + ("\n    %s: %s" % \
                               (string.capitalize(name), data))
                lines.append("%s=%s" % (name, repr(data)[1:-1]))

        # The [setup] section contains entries controlling
        # the installer runtime.
        lines.append("\n[Setup]")
        if self.install_script:
            lines.append("install_script=%s" % self.install_script)
        lines.append("info=%s" % repr(info)[1:-1])
        lines.append("target_compile=%d" % (not self.no_target_compile))
        lines.append("target_optimize=%d" % (not self.no_target_optimize))
        if self.target_version:
            lines.append("target_version=%s" % self.target_version)

        title = self.title or self.distribution.get_fullname()
        lines.append("title=%s" % repr(title)[1:-1])
        import time
        import distutils
        build_info = "Built %s with distutils-%s" % \
                     (time.ctime(time.time()), distutils.__version__)
        lines.append("build_info=%s" % build_info)
        return string.join(lines, "\n")

    # get_inidata()

    def create_exe (self, arcname, fullname, bitmap=None):
        import struct

        self.mkpath(self.dist_dir)

        cfgdata = self.get_inidata()

        if self.target_version:
            # if we create an installer for a specific python version,
            # it's better to include this in the name
            installer_name = os.path.join(self.dist_dir,
                                          "%s.win32-py%s.exe" %
                                           (fullname, self.target_version))
        else:
            installer_name = os.path.join(self.dist_dir,
                                          "%s.win32.exe" % fullname)
        self.announce("creating %s" % installer_name)

        if bitmap:
            bitmapdata = open(bitmap, "rb").read()
            bitmaplen = len(bitmapdata)
        else:
            bitmaplen = 0

        file = open(installer_name, "wb")
        file.write(self.get_exe_bytes())
        if bitmap:
            file.write(bitmapdata)

        # Append the pre-install script
        cfgdata = cfgdata + "\0"
        if self.pre_install_script:
            script_data = open(self.pre_install_script, "r").read()
            cfgdata = cfgdata + script_data + "\n\0"
        else:
            # empty pre-install script
            cfgdata = cfgdata + "\0"
        file.write(cfgdata)
        header = struct.pack("<iii",
                             0x1234567A,       # tag
                             len(cfgdata),     # length
                             bitmaplen,        # number of bytes in bitmap
                             )
        file.write(header)
        file.write(open(arcname, "rb").read())

    # create_exe()

    def get_exe_bytes (self):
        from distutils.msvccompiler import get_build_version
        # wininst-x.y.exe is in the same directory as this file
        directory = os.path.dirname(__file__)
        # we must use a wininst-x.y.exe built with the same C compiler
        # used for python.  XXX What about mingw, borland, and so on?
        filename = os.path.join(directory, "wininst-%s.exe" % get_build_version())
        return open(filename, "rb").read()
# class bdist_wininst
<|endoftext|>


<commit_before>#!/bin/env python
#------------------------------------------------------------------------
#           Copyright (c) 1997-2001 by Total Control Software
#                         All Rights Reserved
#------------------------------------------------------------------------
#
# Module Name:  dbShelve.py
#
# Description:  A reimplementation of the standard shelve.py that
#               forces the use of cPickle, and DB.
#
# Creation Date:    11/3/97 3:39:04PM
#
# License:      This is free software.  You may use this software for any
#               purpose including modification/redistribution, so long as
#               this header remains intact and that you do not claim any
#               rights of ownership or authorship of this software.  This
#               software has been tested, but no warranty is expressed or
#               implied.
#
# 13-Dec-2000:  Updated to be used with the new bsddb3 package.
#               Added DBShelfCursor class.
#
#------------------------------------------------------------------------

"""Manage shelves of pickled objects using bsddb database files for the
storage.
"""

#------------------------------------------------------------------------

import cPickle
import db
import sys

#At version 2.3 cPickle switched to using protocol instead of bin and
#DictMixin was added
if sys.version_info[:3] >= (2, 3, 0):
    HIGHEST_PROTOCOL = cPickle.HIGHEST_PROTOCOL
    def _dumps(object, protocol):
        return cPickle.dumps(object, protocol=protocol)
    from UserDict import DictMixin
else:
    HIGHEST_PROTOCOL = None
    def _dumps(object, protocol):
        return cPickle.dumps(object, bin=protocol)
    class DictMixin: pass

#------------------------------------------------------------------------

def open(filename, flags=db.DB_CREATE, mode=0660, filetype=db.DB_HASH,
         dbenv=None, dbname=None):
    """
    A simple factory function for compatibility with the standard
    shleve.py module.  It can be used like this, where key is a string
    and data is a pickleable object:

        from bsddb import dbshelve
        db = dbshelve.open(filename)

        db[key] = data

        db.close()
    """
    if type(flags) == type(''):
        sflag = flags
        if sflag == 'r':
            flags = db.DB_RDONLY
        elif sflag == 'rw':
            flags = 0
        elif sflag == 'w':
            flags =  db.DB_CREATE
        elif sflag == 'c':
            flags =  db.DB_CREATE
        elif sflag == 'n':
            flags = db.DB_TRUNCATE | db.DB_CREATE
        else:
            raise db.DBError, "flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags"

    d = DBShelf(dbenv)
    d.open(filename, dbname, filetype, flags, mode)
    return d

#---------------------------------------------------------------------------

class DBShelveError(db.DBError): pass

class DBShelf(DictMixin):
    """A shelf to hold pickled objects, built upon a bsddb DB object.  It
    automatically pickles/unpickles data objects going to/from the DB.
    """
    def __init__(self, dbenv=None):
        self.db = db.DB(dbenv)
        if HIGHEST_PROTOCOL:
            self.protocol = HIGHEST_PROTOCOL
        else:
            self.protocol = 1

    def __del__(self):
        self.close()

    def __getattr__(self, name):
        """Many methods we can just pass through to the DB object.
        (See below)
        """
        return getattr(self.db, name)

    #-----------------------------------
    # Dictionary access methods

    def __len__(self):
        return len(self.db)

    def __getitem__(self, key):
        data = self.db[key]
        return cPickle.loads(data)

    def __setitem__(self, key, value):
        data = _dumps(value, self.protocol)
        self.db[key] = data

    def __delitem__(self, key):
        del self.db[key]

    def keys(self, txn=None):
        if txn != None:
            return self.db.keys(txn)
        else:
            return self.db.keys()

    def items(self, txn=None):
        if txn != None:
            items = self.db.items(txn)
        else:
            items = self.db.items()
        newitems = []

        for k, v in items:
            newitems.append( (k, cPickle.loads(v)) )
        return newitems

    def values(self, txn=None):
        if txn != None:
            values = self.db.values(txn)
        else:
            values = self.db.values()

        return map(cPickle.loads, values)

    #-----------------------------------
    # Other methods

    def __append(self, value, txn=None):
        data = _dumps(value, self.protocol)
        return self.db.append(data, txn)

    def append(self, value, txn=None):
        if self.get_type() == db.DB_RECNO:
            self.append = self.__append
            return self.append(value, txn=txn)
        raise DBShelveError, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO"

    def associate(self, secondaryDB, callback, flags=0):
        def _shelf_callback(priKey, priData, realCallback=callback):
            data = cPickle.loads(priData)
            return realCallback(priKey, data)
        return self.db.associate(secondaryDB, _shelf_callback, flags)

    #def get(self, key, default=None, txn=None, flags=0):
    def get(self, *args, **kw):
        # We do it with *args and **kw so if the default value wasn't
        # given nothing is passed to the extension module.  That way
        # an exception can be raised if set_get_returns_none is turned
        # off.
        data = apply(self.db.get, args, kw)
        try:
            return cPickle.loads(data)
        except (TypeError, cPickle.UnpicklingError):
            return data  # we may be getting the default value, or None,
                         # so it doesn't need unpickled.

    def get_both(self, key, value, txn=None, flags=0):
        data = _dumps(value, self.protocol)
        data = self.db.get(key, data, txn, flags)
        return cPickle.loads(data)

    def cursor(self, txn=None, flags=0):
        c = DBShelfCursor(self.db.cursor(txn, flags))
        c.protocol = self.protocol
        return c

    def put(self, key, value, txn=None, flags=0):
        data = _dumps(value, self.protocol)
        return self.db.put(key, data, txn, flags)

    def join(self, cursorList, flags=0):
        raise NotImplementedError

    #----------------------------------------------
    # Methods allowed to pass-through to self.db
    #
    #    close,  delete, fd, get_byteswapped, get_type, has_key,
    #    key_range, open, remove, rename, stat, sync,
    #    upgrade, verify, and all set_* methods.

#---------------------------------------------------------------------------

class DBShelfCursor:
    """
    """
    def __init__(self, cursor):
        self.dbc = cursor

    def __del__(self):
        self.close()

    def __getattr__(self, name):
        """Some methods we can just pass through to the cursor object.  (See below)"""
        return getattr(self.dbc, name)

    #----------------------------------------------

    def dup(self, flags=0):
        c = DBShelfCursor(self.dbc.dup(flags))
        c.protocol = self.protocol
        return c

    def put(self, key, value, flags=0):
        data = _dumps(value, self.protocol)
        return self.dbc.put(key, data, flags)

    def get(self, *args):
        count = len(args)  # a method overloading hack
        method = getattr(self, 'get_%d' % count)
        apply(method, args)

    def get_1(self, flags):
        rec = self.dbc.get(flags)
        return self._extract(rec)

    def get_2(self, key, flags):
        rec = self.dbc.get(key, flags)
        return self._extract(rec)

    def get_3(self, key, value, flags):
        data = _dumps(value, self.protocol)
        rec = self.dbc.get(key, flags)
        return self._extract(rec)

    def current(self, flags=0): return self.get_1(flags|db.DB_CURRENT)
    def first(self, flags=0): return self.get_1(flags|db.DB_FIRST)
    def last(self, flags=0): return self.get_1(flags|db.DB_LAST)
    def next(self, flags=0): return self.get_1(flags|db.DB_NEXT)
    def prev(self, flags=0): return self.get_1(flags|db.DB_PREV)
    def consume(self, flags=0): return self.get_1(flags|db.DB_CONSUME)
    def next_dup(self, flags=0): return self.get_1(flags|db.DB_NEXT_DUP)
    def next_nodup(self, flags=0): return self.get_1(flags|db.DB_NEXT_NODUP)
    def prev_nodup(self, flags=0): return self.get_1(flags|db.DB_PREV_NODUP)

    def get_both(self, key, value, flags=0):
        data = _dumps(value, self.protocol)
        rec = self.dbc.get_both(key, flags)
        return self._extract(rec)

    def set(self, key, flags=0):
        rec = self.dbc.set(key, flags)
        return self._extract(rec)

    def set_range(self, key, flags=0):
        rec = self.dbc.set_range(key, flags)
        return self._extract(rec)

    def set_recno(self, recno, flags=0):
        rec = self.dbc.set_recno(recno, flags)
        return self._extract(rec)

    set_both = get_both

    def _extract(self, rec):
        if rec is None:
            return None
        else:
            key, data = rec
            return key, cPickle.loads(data)

    #----------------------------------------------
    # Methods allowed to pass-through to self.dbc
    #
    # close, count, delete, get_recno, join_item

#---------------------------------------------------------------------------
<commit_msg>Fix an uncollectable reference leak in bsddb.db.DBShelf.append<commit_after>#!/bin/env python
#------------------------------------------------------------------------
#           Copyright (c) 1997-2001 by Total Control Software
#                         All Rights Reserved
#------------------------------------------------------------------------
#
# Module Name:  dbShelve.py
#
# Description:  A reimplementation of the standard shelve.py that
#               forces the use of cPickle, and DB.
#
# Creation Date:    11/3/97 3:39:04PM
#
# License:      This is free software.  You may use this software for any
#               purpose including modification/redistribution, so long as
#               this header remains intact and that you do not claim any
#               rights of ownership or authorship of this software.  This
#               software has been tested, but no warranty is expressed or
#               implied.
#
# 13-Dec-2000:  Updated to be used with the new bsddb3 package.
#               Added DBShelfCursor class.
#
#------------------------------------------------------------------------

"""Manage shelves of pickled objects using bsddb database files for the
storage.
"""

#------------------------------------------------------------------------

import cPickle
import db
import sys

#At version 2.3 cPickle switched to using protocol instead of bin and
#DictMixin was added
if sys.version_info[:3] >= (2, 3, 0):
    HIGHEST_PROTOCOL = cPickle.HIGHEST_PROTOCOL
    def _dumps(object, protocol):
        return cPickle.dumps(object, protocol=protocol)
    from UserDict import DictMixin
else:
    HIGHEST_PROTOCOL = None
    def _dumps(object, protocol):
        return cPickle.dumps(object, bin=protocol)
    class DictMixin: pass

#------------------------------------------------------------------------

def open(filename, flags=db.DB_CREATE, mode=0660, filetype=db.DB_HASH,
         dbenv=None, dbname=None):
    """
    A simple factory function for compatibility with the standard
    shleve.py module.  It can be used like this, where key is a string
    and data is a pickleable object:

        from bsddb import dbshelve
        db = dbshelve.open(filename)

        db[key] = data

        db.close()
    """
    if type(flags) == type(''):
        sflag = flags
        if sflag == 'r':
            flags = db.DB_RDONLY
        elif sflag == 'rw':
            flags = 0
        elif sflag == 'w':
            flags =  db.DB_CREATE
        elif sflag == 'c':
            flags =  db.DB_CREATE
        elif sflag == 'n':
            flags = db.DB_TRUNCATE | db.DB_CREATE
        else:
            raise db.DBError, "flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags"

    d = DBShelf(dbenv)
    d.open(filename, dbname, filetype, flags, mode)
    return d

#---------------------------------------------------------------------------

class DBShelveError(db.DBError): pass

class DBShelf(DictMixin):
    """A shelf to hold pickled objects, built upon a bsddb DB object.  It
    automatically pickles/unpickles data objects going to/from the DB.
    """
    def __init__(self, dbenv=None):
        self.db = db.DB(dbenv)
        self._closed = True
        if HIGHEST_PROTOCOL:
            self.protocol = HIGHEST_PROTOCOL
        else:
            self.protocol = 1

    def __del__(self):
        self.close()

    def __getattr__(self, name):
        """Many methods we can just pass through to the DB object.
        (See below)
        """
        return getattr(self.db, name)

    #-----------------------------------
    # Dictionary access methods

    def __len__(self):
        return len(self.db)

    def __getitem__(self, key):
        data = self.db[key]
        return cPickle.loads(data)

    def __setitem__(self, key, value):
        data = _dumps(value, self.protocol)
        self.db[key] = data

    def __delitem__(self, key):
        del self.db[key]

    def keys(self, txn=None):
        if txn != None:
            return self.db.keys(txn)
        else:
            return self.db.keys()

    def open(self, *args, **kwargs):
        self.db.open(*args, **kwargs)
        self._closed = False

    def close(self, *args, **kwargs):
        self.db.close(*args, **kwargs)
        self._closed = True

    def __repr__(self):
        if self._closed:
            return '<DBShelf @ 0x%x - closed>' % (id(self))
        else:
            return repr(dict(self.iteritems()))

    def items(self, txn=None):
        if txn != None:
            items = self.db.items(txn)
        else:
            items = self.db.items()
        newitems = []

        for k, v in items:
            newitems.append( (k, cPickle.loads(v)) )
        return newitems

    def values(self, txn=None):
        if txn != None:
            values = self.db.values(txn)
        else:
            values = self.db.values()

        return map(cPickle.loads, values)

    #-----------------------------------
    # Other methods

    def __append(self, value, txn=None):
        data = _dumps(value, self.protocol)
        return self.db.append(data, txn)

    def append(self, value, txn=None):
        if self.get_type() == db.DB_RECNO:
            return self.__append(value, txn=txn)
        raise DBShelveError, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO"

    def associate(self, secondaryDB, callback, flags=0):
        def _shelf_callback(priKey, priData, realCallback=callback):
            data = cPickle.loads(priData)
            return realCallback(priKey, data)
        return self.db.associate(secondaryDB, _shelf_callback, flags)

    #def get(self, key, default=None, txn=None, flags=0):
    def get(self, *args, **kw):
        # We do it with *args and **kw so if the default value wasn't
        # given nothing is passed to the extension module.  That way
        # an exception can be raised if set_get_returns_none is turned
        # off.
        data = apply(self.db.get, args, kw)
        try:
            return cPickle.loads(data)
        except (TypeError, cPickle.UnpicklingError):
            return data  # we may be getting the default value, or None,
                         # so it doesn't need unpickled.

    def get_both(self, key, value, txn=None, flags=0):
        data = _dumps(value, self.protocol)
        data = self.db.get(key, data, txn, flags)
        return cPickle.loads(data)

    def cursor(self, txn=None, flags=0):
        c = DBShelfCursor(self.db.cursor(txn, flags))
        c.protocol = self.protocol
        return c

    def put(self, key, value, txn=None, flags=0):
        data = _dumps(value, self.protocol)
        return self.db.put(key, data, txn, flags)

    def join(self, cursorList, flags=0):
        raise NotImplementedError

    #----------------------------------------------
    # Methods allowed to pass-through to self.db
    #
    #    close,  delete, fd, get_byteswapped, get_type, has_key,
    #    key_range, open, remove, rename, stat, sync,
    #    upgrade, verify, and all set_* methods.

#---------------------------------------------------------------------------

class DBShelfCursor:
    """
    """
    def __init__(self, cursor):
        self.dbc = cursor

    def __del__(self):
        self.close()

    def __getattr__(self, name):
        """Some methods we can just pass through to the cursor object.  (See below)"""
        return getattr(self.dbc, name)

    #----------------------------------------------

    def dup(self, flags=0):
        c = DBShelfCursor(self.dbc.dup(flags))
        c.protocol = self.protocol
        return c

    def put(self, key, value, flags=0):
        data = _dumps(value, self.protocol)
        return self.dbc.put(key, data, flags)

    def get(self, *args):
        count = len(args)  # a method overloading hack
        method = getattr(self, 'get_%d' % count)
        apply(method, args)

    def get_1(self, flags):
        rec = self.dbc.get(flags)
        return self._extract(rec)

    def get_2(self, key, flags):
        rec = self.dbc.get(key, flags)
        return self._extract(rec)

    def get_3(self, key, value, flags):
        data = _dumps(value, self.protocol)
        rec = self.dbc.get(key, flags)
        return self._extract(rec)

    def current(self, flags=0): return self.get_1(flags|db.DB_CURRENT)
    def first(self, flags=0): return self.get_1(flags|db.DB_FIRST)
    def last(self, flags=0): return self.get_1(flags|db.DB_LAST)
    def next(self, flags=0): return self.get_1(flags|db.DB_NEXT)
    def prev(self, flags=0): return self.get_1(flags|db.DB_PREV)
    def consume(self, flags=0): return self.get_1(flags|db.DB_CONSUME)
    def next_dup(self, flags=0): return self.get_1(flags|db.DB_NEXT_DUP)
    def next_nodup(self, flags=0): return self.get_1(flags|db.DB_NEXT_NODUP)
    def prev_nodup(self, flags=0): return self.get_1(flags|db.DB_PREV_NODUP)

    def get_both(self, key, value, flags=0):
        data = _dumps(value, self.protocol)
        rec = self.dbc.get_both(key, flags)
        return self._extract(rec)

    def set(self, key, flags=0):
        rec = self.dbc.set(key, flags)
        return self._extract(rec)

    def set_range(self, key, flags=0):
        rec = self.dbc.set_range(key, flags)
        return self._extract(rec)

    def set_recno(self, recno, flags=0):
        rec = self.dbc.set_recno(recno, flags)
        return self._extract(rec)

    set_both = get_both

    def _extract(self, rec):
        if rec is None:
            return None
        else:
            key, data = rec
            return key, cPickle.loads(data)

    #----------------------------------------------
    # Methods allowed to pass-through to self.dbc
    #
    # close, count, delete, get_recno, join_item

#---------------------------------------------------------------------------
<|endoftext|>