<commit_before>import os

from django.utils.translation import ugettext_lazy as _

# Django settings for wafer project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'wafer.db',
    }
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'Africa/Johannesburg'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
MEDIA_ROOT = os.path.join(project_root, 'media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    # 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '8iysa30^no&oi5kv$k1w)#gsxzrylr-h6%)loz71expnbf7z%)'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
    # 'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'wafer.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'wafer.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or
    # "C:/www/django/templates". Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    'wafer.context_processors.site_info',
    'wafer.context_processors.menu_info',
    'wafer.context_processors.registration_settings',
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'django_medusa',
    'crispy_forms',
    'south',
    'django_nose',
    'registration',
    'markitup',
    'wafer',
    'wafer.invoices',
    'wafer.registration',
    'wafer.talks',
    'wafer.schedule',
    'wafer.users',
    'wafer.sponsors',
    'wafer.pages',
    'wafer.conf_registration',
)

TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

# Django registration:
ACCOUNT_ACTIVATION_DAYS = 7
AUTH_PROFILE_MODULE = 'users.UserProfile'

# Forms:
CRISPY_FAIL_SILENTLY = not DEBUG

import django.template
django.template.add_to_builtins('django.templatetags.future')
django.template.add_to_builtins('django.templatetags.i18n')

CRISPY_TEMPLATE_PACK = 'bootstrap3'

# Wafer settings
WAFER_BILLABLE_ME = "http://billable.me/pdf/"

# Wafer menu settings

WAFER_MENUS = ()
# Example menus entries:
#
#    {"label": _("Home"),
#     "url": '/'},
#    {"menu": "sponsors",
#     "label": _("Sponsors"),
#     "items": [
#         {"name": "sponsors", "label": _("Our sponsors"),
#          "url": reverse_lazy("wafer_sponsors")},
#         {"name": "packages", "label": _("Sponsorship packages"),
#          "url": reverse_lazy("wafer_sponsorship_packages")},
#     ]},
#    {"label": _("Talks"),
#     "url": reverse_lazy("wafer_users_talks")},

WAFER_DYNAMIC_MENUS = (
    'wafer.pages.models.page_menus',
)

# Registration mode
# This controls what happens when a user attempts to register
# if both WAITLIST and REGISTRATION_OPEN are false, no registration is possible
# if REGISTRTION_OPEN is true, and WAITLIST is false, registration proceeds
# as normal (registration items are created in the admin interface)
# if WAITLIST is true, registration puts people on the waitlist rather
# than registering. In this case, REGISTRATION_OPEN determines message sent
# to the user.
# Registration will also go to the WAITLIST if REGISTER_LIMIT is > 0 and
# the number of attendees exceeds that limit
WAFER_WAITLIST_ON = False
WAFER_REGISTRATION_OPEN = False
# Set to 0 for no limit
WAFER_REGISTRATION_LIMIT = 250

# Log in with GitHub:
# AUTHENTICATION_BACKENDS = (
#     'django.contrib.auth.backends.ModelBackend',
#     'wafer.registration.backends.GitHubBackend',
# )
# WAFER_GITHUB_CLIENT_ID = 'register on github'
# WAFER_GITHUB_CLIENT_SECRET = 'to get these secrets'

# Set this to true to disable the login button on the navigation toolbar
WAFER_HIDE_LOGIN = False

# Talk submissions open
# Set this to False to disable talk submissions
WAFER_TALKS_OPEN = True

# django_medusa -- disk-based renderer
MEDUSA_RENDERER_CLASS = "wafer.management.static.WaferDiskStaticSiteRenderer"
MEDUSA_DEPLOY_DIR = os.path.join(project_root, 'static_mirror')
MARKITUP_FILTER = ('markdown.markdown', {'safe_mode': True})
<commit_msg>Remove currently unused apps<commit_after>import os

from django.utils.translation import ugettext_lazy as _

# Django settings for wafer project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'wafer.db',
    }
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'Africa/Johannesburg'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
MEDIA_ROOT = os.path.join(project_root, 'media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    # 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '8iysa30^no&oi5kv$k1w)#gsxzrylr-h6%)loz71expnbf7z%)'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
    # 'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'wafer.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'wafer.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or
    # "C:/www/django/templates". Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    'wafer.context_processors.site_info',
    'wafer.context_processors.menu_info',
    'wafer.context_processors.registration_settings',
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'django_medusa',
    'crispy_forms',
    'south',
    'django_nose',
    'registration',
    'markitup',
    'wafer',
    'wafer.registration',
    'wafer.talks',
    'wafer.schedule',
    'wafer.users',
    'wafer.sponsors',
    'wafer.pages',
)

TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

# Django registration:
ACCOUNT_ACTIVATION_DAYS = 7
AUTH_PROFILE_MODULE = 'users.UserProfile'

# Forms:
CRISPY_FAIL_SILENTLY = not DEBUG

import django.template
django.template.add_to_builtins('django.templatetags.future')
django.template.add_to_builtins('django.templatetags.i18n')

CRISPY_TEMPLATE_PACK = 'bootstrap3'

# Wafer settings
WAFER_BILLABLE_ME = "http://billable.me/pdf/"

# Wafer menu settings

WAFER_MENUS = ()
# Example menus entries:
#
#    {"label": _("Home"),
#     "url": '/'},
#    {"menu": "sponsors",
#     "label": _("Sponsors"),
#     "items": [
#         {"name": "sponsors", "label": _("Our sponsors"),
#          "url": reverse_lazy("wafer_sponsors")},
#         {"name": "packages", "label": _("Sponsorship packages"),
#          "url": reverse_lazy("wafer_sponsorship_packages")},
#     ]},
#    {"label": _("Talks"),
#     "url": reverse_lazy("wafer_users_talks")},

WAFER_DYNAMIC_MENUS = (
    'wafer.pages.models.page_menus',
)

# Registration mode
# This controls what happens when a user attempts to register
# if both WAITLIST and REGISTRATION_OPEN are false, no registration is possible
# if REGISTRTION_OPEN is true, and WAITLIST is false, registration proceeds
# as normal (registration items are created in the admin interface)
# if WAITLIST is true, registration puts people on the waitlist rather
# than registering. In this case, REGISTRATION_OPEN determines message sent
# to the user.
# Registration will also go to the WAITLIST if REGISTER_LIMIT is > 0 and
# the number of attendees exceeds that limit
WAFER_WAITLIST_ON = False
WAFER_REGISTRATION_OPEN = False
# Set to 0 for no limit
WAFER_REGISTRATION_LIMIT = 250

# Log in with GitHub:
# AUTHENTICATION_BACKENDS = (
#     'django.contrib.auth.backends.ModelBackend',
#     'wafer.registration.backends.GitHubBackend',
# )
# WAFER_GITHUB_CLIENT_ID = 'register on github'
# WAFER_GITHUB_CLIENT_SECRET = 'to get these secrets'

# Set this to true to disable the login button on the navigation toolbar
WAFER_HIDE_LOGIN = False

# Talk submissions open
# Set this to False to disable talk submissions
WAFER_TALKS_OPEN = True

# django_medusa -- disk-based renderer
MEDUSA_RENDERER_CLASS = "wafer.management.static.WaferDiskStaticSiteRenderer"
MEDUSA_DEPLOY_DIR = os.path.join(project_root, 'static_mirror')
MARKITUP_FILTER = ('markdown.markdown', {'safe_mode': True})
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
_release_

Implement git cirrus release command

"""
import os
import sys
import datetime
import itertools
from collections import OrderedDict
from fabric.operations import put, local

from argparse import ArgumentParser
from cirrus.configuration import load_configuration
from cirrus.configuration import get_pypi_auth
from cirrus.github_tools import build_release_notes
from cirrus.git_tools import checkout_and_pull, push
from cirrus.git_tools import branch, merge
from cirrus.git_tools import commit_files
from cirrus.git_tools import tag_release, get_active_branch
from cirrus.utils import update_file, update_version
from cirrus.fabric_helpers import FabricHelper
from cirrus.logger import get_logger

LOGGER = get_logger()

def highlander(iterable):
    """check only single True value in iterable"""
    # There Can Be Only One!!!
    i = iter(iterable)
    return any(i) and not any(i)

def parse_version(version):
    """
    _parse_version_

    Parse semantic major.minor.micro version string

    :param version: X.Y.Z format version string

    :returns: dictionary containing major, minor, micro versions
    as integers

    """
    split = version.split('.', 2)
    return {
        'major': int(split[0]),
        'minor': int(split[1]),
        'micro': int(split[2]),
    }

def bump_version_field(version, field='major'):
    """
    parse the version and update the major, minor and micro
    version specified by field
    Return the updated version string
    """
    vers_params = parse_version(version)
    vers_params[field] += 1
    if field == 'major':
        vers_params['minor'] = 0
        vers_params['micro'] = 0
    elif field == 'minor':
        vers_params['micro'] = 0

    return "{major}.{minor}.{micro}".format(**vers_params)

def artifact_name(config):
    """
    given cirrus config, build the expected
    artifact name
    """
    artifact_name = "{0}-{1}.tar.gz".format(
        config.package_name(),
        config.package_version()
    )
    build_artifact = os.path.join(
        os.getcwd(),
        'dist',
        artifact_name
    )
    return build_artifact

def release_branch_name(config):
    """
    build expected release branch name from current config

    """
    branch_name = "{0}{1}".format(
        config.gitflow_release_prefix(),
        config.package_version()
    )
    return branch_name

def build_parser(argslist):
    """
    _build_parser_

    Set up command line parser for the release command

    """
    parser = ArgumentParser(
        description='git cirrus release command'
    )
    parser.add_argument('command', nargs='?')

    subparsers = parser.add_subparsers(dest='command')
    new_command = subparsers.add_parser('new')
    new_command.add_argument(
        '--micro',
        action='store_true',
        dest='micro'
    )
    new_command.add_argument(
        '--minor',
        action='store_true',
        dest='minor'
    )
    new_command.add_argument(
        '--major',
        action='store_true',
        dest='major'
    )
    new_command.add_argument(
        '--bump',
        nargs='+',
        help='package versions (pkg==0.0.0) to update in requirements.txt'
    )

    build_command = subparsers.add_parser('build')

    upload_command = subparsers.add_parser('upload')
    upload_command.add_argument(
        '--test',
        action='store_true',
        dest='test',
        help="test only, do not actually push or upload"
    )
    upload_command.add_argument(
        '--no-upload',
        action='store_true',
        dest='no_upload',
        help="do not upload build artifact to pypi"
    )
    upload_command.add_argument(
        '--pypi-sudo',
        action='store_true',
        dest='pypi_sudo',
        help="use sudo to upload build artifact to pypi"
    )
    upload_command.add_argument(
        '--no-pypi-sudo',
        action='store_false',
        dest='pypi_sudo',
        help="do not use sudo to upload build artifact to pypi"
    )
    upload_command.set_defaults(pypi_sudo=True)

    opts = parser.parse_args(argslist)
    return opts

def new_release(opts):
    """
    _new_release_

    - Create a new release branch in the local repo
    - Edit the conf to bump the version
    - Edit the history file with release notes

    """
    LOGGER.info("Creating new release...")
    if not highlander([opts.major, opts.minor, opts.micro]):
        msg = "Can only specify one of --major, --minor or --micro"
        LOGGER.error(msg)
        raise RuntimeError(msg)

    fields = ['major', 'minor', 'micro']
    mask = [opts.major, opts.minor, opts.micro]
    field = [x for x in itertools.compress(fields, mask)][0]

    config = load_configuration()

    if opts.bump:
        for pkg in opts.bump:
            if '==' not in pkg:
                msg = 'Malformed version expression.  Please use "pkg==0.0.0"'
                LOGGER.error(msg)
                raise RuntimeError(msg)
        try:
            update_requirements('requirements.txt', opts.bump)
        except Exception as ex:
            # halt on any problem updating requirements
            LOGGER.exception('Failed to update requirements.txt -- {}'.format(ex))
            raise RuntimeError(ex)

    # version bump:
    current_version = config.package_version()
    new_version = bump_version_field(current_version, field)

    # release branch
    branch_name = "{0}{1}".format(
        config.gitflow_release_prefix(),
        new_version
    )
    LOGGER.info('release branch is {0}'.format(branch_name))

    # need to be on the latest develop
    repo_dir = os.getcwd()
    main_branch = config.gitflow_branch_name()
    checkout_and_pull(repo_dir,  main_branch)

    # create release branch
    branch(repo_dir, branch_name, main_branch)

    # update cirrus conf
    config.update_package_version(new_version)
    changes = ['cirrus.conf']

    if opts.bump:
        changes.append('requirements.txt')

    # update release notes file
    relnotes_file, relnotes_sentinel = config.release_notes()
    if (relnotes_file is not None) and (relnotes_sentinel is not None):
        LOGGER.info('Updating release notes in {0}'.format(relnotes_file))
        relnotes = "Release: {0} Created: {1}\n".format(
            new_version,
            datetime.datetime.utcnow().isoformat()
        )
        relnotes += build_release_notes(
            config.organisation_name(),
            config.package_name(),
            current_version,
            config.release_notes_format()
        )
        update_file(relnotes_file, relnotes_sentinel, relnotes)
        changes.append(relnotes_file)

    # update __version__ or equivalent
    version_file, version_attr = config.version_file()
    if version_file is not None:
        LOGGER.info('Updating {0} attribute in {1}'.format(version_file, version_attr))
        update_version(version_file, new_version, version_attr)
        changes.append(version_file)

    # update files changed
    msg = "cirrus release: new release created for {0}".format(branch_name)
    LOGGER.info('Committing files: {0}'.format(','.join(changes)))
    LOGGER.info(msg)
    commit_files(repo_dir, msg, *changes)
    return

def upload_release(opts):
    """
    _upload_release_
    """
    LOGGER.info("Uploading release...")
    config = load_configuration()
    build_artifact = artifact_name(config)
    LOGGER.info("Uploading artifact: {0}".format(build_artifact))
    repo_dir = os.getcwd()
    curr_branch = get_active_branch(repo_dir)
    expected_branch = release_branch_name(config)

    if curr_branch.name != expected_branch:
        msg = (
            "Not on the expected release branch according "
            "to cirrus.conf\n Expected:{0} but on {1}"
        ).format(expected_branch, curr_branch)
        LOGGER.error(msg)
        raise RuntimeError(msg)

    if not os.path.exists(build_artifact):
        msg = (
            "Expected build artifact: {0} Not Found, upload aborted\n"
            "Did you run git cirrus release build?"
        ).format(build_artifact)
        LOGGER.error(msg)
        raise RuntimeError(msg)

    # upload to pypi via fabric over ssh
    if opts.no_upload or opts.test:
        LOGGER.info("Uploading to pypi disabled by test or no-upload option...")
    else:
        pypi_conf = config.pypi_config()
        pypi_auth = get_pypi_auth()
        package_dir = pypi_conf['pypi_upload_path']
        LOGGER.info("Uploading {0} to {1}".format(build_artifact, pypi_conf['pypi_url']))
        with FabricHelper(
                pypi_conf['pypi_url'],
                pypi_auth['username'],
                pypi_auth['ssh_key']):

            # fabric put the file onto the pypi server
            put(build_artifact, package_dir, use_sudo=opts.pypi_sudo)

    # merge in release branches and tag, push to remote
    tag = config.package_version()
    master = config.gitflow_master_name()
    develop = config.gitflow_branch_name()

    # merge release branch into master
    LOGGER.info("Tagging and pushing {0}".format(tag))
    checkout_and_pull(repo_dir, master)
    merge(repo_dir, master, expected_branch)

    if not opts.test:
        LOGGER.info("pushing to remote...")
        push(repo_dir)
    do_push = not opts.test
    tag_release(repo_dir, tag, master, push=do_push)

    # Merge release branch back to develop
    # push to develop
    LOGGER.info("Merging back to develop...")
    checkout_and_pull(repo_dir, develop)
    merge(repo_dir, develop, expected_branch)
    if not opts.test:
        LOGGER.info("pushing to remote...")
        push(repo_dir)
    LOGGER.info("Release {0} has been tagged and uploaded".format(tag))
    # TODO: clean up release branch
    return

def build_release(opts):
    """
    _build_release_

    run python setup.py sdist to create the release artifact

    """
    LOGGER.info("Building release...")
    config = load_configuration()
    local('python setup.py sdist')
    build_artifact = artifact_name(config)
    if not os.path.exists(build_artifact):
        msg = "Expected build artifact: {0} Not Found".format(build_artifact)
        LOGGER.error(msg)
        raise RuntimeError(msg)
    LOGGER.info("Release artifact created: {0}".format(build_artifact))
    return build_artifact

def update_requirements(path, versions):
    """
    _update_requirements_

    Update requirements.txt with the provided versions list, where each item in
    such a list looks exactly like a version specifier in a requirements.txt
    file.  Items without '==' are ignored.

    Example:
    ['foo==0.0.9', 'bar==1.2']
    """
    LOGGER.info('Updating {}'.format(path))

    with open(path, 'r+') as fh:
        # original requirements.txt in its pristine order
        reqs = OrderedDict(tuple(pkg.split('=='))
                           for pkg in fh.readlines() if '==' in pkg)

        for pkg in versions:
            new_pkg, new_version = pkg.split('==')
            reqs[new_pkg] = '{}\n'.format(new_version)
            LOGGER.info('{}=={}'.format(new_pkg, new_version))

        # overwrite original requirements.txt with updated version
        fh.seek(0)
        fh.writelines(['{}=={}'.format(pkg, version)
                      for pkg, version in reqs.iteritems()])
        fh.truncate()

def main():
    opts = build_parser(sys.argv)
    if opts.command == 'new':
        new_release(opts)

    if opts.command == 'upload':
        upload_release(opts)

    if opts.command == 'build':
        build_release(opts)

if __name__ == '__main__':

    main()
<commit_msg>Move requirements update after checkout_and_pull<commit_after>#!/usr/bin/env python
"""
_release_

Implement git cirrus release command

"""
import os
import sys
import datetime
import itertools
from collections import OrderedDict
from fabric.operations import put, local

from argparse import ArgumentParser
from cirrus.configuration import load_configuration
from cirrus.configuration import get_pypi_auth
from cirrus.github_tools import build_release_notes
from cirrus.git_tools import checkout_and_pull, push
from cirrus.git_tools import branch, merge
from cirrus.git_tools import commit_files
from cirrus.git_tools import tag_release, get_active_branch
from cirrus.utils import update_file, update_version
from cirrus.fabric_helpers import FabricHelper
from cirrus.logger import get_logger

LOGGER = get_logger()

def highlander(iterable):
    """check only single True value in iterable"""
    # There Can Be Only One!!!
    i = iter(iterable)
    return any(i) and not any(i)

def parse_version(version):
    """
    _parse_version_

    Parse semantic major.minor.micro version string

    :param version: X.Y.Z format version string

    :returns: dictionary containing major, minor, micro versions
    as integers

    """
    split = version.split('.', 2)
    return {
        'major': int(split[0]),
        'minor': int(split[1]),
        'micro': int(split[2]),
    }

def bump_version_field(version, field='major'):
    """
    parse the version and update the major, minor and micro
    version specified by field
    Return the updated version string
    """
    vers_params = parse_version(version)
    vers_params[field] += 1
    if field == 'major':
        vers_params['minor'] = 0
        vers_params['micro'] = 0
    elif field == 'minor':
        vers_params['micro'] = 0

    return "{major}.{minor}.{micro}".format(**vers_params)

def artifact_name(config):
    """
    given cirrus config, build the expected
    artifact name
    """
    artifact_name = "{0}-{1}.tar.gz".format(
        config.package_name(),
        config.package_version()
    )
    build_artifact = os.path.join(
        os.getcwd(),
        'dist',
        artifact_name
    )
    return build_artifact

def release_branch_name(config):
    """
    build expected release branch name from current config

    """
    branch_name = "{0}{1}".format(
        config.gitflow_release_prefix(),
        config.package_version()
    )
    return branch_name

def build_parser(argslist):
    """
    _build_parser_

    Set up command line parser for the release command

    """
    parser = ArgumentParser(
        description='git cirrus release command'
    )
    parser.add_argument('command', nargs='?')

    subparsers = parser.add_subparsers(dest='command')
    new_command = subparsers.add_parser('new')
    new_command.add_argument(
        '--micro',
        action='store_true',
        dest='micro'
    )
    new_command.add_argument(
        '--minor',
        action='store_true',
        dest='minor'
    )
    new_command.add_argument(
        '--major',
        action='store_true',
        dest='major'
    )
    new_command.add_argument(
        '--bump',
        nargs='+',
        help='package versions (pkg==0.0.0) to update in requirements.txt'
    )

    build_command = subparsers.add_parser('build')

    upload_command = subparsers.add_parser('upload')
    upload_command.add_argument(
        '--test',
        action='store_true',
        dest='test',
        help="test only, do not actually push or upload"
    )
    upload_command.add_argument(
        '--no-upload',
        action='store_true',
        dest='no_upload',
        help="do not upload build artifact to pypi"
    )
    upload_command.add_argument(
        '--pypi-sudo',
        action='store_true',
        dest='pypi_sudo',
        help="use sudo to upload build artifact to pypi"
    )
    upload_command.add_argument(
        '--no-pypi-sudo',
        action='store_false',
        dest='pypi_sudo',
        help="do not use sudo to upload build artifact to pypi"
    )
    upload_command.set_defaults(pypi_sudo=True)

    opts = parser.parse_args(argslist)
    return opts

def new_release(opts):
    """
    _new_release_

    - Create a new release branch in the local repo
    - Edit the conf to bump the version
    - Edit the history file with release notes

    """
    LOGGER.info("Creating new release...")
    if not highlander([opts.major, opts.minor, opts.micro]):
        msg = "Can only specify one of --major, --minor or --micro"
        LOGGER.error(msg)
        raise RuntimeError(msg)

    fields = ['major', 'minor', 'micro']
    mask = [opts.major, opts.minor, opts.micro]
    field = [x for x in itertools.compress(fields, mask)][0]

    config = load_configuration()

    # version bump:
    current_version = config.package_version()
    new_version = bump_version_field(current_version, field)

    # release branch
    branch_name = "{0}{1}".format(
        config.gitflow_release_prefix(),
        new_version
    )
    LOGGER.info('release branch is {0}'.format(branch_name))

    # need to be on the latest develop
    repo_dir = os.getcwd()
    main_branch = config.gitflow_branch_name()
    checkout_and_pull(repo_dir,  main_branch)

    # create release branch
    branch(repo_dir, branch_name, main_branch)

    # update cirrus conf
    config.update_package_version(new_version)
    changes = ['cirrus.conf']

    if opts.bump:
        for pkg in opts.bump:
            if '==' not in pkg:
                msg = 'Malformed version expression.  Please use "pkg==0.0.0"'
                LOGGER.error(msg)
                raise RuntimeError(msg)
        try:
            update_requirements('requirements.txt', opts.bump)
            changes.append('requirements.txt')
        except Exception as ex:
            # halt on any problem updating requirements
            LOGGER.exception('Failed to update requirements.txt -- {}'.format(ex))
            raise RuntimeError(ex)

    # update release notes file
    relnotes_file, relnotes_sentinel = config.release_notes()
    if (relnotes_file is not None) and (relnotes_sentinel is not None):
        LOGGER.info('Updating release notes in {0}'.format(relnotes_file))
        relnotes = "Release: {0} Created: {1}\n".format(
            new_version,
            datetime.datetime.utcnow().isoformat()
        )
        relnotes += build_release_notes(
            config.organisation_name(),
            config.package_name(),
            current_version,
            config.release_notes_format()
        )
        update_file(relnotes_file, relnotes_sentinel, relnotes)
        changes.append(relnotes_file)

    # update __version__ or equivalent
    version_file, version_attr = config.version_file()
    if version_file is not None:
        LOGGER.info('Updating {0} attribute in {1}'.format(version_file, version_attr))
        update_version(version_file, new_version, version_attr)
        changes.append(version_file)

    # update files changed
    msg = "cirrus release: new release created for {0}".format(branch_name)
    LOGGER.info('Committing files: {0}'.format(','.join(changes)))
    LOGGER.info(msg)
    commit_files(repo_dir, msg, *changes)
    return

def upload_release(opts):
    """
    _upload_release_
    """
    LOGGER.info("Uploading release...")
    config = load_configuration()
    build_artifact = artifact_name(config)
    LOGGER.info("Uploading artifact: {0}".format(build_artifact))
    repo_dir = os.getcwd()
    curr_branch = get_active_branch(repo_dir)
    expected_branch = release_branch_name(config)

    if curr_branch.name != expected_branch:
        msg = (
            "Not on the expected release branch according "
            "to cirrus.conf\n Expected:{0} but on {1}"
        ).format(expected_branch, curr_branch)
        LOGGER.error(msg)
        raise RuntimeError(msg)

    if not os.path.exists(build_artifact):
        msg = (
            "Expected build artifact: {0} Not Found, upload aborted\n"
            "Did you run git cirrus release build?"
        ).format(build_artifact)
        LOGGER.error(msg)
        raise RuntimeError(msg)

    # upload to pypi via fabric over ssh
    if opts.no_upload or opts.test:
        LOGGER.info("Uploading to pypi disabled by test or no-upload option...")
    else:
        pypi_conf = config.pypi_config()
        pypi_auth = get_pypi_auth()
        package_dir = pypi_conf['pypi_upload_path']
        LOGGER.info("Uploading {0} to {1}".format(build_artifact, pypi_conf['pypi_url']))
        with FabricHelper(
                pypi_conf['pypi_url'],
                pypi_auth['username'],
                pypi_auth['ssh_key']):

            # fabric put the file onto the pypi server
            put(build_artifact, package_dir, use_sudo=opts.pypi_sudo)

    # merge in release branches and tag, push to remote
    tag = config.package_version()
    master = config.gitflow_master_name()
    develop = config.gitflow_branch_name()

    # merge release branch into master
    LOGGER.info("Tagging and pushing {0}".format(tag))
    checkout_and_pull(repo_dir, master)
    merge(repo_dir, master, expected_branch)

    if not opts.test:
        LOGGER.info("pushing to remote...")
        push(repo_dir)
    do_push = not opts.test
    tag_release(repo_dir, tag, master, push=do_push)

    # Merge release branch back to develop
    # push to develop
    LOGGER.info("Merging back to develop...")
    checkout_and_pull(repo_dir, develop)
    merge(repo_dir, develop, expected_branch)
    if not opts.test:
        LOGGER.info("pushing to remote...")
        push(repo_dir)
    LOGGER.info("Release {0} has been tagged and uploaded".format(tag))
    # TODO: clean up release branch
    return

def build_release(opts):
    """
    _build_release_

    run python setup.py sdist to create the release artifact

    """
    LOGGER.info("Building release...")
    config = load_configuration()
    local('python setup.py sdist')
    build_artifact = artifact_name(config)
    if not os.path.exists(build_artifact):
        msg = "Expected build artifact: {0} Not Found".format(build_artifact)
        LOGGER.error(msg)
        raise RuntimeError(msg)
    LOGGER.info("Release artifact created: {0}".format(build_artifact))
    return build_artifact

def update_requirements(path, versions):
    """
    _update_requirements_

    Update requirements.txt with the provided versions list, where each item in
    such a list looks exactly like a version specifier in a requirements.txt
    file.  Items without '==' are ignored.

    Example:
    ['foo==0.0.9', 'bar==1.2']
    """
    LOGGER.info('Updating {}'.format(path))

    with open(path, 'r+') as fh:
        # original requirements.txt in its pristine order
        reqs = OrderedDict(tuple(pkg.split('=='))
                           for pkg in fh.readlines() if '==' in pkg)

        for pkg in versions:
            new_pkg, new_version = pkg.split('==')
            reqs[new_pkg] = '{}\n'.format(new_version)
            LOGGER.info('{}=={}'.format(new_pkg, new_version))

        # overwrite original requirements.txt with updated version
        fh.seek(0)
        fh.writelines(['{}=={}'.format(pkg, version)
                      for pkg, version in reqs.iteritems()])
        fh.truncate()

def main():
    opts = build_parser(sys.argv)
    if opts.command == 'new':
        new_release(opts)

    if opts.command == 'upload':
        upload_release(opts)

    if opts.command == 'build':
        build_release(opts)

if __name__ == '__main__':

    main()
<|endoftext|>


<commit_before>import os

from django.utils.translation import ugettext_lazy as _

# Django settings for wafer project.

ADMINS = (
    # The logging config below mails admins
    # ('Your Name', 'your_email@example.com'),
)

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'wafer.db',
    }
}

if os.environ.get('TESTDB', None) == 'postgres':
    DATABASES['default'].update({
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'USER': 'postgres',
        'NAME': 'wafer',
        })

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'UTC'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
MEDIA_ROOT = os.path.join(project_root, 'media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = os.path.join(project_root, 'static')

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    # 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '8iysa30^no&oi5kv$k1w)#gsxzrylr-h6%)loz71expnbf7z%)'

TEMPLATES = [
    {
        'APP_DIRS': True,
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': (
            # Put strings here, like "/home/html/django_templates" or
            # "C:/www/django/templates". Always use forward slashes, even on Windows.
            # Don't forget to use absolute paths, not relative paths.
        ),
        'OPTIONS': {
            'context_processors': (
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
                'wafer.context_processors.site_info',
                'wafer.context_processors.navigation_info',
                'wafer.context_processors.menu_info',
                'wafer.context_processors.registration_settings'
            )
        }
    },
]

MIDDLEWARE = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.contrib.redirects.middleware.RedirectFallbackMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'wafer.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'wafer.wsgi.application'

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.redirects',
    'reversion',
    'bakery',
    'crispy_forms',
    'django_nose',
    'rest_framework',
    'easy_select2',
    'wafer',
    'wafer.kv',
    'wafer.registration',
    'wafer.talks',
    'wafer.schedule',
    'wafer.users',
    'wafer.sponsors',
    'wafer.pages',
    'wafer.tickets',
    'wafer.compare',
    # Django isn't finding the overridden templates
    'markitup',
    'registration',
    'django.contrib.admin',
)

TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

# Django registration:
ACCOUNT_ACTIVATION_DAYS = 7

AUTH_USER_MODEL = 'auth.User'

# Forms:
CRISPY_TEMPLATE_PACK = 'bootstrap4'

# Wafer cache settings
# We assume that the WAFER_CACHE is cross-process
WAFER_CACHE = 'wafer_cache'
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
    },
    WAFER_CACHE: {
        'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
        'LOCATION': 'wafer_cache_table',
    },
}

# Wafer menu settings

WAFER_MENUS = ()
# Example menus entries:
#
#    {"label": _("Home"),
#     "url": '/'},
#    {"menu": "sponsors",
#     "label": _("Sponsors"),
#     "items": [
#         {"name": "sponsors", "label": _("Our sponsors"),
#          "url": reverse_lazy("wafer_sponsors")},
#         {"name": "packages", "label": _("Sponsorship packages"),
#          "url": reverse_lazy("wafer_sponsorship_packages")},
#     ]},
#    {"label": _("Talks"),
#     "url": reverse_lazy("wafer_users_talks")},

WAFER_DYNAMIC_MENUS = (
    'wafer.pages.models.page_menus',
    'wafer.sponsors.models.sponsor_menu',
)

# Enabled SSO mechanims:
WAFER_SSO = (
    # 'github',
    # 'debian',
)

# Log in with GitHub:
# WAFER_GITHUB_CLIENT_ID = 'register on github'
# WAFER_GITHUB_CLIENT_SECRET = 'to get these secrets'

# Log in with Debian SSO:
# Requires some Apache config:
# SSLCACertificateFile /srv/sso.debian.org/etc/debsso.crt
# SSLCARevocationCheck chain
# SSLCARevocationFile /srv/sso.debian.org/etc/debsso.crl
# <Location /accounts/debian-login/>
#     SSLOptions +StdEnvVars
#     SSLVerifyClient optional
# </Location>
# WAFER_DEBIAN_NM_API_KEY = 'obtain one from https://nm.debian.org/apikeys/'

# Set this to true to disable the login button on the navigation toolbar
WAFER_HIDE_LOGIN = False

# Talk submissions open
# Set this to False to disable talk submissions
WAFER_TALKS_OPEN = True

# The form used for talk submission
WAFER_TALK_FORM = 'wafer.talks.forms.TalkForm'

# Ask speakers for video release, and an email address of a reviewer
WAFER_VIDEO = True
WAFER_VIDEO_REVIEWER = True
WAFER_VIDEO_LICENSE = 'CC BY-SA 4.0'
WAFER_VIDEO_LICENSE_URL = 'https://creativecommons.org/licenses/by-sa/4.0/'

# Range of scores for talk reviews (inclusive)
WAFER_TALK_REVIEW_SCORES = (-2, 2)

# Set this to False to disable registration
WAFER_REGISTRATION_OPEN = True

# WAFER_REGISTRATION_MODE can be 'ticket' for Quicket tickets, or 'custom' if
# you implement your own registration system. If 'custom', then you *must*
# define a URL named 'register' in your application so we can link to it.
WAFER_REGISTRATION_MODE = 'ticket'
# WAFER_USER_IS_REGISTERED should return a boolean, when passed a Django user.
WAFER_USER_IS_REGISTERED = 'wafer.tickets.models.user_is_registered'

# Allow registered and anonymous users to see registered users
WAFER_PUBLIC_ATTENDEE_LIST = True

# Ticket registration with Quicket
# WAFER_TICKET_SECRET = "i'm a shared secret"

# django-bakery -- disk-based renderer
BUILD_DIR = os.path.join(project_root, 'static_mirror')

MARKITUP_FILTER = ('markdown.markdown', {'safe_mode': True})
JQUERY_URL = 'vendor/jquery/dist/jquery.min.js'
SELECT2_USE_BUNDLED_JQUERY = False

REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
    'PAGE_SIZE': 50
}

BAKERY_VIEWS = (
    'wafer.pages.views.ShowPage',
    'wafer.schedule.views.VenueView',
    'wafer.schedule.views.ScheduleView',
    'wafer.schedule.views.ScheduleXmlView',
    'wafer.sponsors.views.ShowSponsors',
    'wafer.sponsors.views.ShowPackages',
    'wafer.sponsors.views.SponsorView',
    'wafer.talks.views.TalkView',
    'wafer.talks.views.Speakers',
    'wafer.talks.views.TracksView',
    'wafer.talks.views.TalkTypesView',
    'wafer.talks.views.UsersTalks',
    'wafer.users.views.UsersView',
    'wafer.users.views.ProfileView',
)
<commit_msg>Add iCal View to export list<commit_after>import os

from django.utils.translation import ugettext_lazy as _

# Django settings for wafer project.

ADMINS = (
    # The logging config below mails admins
    # ('Your Name', 'your_email@example.com'),
)

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'wafer.db',
    }
}

if os.environ.get('TESTDB', None) == 'postgres':
    DATABASES['default'].update({
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'USER': 'postgres',
        'NAME': 'wafer',
        })

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'UTC'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
MEDIA_ROOT = os.path.join(project_root, 'media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = os.path.join(project_root, 'static')

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    # 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '8iysa30^no&oi5kv$k1w)#gsxzrylr-h6%)loz71expnbf7z%)'

TEMPLATES = [
    {
        'APP_DIRS': True,
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': (
            # Put strings here, like "/home/html/django_templates" or
            # "C:/www/django/templates". Always use forward slashes, even on Windows.
            # Don't forget to use absolute paths, not relative paths.
        ),
        'OPTIONS': {
            'context_processors': (
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.debug',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
                'wafer.context_processors.site_info',
                'wafer.context_processors.navigation_info',
                'wafer.context_processors.menu_info',
                'wafer.context_processors.registration_settings'
            )
        }
    },
]

MIDDLEWARE = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.contrib.redirects.middleware.RedirectFallbackMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'wafer.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'wafer.wsgi.application'

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.redirects',
    'reversion',
    'bakery',
    'crispy_forms',
    'django_nose',
    'rest_framework',
    'easy_select2',
    'wafer',
    'wafer.kv',
    'wafer.registration',
    'wafer.talks',
    'wafer.schedule',
    'wafer.users',
    'wafer.sponsors',
    'wafer.pages',
    'wafer.tickets',
    'wafer.compare',
    # Django isn't finding the overridden templates
    'markitup',
    'registration',
    'django.contrib.admin',
)

TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

# Django registration:
ACCOUNT_ACTIVATION_DAYS = 7

AUTH_USER_MODEL = 'auth.User'

# Forms:
CRISPY_TEMPLATE_PACK = 'bootstrap4'

# Wafer cache settings
# We assume that the WAFER_CACHE is cross-process
WAFER_CACHE = 'wafer_cache'
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
    },
    WAFER_CACHE: {
        'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
        'LOCATION': 'wafer_cache_table',
    },
}

# Wafer menu settings

WAFER_MENUS = ()
# Example menus entries:
#
#    {"label": _("Home"),
#     "url": '/'},
#    {"menu": "sponsors",
#     "label": _("Sponsors"),
#     "items": [
#         {"name": "sponsors", "label": _("Our sponsors"),
#          "url": reverse_lazy("wafer_sponsors")},
#         {"name": "packages", "label": _("Sponsorship packages"),
#          "url": reverse_lazy("wafer_sponsorship_packages")},
#     ]},
#    {"label": _("Talks"),
#     "url": reverse_lazy("wafer_users_talks")},

WAFER_DYNAMIC_MENUS = (
    'wafer.pages.models.page_menus',
    'wafer.sponsors.models.sponsor_menu',
)

# Enabled SSO mechanims:
WAFER_SSO = (
    # 'github',
    # 'debian',
)

# Log in with GitHub:
# WAFER_GITHUB_CLIENT_ID = 'register on github'
# WAFER_GITHUB_CLIENT_SECRET = 'to get these secrets'

# Log in with Debian SSO:
# Requires some Apache config:
# SSLCACertificateFile /srv/sso.debian.org/etc/debsso.crt
# SSLCARevocationCheck chain
# SSLCARevocationFile /srv/sso.debian.org/etc/debsso.crl
# <Location /accounts/debian-login/>
#     SSLOptions +StdEnvVars
#     SSLVerifyClient optional
# </Location>
# WAFER_DEBIAN_NM_API_KEY = 'obtain one from https://nm.debian.org/apikeys/'

# Set this to true to disable the login button on the navigation toolbar
WAFER_HIDE_LOGIN = False

# Talk submissions open
# Set this to False to disable talk submissions
WAFER_TALKS_OPEN = True

# The form used for talk submission
WAFER_TALK_FORM = 'wafer.talks.forms.TalkForm'

# Ask speakers for video release, and an email address of a reviewer
WAFER_VIDEO = True
WAFER_VIDEO_REVIEWER = True
WAFER_VIDEO_LICENSE = 'CC BY-SA 4.0'
WAFER_VIDEO_LICENSE_URL = 'https://creativecommons.org/licenses/by-sa/4.0/'

# Range of scores for talk reviews (inclusive)
WAFER_TALK_REVIEW_SCORES = (-2, 2)

# Set this to False to disable registration
WAFER_REGISTRATION_OPEN = True

# WAFER_REGISTRATION_MODE can be 'ticket' for Quicket tickets, or 'custom' if
# you implement your own registration system. If 'custom', then you *must*
# define a URL named 'register' in your application so we can link to it.
WAFER_REGISTRATION_MODE = 'ticket'
# WAFER_USER_IS_REGISTERED should return a boolean, when passed a Django user.
WAFER_USER_IS_REGISTERED = 'wafer.tickets.models.user_is_registered'

# Allow registered and anonymous users to see registered users
WAFER_PUBLIC_ATTENDEE_LIST = True

# Ticket registration with Quicket
# WAFER_TICKET_SECRET = "i'm a shared secret"

# django-bakery -- disk-based renderer
BUILD_DIR = os.path.join(project_root, 'static_mirror')

MARKITUP_FILTER = ('markdown.markdown', {'safe_mode': True})
JQUERY_URL = 'vendor/jquery/dist/jquery.min.js'
SELECT2_USE_BUNDLED_JQUERY = False

REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
    'PAGE_SIZE': 50
}

BAKERY_VIEWS = (
    'wafer.pages.views.ShowPage',
    'wafer.schedule.views.VenueView',
    'wafer.schedule.views.ScheduleView',
    'wafer.schedule.views.ScheduleXmlView',
    'wafer.schedule.views.ICalView',
    'wafer.sponsors.views.ShowSponsors',
    'wafer.sponsors.views.ShowPackages',
    'wafer.sponsors.views.SponsorView',
    'wafer.talks.views.TalkView',
    'wafer.talks.views.Speakers',
    'wafer.talks.views.TracksView',
    'wafer.talks.views.TalkTypesView',
    'wafer.talks.views.UsersTalks',
    'wafer.users.views.UsersView',
    'wafer.users.views.ProfileView',
)
<|endoftext|>


<commit_before># Copyright (C) 2016 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>

"""Test api helper.

This module contains an api helper that is used for simulating api calls to our
app. This api helper is used instead of the TestCase.client in cases where we
need to access the API during the class setup stage, when the default flaks
test client is not yet ready.

This api helper also helps with delete and put requests where it fetches the
latest etag needed for such requests.
"""
import logging

import flask

from ggrc import db
from ggrc import builder
from ggrc.app import app
from ggrc.services.common import Resource

# style: should the class name be all capitals?
class Api(object):

  def __init__(self):
    self.tc = app.test_client()
    self.tc.get("/login")
    self.resource = Resource()
    self.headers = {'Content-Type': 'application/json',
                    "X-Requested-By": "gGRC"
                    }
    self.user_headers = {}
    self.person_name = None
    self.person_email = None

  def set_user(self, person=None):
    # Refresh the person instance from the db:
    if person:
      person = person.__class__.query.get(person.id)
      self.user_headers = {
          "X-ggrc-user": self.resource.as_json({
              "name": person.name,
              "email": person.email,
          })
      }
      self.person_name, self.person_email = person.name, person.email
    else:
      self.user_headers = {}
      self.person_name, self.person_email = None, None

    self.tc.get("/logout")
    self.tc.get("/login", headers=self.user_headers)
    db.session.commit()
    db.session.flush()

  def api_link(self, obj, obj_id=None):
    obj_id = "" if obj_id is None else "/" + str(obj_id)
    return "/api/%s%s" % (obj._inflector.table_plural, obj_id)

  def data_to_json(self, response):
    """ add docoded json to response object """
    try:
      response.json = flask.json.loads(response.data)
    except StandardError:
      response.json = None
    return response

  def send_request(self, request, obj, data, headers=None, api_link=None):
    headers = headers or {}
    if api_link is None:
      api_link = self.api_link(obj)

    headers.update(self.headers)
    headers.update(self.user_headers)

    json_data = self.resource.as_json(data)
    logging.info("request json" + json_data)
    response = request(api_link, data=json_data, headers=headers.items())
    if response.status_code == 302:
      self.set_user()
      response = request(api_link, data=json_data, headers=headers.items())
    return self.data_to_json(response)

  def put(self, obj, data):
    response = self.get(obj, obj.id)
    headers = {
        "If-Match": response.headers.get("Etag"),
        "If-Unmodified-Since": response.headers.get("Last-Modified")
    }
    api_link = self.api_link(obj, obj.id)
    return self.send_request(
        self.tc.put, obj, data, headers=headers, api_link=api_link)

  def post(self, obj, data):
    return self.send_request(self.tc.post, obj, data)

  def get(self, obj, id_):
    return self.data_to_json(self.tc.get(self.api_link(obj, id_)))

  def get_collection(self, obj, ids):
    return self.data_to_json(self.tc.get(
        "{}?ids={}".format(self.api_link(obj), ids)))

  def get_query(self, obj, query):
    return self.data_to_json(self.tc.get(
        "{}?{}".format(self.api_link(obj), query)))

  def modify_object(self, obj, data=None):
    """Make a put call for a given object.

    Args:
      obj (db.Model): Object that should be modified, and it can also contain
        the changes that should be made.
      data (dict): dictionary containing the changed values for the object.
        This is not mandatory if the object itself has the given changes.

    Returns:
      response, db.Model: The put response from the server and the modified
        object if the put request was successful.
    """
    obj_dict = builder.json.publish(obj)
    builder.json.publish_representation(obj_dict)
    obj_dict.update(data)
    data = {obj._inflector.table_singular: obj_dict}
    return self.put(obj, data)

  def delete(self, obj):
    """Delete api call helper.

    This function helps creating delete calls for a specific object by fetching
    the data and setting the appropriate etag needed for the delete api calls.

    Args:
      obj (sqlalchemy model): object that we wish to delete.

    Returns:
      Server response.
    """
    response = self.get(obj, obj.id)
    headers = {
        "If-Match": response.headers.get("Etag"),
        "If-Unmodified-Since": response.headers.get("Last-Modified")
    }
    headers.update(self.headers)
    api_link = self.api_link(obj, obj.id)
    return self.tc.delete(api_link, headers=headers)

  def search(self, types, q="", counts=False, relevant_objects=None):
    query = '/search?q={}&types={}&counts_only={}'.format(q, types, counts)
    if relevant_objects is not None:
      query += '&relevant_objects=' + relevant_objects
    return (self.tc.get(query), self.headers)
<commit_msg>Fix Api.modify_object data parameter<commit_after># Copyright (C) 2016 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>

"""Test api helper.

This module contains an api helper that is used for simulating api calls to our
app. This api helper is used instead of the TestCase.client in cases where we
need to access the API during the class setup stage, when the default flaks
test client is not yet ready.

This api helper also helps with delete and put requests where it fetches the
latest etag needed for such requests.
"""
import logging

import flask

from ggrc import db
from ggrc import builder
from ggrc.app import app
from ggrc.services.common import Resource

# style: should the class name be all capitals?
class Api(object):

  def __init__(self):
    self.tc = app.test_client()
    self.tc.get("/login")
    self.resource = Resource()
    self.headers = {'Content-Type': 'application/json',
                    "X-Requested-By": "gGRC"
                    }
    self.user_headers = {}
    self.person_name = None
    self.person_email = None

  def set_user(self, person=None):
    # Refresh the person instance from the db:
    if person:
      person = person.__class__.query.get(person.id)
      self.user_headers = {
          "X-ggrc-user": self.resource.as_json({
              "name": person.name,
              "email": person.email,
          })
      }
      self.person_name, self.person_email = person.name, person.email
    else:
      self.user_headers = {}
      self.person_name, self.person_email = None, None

    self.tc.get("/logout")
    self.tc.get("/login", headers=self.user_headers)
    db.session.commit()
    db.session.flush()

  def api_link(self, obj, obj_id=None):
    obj_id = "" if obj_id is None else "/" + str(obj_id)
    return "/api/%s%s" % (obj._inflector.table_plural, obj_id)

  def data_to_json(self, response):
    """ add docoded json to response object """
    try:
      response.json = flask.json.loads(response.data)
    except StandardError:
      response.json = None
    return response

  def send_request(self, request, obj, data, headers=None, api_link=None):
    headers = headers or {}
    if api_link is None:
      api_link = self.api_link(obj)

    headers.update(self.headers)
    headers.update(self.user_headers)

    json_data = self.resource.as_json(data)
    logging.info("request json" + json_data)
    response = request(api_link, data=json_data, headers=headers.items())
    if response.status_code == 302:
      self.set_user()
      response = request(api_link, data=json_data, headers=headers.items())
    return self.data_to_json(response)

  def put(self, obj, data):
    response = self.get(obj, obj.id)
    headers = {
        "If-Match": response.headers.get("Etag"),
        "If-Unmodified-Since": response.headers.get("Last-Modified")
    }
    api_link = self.api_link(obj, obj.id)
    return self.send_request(
        self.tc.put, obj, data, headers=headers, api_link=api_link)

  def post(self, obj, data):
    return self.send_request(self.tc.post, obj, data)

  def get(self, obj, id_):
    return self.data_to_json(self.tc.get(self.api_link(obj, id_)))

  def get_collection(self, obj, ids):
    return self.data_to_json(self.tc.get(
        "{}?ids={}".format(self.api_link(obj), ids)))

  def get_query(self, obj, query):
    return self.data_to_json(self.tc.get(
        "{}?{}".format(self.api_link(obj), query)))

  def modify_object(self, obj, data=None):
    """Make a put call for a given object.

    Args:
      obj (db.Model): Object that should be modified, and it can also contain
        the changes that should be made.
      data (dict): dictionary containing the changed values for the object.
        This is not mandatory if the object itself has the given changes.

    Returns:
      response, db.Model: The put response from the server and the modified
        object if the put request was successful.
    """
    if data is None:
      data = {}
    obj_dict = builder.json.publish(obj)
    builder.json.publish_representation(obj_dict)
    obj_dict.update(data)
    data = {obj._inflector.table_singular: obj_dict}
    return self.put(obj, data)

  def delete(self, obj):
    """Delete api call helper.

    This function helps creating delete calls for a specific object by fetching
    the data and setting the appropriate etag needed for the delete api calls.

    Args:
      obj (sqlalchemy model): object that we wish to delete.

    Returns:
      Server response.
    """
    response = self.get(obj, obj.id)
    headers = {
        "If-Match": response.headers.get("Etag"),
        "If-Unmodified-Since": response.headers.get("Last-Modified")
    }
    headers.update(self.headers)
    api_link = self.api_link(obj, obj.id)
    return self.tc.delete(api_link, headers=headers)

  def search(self, types, q="", counts=False, relevant_objects=None):
    query = '/search?q={}&types={}&counts_only={}'.format(q, types, counts)
    if relevant_objects is not None:
      query += '&relevant_objects=' + relevant_objects
    return (self.tc.get(query), self.headers)
<|endoftext|>


<commit_before>"""
Copyright (c) 2014 Marshall Farrier
license http://opensource.org/licenses/MIT

@author: Marshall Farrier
@contact: marshalldfarrier@gmail.com
@since: 2014-11-07
@summary: Chart financial data

Since detailed charts are readily available online, these charts
serve 2 purposes:
1. Verify data integrity
2. Visualize events as defined by custom algorithms.
The functions in this submodule wrap pandas and matplotlib
for ease of use in serving these 2 purposes.

Resources:
http://matplotlib.org/api/finance_api.html
http://stackoverflow.com/questions/22027415/
http://sentdex.com/sentiment-analysisbig-data-and-python-tutorials-algorithmic-trading/how-to-chart-stocks-and-forex-doing-your-own-financial-charting/
http://stackoverflow.com/questions/19580116/plotting-candlestick-data-from-a-dataframe-in-python
http://sentdex.com/sentiment-analysisbig-data-and-python-tutorials-algorithmic-trading/how-to-chart-stocks-and-forex-doing-your-own-financial-charting/
http://pandas.pydata.org/pandas-docs/version/0.15.0/visualization.html
"""

import datetime as dt

import matplotlib.dates as mdates
import matplotlib.finance as fplt
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker

def candlestick(df, **kwargs):
    """
    candlestick(df, title='GE', fname='foo.png', events=evdf, eventcolors=['r', 'g'],
            bollinger=bolldf, sma=smadf)

    Show and optionally save candlestick chart of a DataFrame as retrieved using data.get().

    Parameters
    ---
    df : DataFrame containing columns 'Open', 'High', 'Low', 'Close' and 'Volume'
        source data
    title : str, optional
        title to be used for the chart
    fname : str, optional
        If provided, the chart will be saved to a file named `fname`. `fname`
        should also include the extension '.png' or '.pdf'
    events : DataFrame, optional
        must have the same index as df.
        Up to 4 columns of events will be mapped. The order of the columns will
        determine the marker to be assigned to the event. The color order is:
        ['g^', 'ro', 'bs', 'k^']
        Non-events in this DataFrame should have a value like np.NAN,
        which matplotlib will not plot.
        The events DataFrame should use dtype='float'. If not, a bug in numpy (or matplotlib)
        can lead to a TypeError:
        http://matplotlib.1069221.n5.nabble.com/type-error-with-python-3-2-and-version-1-1-1-of-matplotlib-numpy-error-td38784.html
    bollinger : DataFrame, optional
        if present Bollinger bands will be overlaid
        must have same index as df
        must contain columns 'Upper' and 'Lower'
    sma : DataFrame, optional
        if present, first data column will be overlaid as simple moving average
        must have same index as df
    """
    _make_chart(df, 'candlestick', **kwargs)

def adj_close(df, **kwargs):
    """
    adj_close(df, title='GE', fname='foo.png', events=evdf, eventcolors=['r', 'g'],
            bollinger=bolldf, sma=smadf)

    Show and optionally save adj_close chart of a DataFrame as retrieved using data.get().

    Parameters
    ---
    df : DataFrame containing columns 'Open', 'High', 'Low', 'Close' and 'Volume'
        source data
    title : str, optional
        title to be used for the chart
    fname : str, optional
        If provided, the chart will be saved to a file named `fname`. `fname`
        should also include the extension '.png' or '.pdf'
    events : DataFrame, optional
        must have the same index as df.
        Up to 4 columns of events will be mapped. The order of the columns will
        determine the marker to be assigned to the event. The color order is:
        ['g^', 'ro', 'bs', 'k^']
        Non-events in this DataFrame should have a value like np.NAN,
        which matplotlib will not plot.
        The events DataFrame should use dtype='float'. If not, a bug in numpy (or matplotlib)
        can lead to a TypeError:
        http://matplotlib.1069221.n5.nabble.com/type-error-with-python-3-2-and-version-1-1-1-of-matplotlib-numpy-error-td38784.html
    bollinger : DataFrame, optional
        if present Bollinger bands will be overlaid
        must have same index as df
        must contain columns 'Upper' and 'Lower'
    sma : DataFrame, optional
        if present, first data column will be overlaid as simple moving average
        must have same index as df
    """
    _make_chart(df, 'adj_close', **kwargs)

def _make_chart(df, chart_type, **kwargs):
    fig = plt.figure()
    ax1 = plt.subplot2grid((5, 4), (0, 0), rowspan=4, colspan=4)
    ax1.grid(True)
    plt.ylabel('Price')
    plt.setp(plt.gca().get_xticklabels(), visible=False)
    if chart_type == 'candlestick':
        _candlestick_ax(df, ax1)
    elif chart_type == 'adj_close':
        _adj_close_ax(df, ax1)
    if 'sma' in kwargs:
        _plot_sma(kwargs['sma'])
    if 'bollinger' in kwargs:
        _plot_bollinger(kwargs['bollinger'])
    if 'events' in kwargs:
        _plot_events(kwargs['events'])

    ax2 = plt.subplot2grid((5, 4), (4, 0), sharex=ax1, rowspan=1, colspan=4)
    ax2.bar(df.index, df.loc[:, 'Volume'])
    ax2.xaxis.set_major_locator(mticker.MaxNLocator(12))
    ax2.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
    ax2.xaxis.set_minor_locator(mdates.DayLocator())
    ax2.yaxis.set_ticklabels([])
    ax2.grid(True)
    plt.ylabel('Volume')
    plt.xlabel('Date')
    plt.setp(plt.gca().get_xticklabels(), rotation=45, horizontalalignment='right')
    plt.subplots_adjust(left=.09, bottom=.18, right=.94, top=0.94, wspace=.20, hspace=0)
    if 'title' in kwargs:
        plt.suptitle(kwargs['title'])
    if 'fname' in kwargs:
        plt.savefig(kwargs['fname'], bbox_inches='tight')
    plt.show()
    plt.close()

def _candlestick_ax(df, ax):
    """
    # Alternatively: (but hard to get dates set up properly)
    plt.xticks(range(len(df.index)), df.index, rotation=45)
    fplt.candlestick2_ohlc(ax, df.loc[:, 'Open'].values, df.loc[:, 'High'].values,
            df.loc[:, 'Low'].values, df.loc[:, 'Close'].values, width=0.2)
    """
    quotes = df.reset_index()
    quotes.loc[:, 'Date'] = mdates.date2num(quotes.loc[:, 'Date'].astype(dt.date))
    fplt.candlestick_ohlc(ax, quotes.values)

def _adj_close_ax(df, ax):
    ax.plot(df.index, df.loc[:, 'Adj Close'])

def _plot_bollinger(bolldf):
    plt.fill_between(bolldf.index, bolldf.loc[:, 'Upper'].values,
            bolldf.loc[:, 'Lower'].values, facecolor='gray', alpha=0.5)

def _plot_sma(smadf):
    lines = plt.plot(smadf.index, smadf.iloc[:, 0].values)
    lines[0].set_color('g')

def _plot_events(events):
    colors = ['gd', 'rv', 'b*', 'k^']
    n_events = min({len(events.columns), len(colors)})
    for i in range(n_events):
        plt.plot(events.index, events.iloc[:, i].values, colors[i], alpha=0.5, ms=12.0)
<commit_msg>Additional reference<commit_after>"""
Copyright (c) 2014 Marshall Farrier
license http://opensource.org/licenses/MIT

@author: Marshall Farrier
@contact: marshalldfarrier@gmail.com
@since: 2014-11-07
@summary: Chart financial data

Since detailed charts are readily available online, these charts
serve 2 purposes:
1. Verify data integrity
2. Visualize events as defined by custom algorithms.
The functions in this submodule wrap pandas and matplotlib
for ease of use in serving these 2 purposes.

Resources:
http://matplotlib.org/api/pyplot_api.html
http://matplotlib.org/api/finance_api.html
http://stackoverflow.com/questions/22027415/
http://sentdex.com/sentiment-analysisbig-data-and-python-tutorials-algorithmic-trading/how-to-chart-stocks-and-forex-doing-your-own-financial-charting/
http://stackoverflow.com/questions/19580116/plotting-candlestick-data-from-a-dataframe-in-python
http://sentdex.com/sentiment-analysisbig-data-and-python-tutorials-algorithmic-trading/how-to-chart-stocks-and-forex-doing-your-own-financial-charting/
http://pandas.pydata.org/pandas-docs/version/0.15.0/visualization.html
"""

import datetime as dt

import matplotlib.dates as mdates
import matplotlib.finance as fplt
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker

def candlestick(df, **kwargs):
    """
    candlestick(df, title='GE', fname='foo.png', events=evdf, eventcolors=['r', 'g'],
            bollinger=bolldf, sma=smadf)

    Show and optionally save candlestick chart of a DataFrame as retrieved using data.get().

    Parameters
    ---
    df : DataFrame containing columns 'Open', 'High', 'Low', 'Close' and 'Volume'
        source data
    title : str, optional
        title to be used for the chart
    fname : str, optional
        If provided, the chart will be saved to a file named `fname`. `fname`
        should also include the extension '.png' or '.pdf'
    events : DataFrame, optional
        must have the same index as df.
        Up to 4 columns of events will be mapped. The order of the columns will
        determine the marker to be assigned to the event. The color order is:
        ['g^', 'ro', 'bs', 'k^']
        Non-events in this DataFrame should have a value like np.NAN,
        which matplotlib will not plot.
        The events DataFrame should use dtype='float'. If not, a bug in numpy (or matplotlib)
        can lead to a TypeError:
        http://matplotlib.1069221.n5.nabble.com/type-error-with-python-3-2-and-version-1-1-1-of-matplotlib-numpy-error-td38784.html
    bollinger : DataFrame, optional
        if present Bollinger bands will be overlaid
        must have same index as df
        must contain columns 'Upper' and 'Lower'
    sma : DataFrame, optional
        if present, first data column will be overlaid as simple moving average
        must have same index as df
    """
    _make_chart(df, 'candlestick', **kwargs)

def adj_close(df, **kwargs):
    """
    adj_close(df, title='GE', fname='foo.png', events=evdf, eventcolors=['r', 'g'],
            bollinger=bolldf, sma=smadf)

    Show and optionally save adj_close chart of a DataFrame as retrieved using data.get().

    Parameters
    ---
    df : DataFrame containing columns 'Open', 'High', 'Low', 'Close' and 'Volume'
        source data
    title : str, optional
        title to be used for the chart
    fname : str, optional
        If provided, the chart will be saved to a file named `fname`. `fname`
        should also include the extension '.png' or '.pdf'
    events : DataFrame, optional
        must have the same index as df.
        Up to 4 columns of events will be mapped. The order of the columns will
        determine the marker to be assigned to the event. The color order is:
        ['g^', 'ro', 'bs', 'k^']
        Non-events in this DataFrame should have a value like np.NAN,
        which matplotlib will not plot.
        The events DataFrame should use dtype='float'. If not, a bug in numpy (or matplotlib)
        can lead to a TypeError:
        http://matplotlib.1069221.n5.nabble.com/type-error-with-python-3-2-and-version-1-1-1-of-matplotlib-numpy-error-td38784.html
    bollinger : DataFrame, optional
        if present Bollinger bands will be overlaid
        must have same index as df
        must contain columns 'Upper' and 'Lower'
    sma : DataFrame, optional
        if present, first data column will be overlaid as simple moving average
        must have same index as df
    """
    _make_chart(df, 'adj_close', **kwargs)

def _make_chart(df, chart_type, **kwargs):
    fig = plt.figure()
    ax1 = plt.subplot2grid((5, 4), (0, 0), rowspan=4, colspan=4)
    ax1.grid(True)
    plt.ylabel('Price')
    plt.setp(plt.gca().get_xticklabels(), visible=False)
    if chart_type == 'candlestick':
        _candlestick_ax(df, ax1)
    elif chart_type == 'adj_close':
        _adj_close_ax(df, ax1)
    if 'sma' in kwargs:
        _plot_sma(kwargs['sma'])
    if 'bollinger' in kwargs:
        _plot_bollinger(kwargs['bollinger'])
    if 'events' in kwargs:
        _plot_events(kwargs['events'])

    ax2 = plt.subplot2grid((5, 4), (4, 0), sharex=ax1, rowspan=1, colspan=4)
    ax2.bar(df.index, df.loc[:, 'Volume'])
    ax2.xaxis.set_major_locator(mticker.MaxNLocator(12))
    ax2.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
    ax2.xaxis.set_minor_locator(mdates.DayLocator())
    ax2.yaxis.set_ticklabels([])
    ax2.grid(True)
    plt.ylabel('Volume')
    plt.xlabel('Date')
    plt.setp(plt.gca().get_xticklabels(), rotation=45, horizontalalignment='right')
    plt.subplots_adjust(left=.09, bottom=.18, right=.94, top=0.94, wspace=.20, hspace=0)
    if 'title' in kwargs:
        plt.suptitle(kwargs['title'])
    if 'fname' in kwargs:
        plt.savefig(kwargs['fname'], bbox_inches='tight')
    plt.show()
    plt.close()

def _candlestick_ax(df, ax):
    """
    # Alternatively: (but hard to get dates set up properly)
    plt.xticks(range(len(df.index)), df.index, rotation=45)
    fplt.candlestick2_ohlc(ax, df.loc[:, 'Open'].values, df.loc[:, 'High'].values,
            df.loc[:, 'Low'].values, df.loc[:, 'Close'].values, width=0.2)
    """
    quotes = df.reset_index()
    quotes.loc[:, 'Date'] = mdates.date2num(quotes.loc[:, 'Date'].astype(dt.date))
    fplt.candlestick_ohlc(ax, quotes.values)

def _adj_close_ax(df, ax):
    ax.plot(df.index, df.loc[:, 'Adj Close'])

def _plot_bollinger(bolldf):
    plt.fill_between(bolldf.index, bolldf.loc[:, 'Upper'].values,
            bolldf.loc[:, 'Lower'].values, facecolor='gray', alpha=0.5)

def _plot_sma(smadf):
    lines = plt.plot(smadf.index, smadf.iloc[:, 0].values)
    lines[0].set_color('g')

def _plot_events(events):
    colors = ['gd', 'rv', 'b*', 'k^']
    n_events = min({len(events.columns), len(colors)})
    for i in range(n_events):
        plt.plot(events.index, events.iloc[:, i].values, colors[i], alpha=0.5, ms=12.0)
<|endoftext|>


<commit_before># Copyright 2020 DeepMind Technologies Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""IMPALA learner class."""

import functools
import itertools
import queue
import threading
from typing import Dict, Tuple
import warnings

import dm_env
import haiku as hk
from examples.impala import agent as agent_lib
from examples.impala import util
import jax
from jax.experimental import optimizers
import jax.numpy as jnp
import numpy as np
import optax
import rlax

# The IMPALA paper sums losses, rather than taking the mean.
# We wrap rlax to do so as well.
def policy_gradient_loss(logits, *args):
  """rlax.policy_gradient_loss, but with sum(loss) and [T, B, ...] inputs."""
  mean_per_batch = jax.vmap(rlax.policy_gradient_loss, in_axes=1)(logits, *args)
  total_loss_per_batch = mean_per_batch * logits.shape[0]
  return jnp.sum(total_loss_per_batch)

def entropy_loss(logits, *args):
  """rlax.entropy_loss, but with sum(loss) and [T, B, ...] inputs."""
  mean_per_batch = jax.vmap(rlax.entropy_loss, in_axes=1)(logits, *args)
  total_loss_per_batch = mean_per_batch * logits.shape[0]
  return jnp.sum(total_loss_per_batch)

class Learner:
  """Manages state and performs updates for IMPALA learner."""

  def __init__(
      self,
      agent: agent_lib.Agent,
      rng_key,
      opt: optax.GradientTransformation,
      batch_size: int,
      discount_factor: float,
      frames_per_iter: int,
      max_abs_reward: float = 0,
      logger=None,
  ):
    if jax.device_count() > 1:
      warnings.warn('Note: the impala example will only take advantage of a '
                    'single accelerator.')

    self._agent = agent
    self._opt = opt
    self._batch_size = batch_size
    self._discount_factor = discount_factor
    self._frames_per_iter = frames_per_iter
    self._max_abs_reward = max_abs_reward

    # Data pipeline objects.
    self._done = False
    self._host_q = queue.Queue(maxsize=self._batch_size)
    self._device_q = queue.Queue(maxsize=1)

    # Prepare the parameters to be served to actors.
    params = agent.initial_params(rng_key)
    self._params_for_actor = (0, jax.device_get(params))

    # Set up logging.
    if logger is None:
      logger = util.NullLogger()
    self._logger = logger

  def _loss(
      self,
      theta: hk.Params,
      trajectories: util.Transition,
  ) -> Tuple[jnp.ndarray, Dict[str, jnp.ndarray]]:
    """Compute vtrace-based actor-critic loss."""
    initial_state = jax.tree_map(lambda t: t[0], trajectories.agent_state)
    learner_outputs = self._agent.unroll(theta, trajectories.timestep,
                                         initial_state)
    v_t = learner_outputs.values[1:]
    # Remove bootstrap timestep from non-timesteps.
    _, actor_out, _ = jax.tree_map(lambda t: t[:-1], trajectories)
    learner_outputs = jax.tree_map(lambda t: t[:-1], learner_outputs)
    v_tm1 = learner_outputs.values

    # Get the discount, reward, step_type from the *next* timestep.
    timestep = jax.tree_map(lambda t: t[1:], trajectories.timestep)
    discounts = timestep.discount * self._discount_factor
    rewards = timestep.reward
    if self._max_abs_reward > 0:
      rewards = jnp.clip(rewards, -self._max_abs_reward, self._max_abs_reward)

    # The step is uninteresting if we transitioned LAST -> FIRST.
    # timestep corresponds to the *next* time step, so we filter for FIRST.
    mask = jnp.not_equal(timestep.step_type, int(dm_env.StepType.FIRST))
    mask = mask.astype(jnp.float32)

    rhos = rlax.categorical_importance_sampling_ratios(
        learner_outputs.policy_logits, actor_out.policy_logits,
        actor_out.action)
    # vmap vtrace_td_error_and_advantage to take/return [T, B, ...].
    vtrace_td_error_and_advantage = jax.vmap(
        rlax.vtrace_td_error_and_advantage, in_axes=1, out_axes=1)

    vtrace_returns = vtrace_td_error_and_advantage(
        v_tm1, v_t, rewards, discounts, rhos)
    pg_advs = vtrace_returns.pg_advantage
    pg_loss = policy_gradient_loss(learner_outputs.policy_logits,
                                   actor_out.action, pg_advs, mask)

    baseline_loss = 0.5 * jnp.sum(jnp.square(vtrace_returns.errors) * mask)
    ent_loss = entropy_loss(learner_outputs.policy_logits, mask)

    total_loss = pg_loss
    total_loss += 0.5 * baseline_loss
    total_loss += 0.01 * ent_loss

    logs = {}
    logs['PG_loss'] = pg_loss
    logs['baseline_loss'] = baseline_loss
    logs['entropy_loss'] = ent_loss
    logs['total_loss'] = total_loss
    return total_loss, logs

  @functools.partial(jax.jit, static_argnums=0)
  def update(self, params, opt_state, batch: util.Transition):
    """The actual update function."""
    (_, logs), grads = jax.value_and_grad(
        self._loss, has_aux=True)(params, batch)

    grad_norm_unclipped = optimizers.l2_norm(grads)
    updates, updated_opt_state = self._opt.update(grads, opt_state)
    params = optax.apply_updates(params, updates)
    weight_norm = optimizers.l2_norm(params)
    logs.update({
        'grad_norm_unclipped': grad_norm_unclipped,
        'weight_norm': weight_norm,
    })
    return params, updated_opt_state, logs

  def enqueue_traj(self, traj: util.Transition):
    """Enqueue trajectory."""
    self._host_q.put(traj)

  def params_for_actor(self) -> Tuple[int, hk.Params]:
    return self._params_for_actor

  def host_to_device_worker(self):
    """Elementary data pipeline."""
    batch = []
    while not self._done:
      # Try to get a batch. Skip the iteration if we couldn't.
      try:
        for _ in range(len(batch), self._batch_size):
          # As long as possible while keeping learner_test time reasonable.
          batch.append(self._host_q.get(timeout=10))
      except queue.Empty:
        continue

      assert len(batch) == self._batch_size
      # Prepare for consumption, then put batch onto device.
      stacked_batch = jax.tree_multimap(lambda *xs: np.stack(xs, axis=1),
                                        *batch)
      self._device_q.put(jax.device_put(stacked_batch))

      # Clean out the built-up batch.
      batch = []

  def run(self, max_iterations: int = -1):
    """Runs the learner for max_iterations updates."""
    # Start host-to-device transfer worker.
    transfer_thread = threading.Thread(target=self.host_to_device_worker)
    transfer_thread.start()

    (num_frames, params) = self._params_for_actor
    opt_state = self._opt.init(params)

    steps = range(max_iterations) if max_iterations != -1 else itertools.count()
    for _ in steps:
      batch = self._device_q.get()
      params, opt_state, logs = self.update(params, opt_state, batch)
      num_frames += self._frames_per_iter

      # Collect parameters to distribute to downstream actors.
      self._params_for_actor = (num_frames, jax.device_get(params))

      # Collect and write logs out.
      logs = jax.device_get(logs)
      logs.update({
          'num_frames': num_frames,
      })
      self._logger.write(logs)

    # Shut down.
    self._done = True
    self._logger.close()
    transfer_thread.join()
<commit_msg>[JAX] move example libraries from `jax.experimental` into `jax.example_libraries`<commit_after># Copyright 2020 DeepMind Technologies Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""IMPALA learner class."""

import functools
import itertools
import queue
import threading
from typing import Dict, Tuple
import warnings

import dm_env
import haiku as hk
from examples.impala import agent as agent_lib
from examples.impala import util
import jax
from jax.example_libraries import optimizers
import jax.numpy as jnp
import numpy as np
import optax
import rlax

# The IMPALA paper sums losses, rather than taking the mean.
# We wrap rlax to do so as well.
def policy_gradient_loss(logits, *args):
  """rlax.policy_gradient_loss, but with sum(loss) and [T, B, ...] inputs."""
  mean_per_batch = jax.vmap(rlax.policy_gradient_loss, in_axes=1)(logits, *args)
  total_loss_per_batch = mean_per_batch * logits.shape[0]
  return jnp.sum(total_loss_per_batch)

def entropy_loss(logits, *args):
  """rlax.entropy_loss, but with sum(loss) and [T, B, ...] inputs."""
  mean_per_batch = jax.vmap(rlax.entropy_loss, in_axes=1)(logits, *args)
  total_loss_per_batch = mean_per_batch * logits.shape[0]
  return jnp.sum(total_loss_per_batch)

class Learner:
  """Manages state and performs updates for IMPALA learner."""

  def __init__(
      self,
      agent: agent_lib.Agent,
      rng_key,
      opt: optax.GradientTransformation,
      batch_size: int,
      discount_factor: float,
      frames_per_iter: int,
      max_abs_reward: float = 0,
      logger=None,
  ):
    if jax.device_count() > 1:
      warnings.warn('Note: the impala example will only take advantage of a '
                    'single accelerator.')

    self._agent = agent
    self._opt = opt
    self._batch_size = batch_size
    self._discount_factor = discount_factor
    self._frames_per_iter = frames_per_iter
    self._max_abs_reward = max_abs_reward

    # Data pipeline objects.
    self._done = False
    self._host_q = queue.Queue(maxsize=self._batch_size)
    self._device_q = queue.Queue(maxsize=1)

    # Prepare the parameters to be served to actors.
    params = agent.initial_params(rng_key)
    self._params_for_actor = (0, jax.device_get(params))

    # Set up logging.
    if logger is None:
      logger = util.NullLogger()
    self._logger = logger

  def _loss(
      self,
      theta: hk.Params,
      trajectories: util.Transition,
  ) -> Tuple[jnp.ndarray, Dict[str, jnp.ndarray]]:
    """Compute vtrace-based actor-critic loss."""
    initial_state = jax.tree_map(lambda t: t[0], trajectories.agent_state)
    learner_outputs = self._agent.unroll(theta, trajectories.timestep,
                                         initial_state)
    v_t = learner_outputs.values[1:]
    # Remove bootstrap timestep from non-timesteps.
    _, actor_out, _ = jax.tree_map(lambda t: t[:-1], trajectories)
    learner_outputs = jax.tree_map(lambda t: t[:-1], learner_outputs)
    v_tm1 = learner_outputs.values

    # Get the discount, reward, step_type from the *next* timestep.
    timestep = jax.tree_map(lambda t: t[1:], trajectories.timestep)
    discounts = timestep.discount * self._discount_factor
    rewards = timestep.reward
    if self._max_abs_reward > 0:
      rewards = jnp.clip(rewards, -self._max_abs_reward, self._max_abs_reward)

    # The step is uninteresting if we transitioned LAST -> FIRST.
    # timestep corresponds to the *next* time step, so we filter for FIRST.
    mask = jnp.not_equal(timestep.step_type, int(dm_env.StepType.FIRST))
    mask = mask.astype(jnp.float32)

    rhos = rlax.categorical_importance_sampling_ratios(
        learner_outputs.policy_logits, actor_out.policy_logits,
        actor_out.action)
    # vmap vtrace_td_error_and_advantage to take/return [T, B, ...].
    vtrace_td_error_and_advantage = jax.vmap(
        rlax.vtrace_td_error_and_advantage, in_axes=1, out_axes=1)

    vtrace_returns = vtrace_td_error_and_advantage(
        v_tm1, v_t, rewards, discounts, rhos)
    pg_advs = vtrace_returns.pg_advantage
    pg_loss = policy_gradient_loss(learner_outputs.policy_logits,
                                   actor_out.action, pg_advs, mask)

    baseline_loss = 0.5 * jnp.sum(jnp.square(vtrace_returns.errors) * mask)
    ent_loss = entropy_loss(learner_outputs.policy_logits, mask)

    total_loss = pg_loss
    total_loss += 0.5 * baseline_loss
    total_loss += 0.01 * ent_loss

    logs = {}
    logs['PG_loss'] = pg_loss
    logs['baseline_loss'] = baseline_loss
    logs['entropy_loss'] = ent_loss
    logs['total_loss'] = total_loss
    return total_loss, logs

  @functools.partial(jax.jit, static_argnums=0)
  def update(self, params, opt_state, batch: util.Transition):
    """The actual update function."""
    (_, logs), grads = jax.value_and_grad(
        self._loss, has_aux=True)(params, batch)

    grad_norm_unclipped = optimizers.l2_norm(grads)
    updates, updated_opt_state = self._opt.update(grads, opt_state)
    params = optax.apply_updates(params, updates)
    weight_norm = optimizers.l2_norm(params)
    logs.update({
        'grad_norm_unclipped': grad_norm_unclipped,
        'weight_norm': weight_norm,
    })
    return params, updated_opt_state, logs

  def enqueue_traj(self, traj: util.Transition):
    """Enqueue trajectory."""
    self._host_q.put(traj)

  def params_for_actor(self) -> Tuple[int, hk.Params]:
    return self._params_for_actor

  def host_to_device_worker(self):
    """Elementary data pipeline."""
    batch = []
    while not self._done:
      # Try to get a batch. Skip the iteration if we couldn't.
      try:
        for _ in range(len(batch), self._batch_size):
          # As long as possible while keeping learner_test time reasonable.
          batch.append(self._host_q.get(timeout=10))
      except queue.Empty:
        continue

      assert len(batch) == self._batch_size
      # Prepare for consumption, then put batch onto device.
      stacked_batch = jax.tree_multimap(lambda *xs: np.stack(xs, axis=1),
                                        *batch)
      self._device_q.put(jax.device_put(stacked_batch))

      # Clean out the built-up batch.
      batch = []

  def run(self, max_iterations: int = -1):
    """Runs the learner for max_iterations updates."""
    # Start host-to-device transfer worker.
    transfer_thread = threading.Thread(target=self.host_to_device_worker)
    transfer_thread.start()

    (num_frames, params) = self._params_for_actor
    opt_state = self._opt.init(params)

    steps = range(max_iterations) if max_iterations != -1 else itertools.count()
    for _ in steps:
      batch = self._device_q.get()
      params, opt_state, logs = self.update(params, opt_state, batch)
      num_frames += self._frames_per_iter

      # Collect parameters to distribute to downstream actors.
      self._params_for_actor = (num_frames, jax.device_get(params))

      # Collect and write logs out.
      logs = jax.device_get(logs)
      logs.update({
          'num_frames': num_frames,
      })
      self._logger.write(logs)

    # Shut down.
    self._done = True
    self._logger.close()
    transfer_thread.join()
<|endoftext|>