<commit_before>"""
Django settings for tarina project.

Generated by 'django-admin startproject' using Django 1.10.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

import dj_database_url

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'b0^9)u2%79sip$^-3rv0c*e#q45sf0-t_=e!zh!ue(ap0lvgmf'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = ['localhost', 'tarina.herokuapp.com']

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'rest_framework',
    'rest_framework.authtoken',
    'corsheaders',

    'users',
    'stories'
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'tarina.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'tarina.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# CORS Headers configuration

CORS_ORIGIN_ALLOW_ALL = True

CORS_ALLOW_HEADERS = (
    'accept',
    'accept-encoding',
    'authorization',
    'content-type',
    'dnt',
    'origin',
    'user-agent',
    'x-csrftoken',
    'x-requested-with',
)

# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/

PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))

STATIC_ROOT = os.path.join(PROJECT_ROOT, 'staticfiles')
STATIC_URL = '/static/'

STATICFILES_DIRS = (
    os.path.join(PROJECT_ROOT, 'static'),
)

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

# Deployment database configuration
db_from_env = dj_database_url.config(conn_max_age=500)
DATABASES['default'].update(db_from_env)
<commit_msg>Added missing CORS headers middleware<commit_after>"""
Django settings for tarina project.

Generated by 'django-admin startproject' using Django 1.10.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

import dj_database_url

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'b0^9)u2%79sip$^-3rv0c*e#q45sf0-t_=e!zh!ue(ap0lvgmf'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = ['localhost', 'tarina.herokuapp.com']

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'rest_framework',
    'rest_framework.authtoken',
    'corsheaders',

    'users',
    'stories'
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'corsheaders.middleware.CorsMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware'
]

ROOT_URLCONF = 'tarina.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'tarina.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# CORS Headers configuration

CORS_ORIGIN_ALLOW_ALL = True

CORS_ALLOW_HEADERS = (
    'accept',
    'accept-encoding',
    'authorization',
    'content-type',
    'dnt',
    'origin',
    'user-agent',
    'x-csrftoken',
    'x-requested-with',
)

# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/

PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))

STATIC_ROOT = os.path.join(PROJECT_ROOT, 'staticfiles')
STATIC_URL = '/static/'

STATICFILES_DIRS = (
    os.path.join(PROJECT_ROOT, 'static'),
)

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

# Deployment database configuration
db_from_env = dj_database_url.config(conn_max_age=500)
DATABASES['default'].update(db_from_env)
<|endoftext|>


<commit_before><commit_msg>client: tables attributes<commit_after><|endoftext|>


<commit_before>import os
import os.path as op

import numpy as np
import nibabel as nib

from dipy.reconst import csdeconv as csd
from dipy.reconst import mcsd
from dipy.reconst import shm
import dipy.data as dpd
import AFQ.utils.models as ut

# Monkey patch fixed spherical harmonics for conda and fixed solve_qp from
# DIPY dev:
from AFQ._fixes import spherical_harmonics, solve_qp
shm.spherical_harmonics = spherical_harmonics
mcsd.solve_qp = solve_qp

__all__ = ["fit_csd"]

def _model(gtab, data, response=None, sh_order=None, msmt=False):
    """
    Helper function that defines a CSD model.
    """
    if sh_order is None:
        ndata = np.sum(~gtab.b0s_mask)
        # See dipy.reconst.shm.calculate_max_order
        L1 = (-3 + np.sqrt(1 + 8 * ndata)) / 2.0
        sh_order = int(L1)
        if np.mod(sh_order, 2) != 0:
            sh_order = sh_order - 1
        if sh_order > 8:
            sh_order = 8

    if msmt:
        my_model = mcsd.MultiShellDeconvModel
        if response is None:
            mask_wm, mask_gm, mask_csf =\
                mcsd.mask_for_response_msmt(gtab, data)
            response_wm, response_gm, response_csf =\
                mcsd.response_from_mask_msmt(gtab, data,
                                             mask_wm, mask_gm, mask_csf)
            response = np.array([response_wm, response_gm, response_csf])
    else:
        my_model = csd.ConstrainedSphericalDeconvModel
        if response is None:
            response, _ = csd.auto_response_ssst(gtab, data, roi_radii=10,
                                                 fa_thr=0.7)

    csdmodel = my_model(gtab, response, sh_order=sh_order)
    return csdmodel

def _fit(gtab, data, mask, response=None, sh_order=None, lambda_=1, tau=0.1,
         msmt=False):
    """
    Helper function that does the core of fitting a model to data.
    """
    return _model(gtab, data, response, sh_order, msmt).fit(data, mask=mask)

def fit_csd(data_files, bval_files, bvec_files, mask=None, response=None,
            b0_threshold=50, sh_order=None, lambda_=1, tau=0.1, out_dir=None,
            msmt=False):
    """
    Fit the CSD model and save file with SH coefficients.

    Parameters
    ----------
    data_files : str or list.
        Files containing DWI data. If this is a str, that's the full path to a
        single file. If it's a list, each entry is a full path.
    bval_files : str or list.
        Equivalent to `data_files`.
    bvec_files : str or list.
        Equivalent to `data_files`.
    mask : ndarray, optional.
        Binary mask, set to True or 1 in voxels to be processed.
        Default: Process all voxels.
    response: tuple, optional.
        The response function to be used by CSD, as a tuple with two elements.
        The first is the eigen-values as an (3,) ndarray and the second is
        the signal value for the response function without diffusion-weighting
        (i.e. S0). If not provided, auto_response will be used to calculate
        these values.
    b0_threshold : float,optional.
      The value of diffusion-weighting under which we consider it to be
      equivalent to 0. Default:50
    sh_order : int, optional.
        default: infer the number of parameters from the number of data
        volumes, but no larger than 8.
    lambda_ : float, optional.
        weight given to the constrained-positivity regularization part of
        the deconvolution equation. Default: 1
    tau : float, optional.
        threshold controlling the amplitude below which the corresponding
        fODF is assumed to be zero.  Ideally, tau should be set to
        zero. However, to improve the stability of the algorithm, tau is
        set to tau*100 % of the mean fODF amplitude (here, 10% by default)
        (see [1]_). Default: 0.1
    out_dir : str, optional
        A full path to a directory to store the maps that get computed.
        Default: file with coefficients gets stored in the same directory as
        the first DWI file in `data_files`.
    msmt : bool, optional
        If False, standard single-shell CSD will be used. Otherwise,

    Returns
    -------
    fname : the full path to the file containing the SH coefficients.
    """
    img, data, gtab, mask = ut.prepare_data(data_files, bval_files, bvec_files,
                                            b0_threshold=b0_threshold)

    csdfit = _fit(gtab, data, mask, response=response, sh_order=sh_order,
                  lambda_=lambda_, tau=tau, msmt=msmt)

    if out_dir is None:
        out_dir = op.join(op.split(data_files)[0], 'dki')

    if not op.exists(out_dir):
        os.makedirs(out_dir)

    aff = img.affine
    fname = op.join(out_dir, 'csd_sh_coeff.nii.gz')
    nib.save(nib.Nifti1Image(csdfit.shm_coeff, aff), fname)
    return fname

def fit_anisotropic_power_map(dwi, gtab, mask=None):
    """
    Fits an anisotropic power map.

    Parameters
    ----------
    dwi : str, ndarray, or nifti1image
        Data to greate map with.

    gtab : GradientTable
        A GradientTable with all the gradient information.

    mask : str or nifti1image, optional
        mask to mask the data with.
        Default: None.

    Returns
    -------
    ndarray containing an anisotropic power map.
    """

    if isinstance(dwi, str):
        dwi = nib.load(dwi)
    if isinstance(dwi, nib.Nifti1Image):
        dwi_data = dwi.get_fdata()
    else:
        dwi_data = dwi

    if isinstance(mask, str):
        mask = nib.load(mask)
    mask = mask.get_fdata()

    model = _model(gtab, dwi_data)
    sphere = dpd.get_sphere('symmetric724')
    peaks = csd.peaks_from_model(
        model=model,
        data=dwi_data,
        sphere=sphere,
        relative_peak_threshold=.5,
        min_separation_angle=25,
        mask=mask)
    ap = shm.anisotropic_power(peaks.shm_coeff)

    return ap
<commit_msg>This function is no longer being used, so we can remove it.<commit_after>import os
import os.path as op

import numpy as np
import nibabel as nib

from dipy.reconst import csdeconv as csd
from dipy.reconst import mcsd
from dipy.reconst import shm
import dipy.data as dpd
import AFQ.utils.models as ut

# Monkey patch fixed spherical harmonics for conda and fixed solve_qp from
# DIPY dev:
from AFQ._fixes import spherical_harmonics, solve_qp
shm.spherical_harmonics = spherical_harmonics
mcsd.solve_qp = solve_qp

__all__ = ["fit_csd"]

def _model(gtab, data, response=None, sh_order=None, msmt=False):
    """
    Helper function that defines a CSD model.
    """
    if sh_order is None:
        ndata = np.sum(~gtab.b0s_mask)
        # See dipy.reconst.shm.calculate_max_order
        L1 = (-3 + np.sqrt(1 + 8 * ndata)) / 2.0
        sh_order = int(L1)
        if np.mod(sh_order, 2) != 0:
            sh_order = sh_order - 1
        if sh_order > 8:
            sh_order = 8

    if msmt:
        my_model = mcsd.MultiShellDeconvModel
        if response is None:
            mask_wm, mask_gm, mask_csf =\
                mcsd.mask_for_response_msmt(gtab, data)
            response_wm, response_gm, response_csf =\
                mcsd.response_from_mask_msmt(gtab, data,
                                             mask_wm, mask_gm, mask_csf)
            response = np.array([response_wm, response_gm, response_csf])
    else:
        my_model = csd.ConstrainedSphericalDeconvModel
        if response is None:
            response, _ = csd.auto_response_ssst(gtab, data, roi_radii=10,
                                                 fa_thr=0.7)

    csdmodel = my_model(gtab, response, sh_order=sh_order)
    return csdmodel

def _fit(gtab, data, mask, response=None, sh_order=None, lambda_=1, tau=0.1,
         msmt=False):
    """
    Helper function that does the core of fitting a model to data.
    """
    return _model(gtab, data, response, sh_order, msmt).fit(data, mask=mask)

def fit_csd(data_files, bval_files, bvec_files, mask=None, response=None,
            b0_threshold=50, sh_order=None, lambda_=1, tau=0.1, out_dir=None,
            msmt=False):
    """
    Fit the CSD model and save file with SH coefficients.

    Parameters
    ----------
    data_files : str or list.
        Files containing DWI data. If this is a str, that's the full path to a
        single file. If it's a list, each entry is a full path.
    bval_files : str or list.
        Equivalent to `data_files`.
    bvec_files : str or list.
        Equivalent to `data_files`.
    mask : ndarray, optional.
        Binary mask, set to True or 1 in voxels to be processed.
        Default: Process all voxels.
    response: tuple, optional.
        The response function to be used by CSD, as a tuple with two elements.
        The first is the eigen-values as an (3,) ndarray and the second is
        the signal value for the response function without diffusion-weighting
        (i.e. S0). If not provided, auto_response will be used to calculate
        these values.
    b0_threshold : float,optional.
      The value of diffusion-weighting under which we consider it to be
      equivalent to 0. Default:50
    sh_order : int, optional.
        default: infer the number of parameters from the number of data
        volumes, but no larger than 8.
    lambda_ : float, optional.
        weight given to the constrained-positivity regularization part of
        the deconvolution equation. Default: 1
    tau : float, optional.
        threshold controlling the amplitude below which the corresponding
        fODF is assumed to be zero.  Ideally, tau should be set to
        zero. However, to improve the stability of the algorithm, tau is
        set to tau*100 % of the mean fODF amplitude (here, 10% by default)
        (see [1]_). Default: 0.1
    out_dir : str, optional
        A full path to a directory to store the maps that get computed.
        Default: file with coefficients gets stored in the same directory as
        the first DWI file in `data_files`.
    msmt : bool, optional
        If False, standard single-shell CSD will be used. Otherwise,

    Returns
    -------
    fname : the full path to the file containing the SH coefficients.
    """
    img, data, gtab, mask = ut.prepare_data(data_files, bval_files, bvec_files,
                                            b0_threshold=b0_threshold)

    csdfit = _fit(gtab, data, mask, response=response, sh_order=sh_order,
                  lambda_=lambda_, tau=tau, msmt=msmt)

    if out_dir is None:
        out_dir = op.join(op.split(data_files)[0], 'dki')

    if not op.exists(out_dir):
        os.makedirs(out_dir)

    aff = img.affine
    fname = op.join(out_dir, 'csd_sh_coeff.nii.gz')
    nib.save(nib.Nifti1Image(csdfit.shm_coeff, aff), fname)
    return fname
<|endoftext|>


<commit_before>import json
import logging
import aiohttp

logging.basicConfig(format='[%(levelname)s] - %(message)s', level=logging.INFO)
config = json.load(open('config.json'))

class Stats:

    def __init__(self, bot):
        self.bot = bot
        self.config = json.load(open('config.json'))

    async def dblpost(self):
        headers = {
            'Authorization': self.config['dbl'],
            'Content-Type': 'application/json'
        }
        data = {
            'server_count': len(self.bot.guilds),
            'shard_count': len(self.bot.shards)
        }
        info = await self.bot.application_info()
        req = await self.bot.session.post(f'https://discordbots.org/api/bots/{info.id}/stats', data=json.dumps(data), headers=headers)
        status = req.status
        if status == 200:
            logging.info('poster[dbl]: done')
        else:
            resp = await req.text()
            logging.error(f'poster[dbl]: oops (code {status})')
            logging.error(f'poster[dbl]: response: {resp}')

    async def dpwpost(self):
        headers = {
                'Authorization': self.config['dbots'],
                'Content-Type': 'application/json'
            }
        data = {
                'server_count': len(self.bot.guilds),
                'shard_count': len(self.bot.shards)
            }
        info = await self.bot.application_info()
        req = await self.bot.session.post(f'https://bots.discord.pw/api/bots/{info.id}/stats', data=json.dumps(data), headers=headers)
        status = req.status
        if status == 200:
            logging.info('poster[dbots]: done')
        else:
            resp = await req.text()
            logging.error(f'poster[dbots]: oops (code {status})')
            logging.error(f'poster[dbots]: response: {resp}')

    async def post(self):
        logging.info('poster: starting post')
        await dblpost()
        await dpwpost()
        logging.info('poster: done')

    async def on_guild_join( self, guild ):
        logging.info('Joined guild "' + str(guild.name) + '" ID: ' + str(guild.id))
        await post()

    async def on_guild_remove( self, guild ):
        logging.info('Left ' + str(guild.name))
        await post()

def setup(bot):
    bot.add_cog(Stats(bot))
<commit_msg>[cogs(stats)] try and fix shit<commit_after>import json
import logging

logging.basicConfig(format='[%(levelname)s] - %(message)s', level=logging.INFO)
config = json.load(open('config.json'))

class Stats:

    def __init__(self, bot):
        self.bot = bot
        self.config = json.load(open('config.json'))
        self.dbl = dblpost
        self.dpw = dpwpost

    async def dblpost(self):
        headers = {
            'Authorization': self.config['dbl'],
            'Content-Type': 'application/json'
        }
        data = {
            'server_count': len(self.bot.guilds),
            'shard_count': len(self.bot.shards)
        }
        info = await self.bot.application_info()
        req = await self.bot.session.post(f'https://discordbots.org/api/bots/{info.id}/stats', data=json.dumps(data), headers=headers)
        status = req.status
        if status == 200:
            logging.info('poster[dbl]: done')
        else:
            resp = await req.text()
            logging.error(f'poster[dbl]: oops (code {status})')
            logging.error(f'poster[dbl]: response: {resp}')

    async def dpwpost(self):
        headers = {
                'Authorization': self.config['dbots'],
                'Content-Type': 'application/json'
            }
        data = {
                'server_count': len(self.bot.guilds),
                'shard_count': len(self.bot.shards)
            }
        info = await self.bot.application_info()
        req = await self.bot.session.post(f'https://bots.discord.pw/api/bots/{info.id}/stats', data=json.dumps(data), headers=headers)
        status = req.status
        if status == 200:
            logging.info('poster[dbots]: done')
        else:
            resp = await req.text()
            logging.error(f'poster[dbots]: oops (code {status})')
            logging.error(f'poster[dbots]: response: {resp}')

    async def post(self):
        logging.info('poster: starting post')
        await self.dbl()
        await self.dpw()
        logging.info('poster: done')

    async def on_guild_join( self, guild ):
        logging.info('Joined guild "' + str(guild.name) + '" ID: ' + str(guild.id))
        await post()

    async def on_guild_remove( self, guild ):
        logging.info('Left ' + str(guild.name))
        await post()

def setup(bot):
    bot.add_cog(Stats(bot))
<|endoftext|>


<commit_before>"""AWS S3 Interface used by tests."""

import boto3
import botocore
from botocore.exceptions import ClientError
import json

from s3configuration import S3Configuration

class S3Interface():
    """Interface to the AWS S3 database."""

    def __init__(self, s3configuration):
        """Create a new interface to the AWS S3.

        Remember the access key, secret access key, region, and deployment
        prefix that will be used later to connect to the AWS S3.
        """
        self.s3_configuration = s3configuration

        # to be set up by the connect() method
        self.s3_resource = None
        self.s3_session = None

    def connect(self):
        """Connect to the AWS S3 database."""
        # we are already connected -> let's use this connection
        if self.s3_resource is not None:
            return

        # create the session used to communicate with the S3 database
        # and check if the operation was successful
        self.s3_session = boto3.session.Session(
            aws_access_key_id=self.s3_configuration.access_key_id,
            aws_secret_access_key=self.s3_configuration.secret_access_key,
            region_name=self.s3_configuration.region_name)

        assert self.s3_session is not None

        use_ssl = True
        endpoint_url = None

        # retrieve the bucket resource and check if the operation was successful
        self.s3_resource = self.s3_session.resource(
            's3',
            config=botocore.client.Config(signature_version='s3v4'),
            use_ssl=use_ssl, endpoint_url=endpoint_url)

        assert self.s3_resource is not None

    def read_all_buckets(self):
        """Read all available buckets from the AWS S3 database."""
        return self.s3_resource.buckets.all()

    def read_bucket_names(self):
        """Read names of all available buckets from the AWS S3 database."""
        buckets = self.read_all_buckets()
        return [bucket.name for bucket in buckets]

    def full_bucket_name(self, bucket_name):
        """Insert deployment prefix to the given bucket name."""
        return "{p}-{b}".format(p=self.deployment_prefix, b=bucket_name)

    @staticmethod
    def package_key(ecosystem, package):
        """Construct a key to the selected package in the given ecosystem."""
        return "{ecosystem}/{package}.json".format(ecosystem=ecosystem,
                                                   package=package)

    @staticmethod
    def package_key_to_metadata(ecosystem, package):
        """Construct a key to the selected package in the given ecosystem."""
        return "{ecosystem}/{package}".format(ecosystem=ecosystem, package=package)

    @staticmethod
    def package_analysis_key(ecosystem, package, metadata):
        """Construct a key to the selected package analysis in the given ecosystem."""
        return "{ecosystem}/{package}/{metadata}.json".format(ecosystem=ecosystem,
                                                              package=package,
                                                              metadata=metadata)

    @staticmethod
    def component_key(ecosystem, package, version):
        """Construct a key to the selected component in the given ecosystem."""
        return "{ecosystem}/{package}/{version}.json".format(ecosystem=ecosystem,
                                                             package=package,
                                                             version=version)

    @staticmethod
    def component_analysis_key(ecosystem, package, version, analysis):
        """Construct a key to the selected component analysis in the given ecosystem."""
        return "{ecosystem}/{package}/{version}/{analysis}.json".format(ecosystem=ecosystem,
                                                                        package=package,
                                                                        version=version,
                                                                        analysis=analysis)

    @staticmethod
    def component_core_package_data_key(ecosystem, package):
        """Construct a key to the selected package in the given ecosystem."""
        return "{ecosystem}/{package}.json".format(ecosystem=ecosystem,
                                                   package=package)

    @staticmethod
    def component_core_package_data_analysis_key(ecosystem, package, analysis):
        """Construct a key to the selected package analysis in the given ecosystem."""
        return "{ecosystem}/{package}/{analysis}.json".format(ecosystem=ecosystem,
                                                              package=package,
                                                              analysis=analysis)

    @property
    def deployment_prefix(self):
        """Get the deployment prefix set up during initialization of this class."""
        return self.s3_configuration.deployment_prefix

    def does_bucket_exist(self, bucket_name):
        """Check if the given bucket exists in the S3 database.

        Return True only when bucket with given name exist and can be read
        by current AWS S3 database user.
        """
        try:
            s3 = self.s3_resource
            assert s3 is not None
            s3.meta.client.head_bucket(Bucket=self.full_bucket_name(bucket_name))
            return True
        except ClientError:
            return False

    def read_object(self, bucket_name, key):
        """Read byte stream from the S3 database, decode it into string, and parse as JSON."""
        s3 = self.s3_resource
        assert s3 is not None
        data = s3.Object(self.full_bucket_name(bucket_name), key).get()['Body'].read().decode()
        return json.loads(data)

    def read_object_metadata(self, bucket_name, key, attribute):
        """Read byte stream from the S3 database, decode it into string, and parse as JSON."""
        s3 = self.s3_resource
        assert s3 is not None
        data = s3.Object(self.full_bucket_name(bucket_name), key).get()[attribute]
        return data

    def read_ecosystems_from_bucket(self, bucket_name):
        """Return list of all ecosystems from selected bucket."""
        bucket = self.s3_resource.Bucket(self.full_bucket_name(bucket_name))
        result = bucket.meta.client.list_objects(Bucket=bucket.name, Delimiter='/')
        names = [o.get('Prefix') for o in result.get('CommonPrefixes')]
        # remove the / at the end of object name
        return [name[:-1] for name in names]

    def read_ecosystems_from_core_package_data(self):
        """Return list of all ecosystems from core-package-data bucket."""
        return self.read_ecosystems_from_bucket("bayesian-core-package-data")

    def read_ecosystems_from_core_data(self):
        """Return list of all ecosystems from core-data bucket."""
        return self.read_ecosystems_from_bucket("bayesian-core-data")

    def read_packages_from_bucket_for_ecosystem(self, ecosystem, bucket_name):
        """Return list of all packages found for the selected ecosystem."""
        if not ecosystem.endswith("/"):
            ecosystem += "/"
        bucket = self.s3_resource.Bucket(self.full_bucket_name(bucket_name))

        # parameters to be passed to list_objects_v2 method
        kwargs = {'Bucket': bucket.name, 'Delimiter': '/', 'Prefix': ecosystem}

        package_names = []

        # the S3 interface supports and requires 'pagination', so we need
        # to get list of package names in a loop
        while True:
            result = bucket.meta.client.list_objects_v2(**kwargs)
            names = [o.get('Prefix') for o in result.get('CommonPrefixes')]
            # names are returned in format "ecosystem/package/"
            # -> we need to get only the package part
            package_names.extend([name[name.find("/") + 1: name.find("/", -1)] for name in names])
            # perform 'pagination', but only when results were truncated
            if not result.get("IsTruncated"):
                break
            # token used by S3 to remember when the next page should begin
            kwargs['ContinuationToken'] = result['NextContinuationToken']

        return package_names

    def read_core_packages_for_ecosystem(self, ecosystem):
        """Return list of all core packages for the selected ecosystem."""
        return self.read_packages_from_bucket_for_ecosystem(ecosystem, "bayesian-core-package-data")

    def read_packages_for_ecosystem(self, ecosystem):
        """Return list of all packages for the selected ecosystem."""
        return self.read_packages_from_bucket_for_ecosystem(ecosystem, "bayesian-core-data")

    def read_object_list(self, bucket_name, ecosystem, package, update_names=True,
                         remove_prefix=False):
        """Read list of objects (JSON files) stored for the given E+P."""
        bucket_name = self.full_bucket_name(bucket_name)
        prefix = S3Interface.package_key_to_metadata(ecosystem, package)
        bucket = self.s3_resource.Bucket(bucket_name)
        result = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Prefix=prefix)
        contents = result["Contents"]
        json_files = [o["Key"] for o in contents]
        if update_names:
            return [json_file[json_file.rfind("/") + 1:] for json_file in json_files]
        elif remove_prefix:
            return [json_file[len(prefix) + 1:] for json_file in json_files]
        else:
            return json_files

    @staticmethod
    def selector_to_key(selector):
        """Construct a key from given selector (that is written in tests w/o underscores)."""
        return selector.lower().replace(" ", "_")
<commit_msg>Removed unused import<commit_after>"""AWS S3 Interface used by tests."""

import boto3
import botocore
from botocore.exceptions import ClientError
import json

class S3Interface():
    """Interface to the AWS S3 database."""

    def __init__(self, s3configuration):
        """Create a new interface to the AWS S3.

        Remember the access key, secret access key, region, and deployment
        prefix that will be used later to connect to the AWS S3.
        """
        self.s3_configuration = s3configuration

        # to be set up by the connect() method
        self.s3_resource = None
        self.s3_session = None

    def connect(self):
        """Connect to the AWS S3 database."""
        # we are already connected -> let's use this connection
        if self.s3_resource is not None:
            return

        # create the session used to communicate with the S3 database
        # and check if the operation was successful
        self.s3_session = boto3.session.Session(
            aws_access_key_id=self.s3_configuration.access_key_id,
            aws_secret_access_key=self.s3_configuration.secret_access_key,
            region_name=self.s3_configuration.region_name)

        assert self.s3_session is not None

        use_ssl = True
        endpoint_url = None

        # retrieve the bucket resource and check if the operation was successful
        self.s3_resource = self.s3_session.resource(
            's3',
            config=botocore.client.Config(signature_version='s3v4'),
            use_ssl=use_ssl, endpoint_url=endpoint_url)

        assert self.s3_resource is not None

    def read_all_buckets(self):
        """Read all available buckets from the AWS S3 database."""
        return self.s3_resource.buckets.all()

    def read_bucket_names(self):
        """Read names of all available buckets from the AWS S3 database."""
        buckets = self.read_all_buckets()
        return [bucket.name for bucket in buckets]

    def full_bucket_name(self, bucket_name):
        """Insert deployment prefix to the given bucket name."""
        return "{p}-{b}".format(p=self.deployment_prefix, b=bucket_name)

    @staticmethod
    def package_key(ecosystem, package):
        """Construct a key to the selected package in the given ecosystem."""
        return "{ecosystem}/{package}.json".format(ecosystem=ecosystem,
                                                   package=package)

    @staticmethod
    def package_key_to_metadata(ecosystem, package):
        """Construct a key to the selected package in the given ecosystem."""
        return "{ecosystem}/{package}".format(ecosystem=ecosystem, package=package)

    @staticmethod
    def package_analysis_key(ecosystem, package, metadata):
        """Construct a key to the selected package analysis in the given ecosystem."""
        return "{ecosystem}/{package}/{metadata}.json".format(ecosystem=ecosystem,
                                                              package=package,
                                                              metadata=metadata)

    @staticmethod
    def component_key(ecosystem, package, version):
        """Construct a key to the selected component in the given ecosystem."""
        return "{ecosystem}/{package}/{version}.json".format(ecosystem=ecosystem,
                                                             package=package,
                                                             version=version)

    @staticmethod
    def component_analysis_key(ecosystem, package, version, analysis):
        """Construct a key to the selected component analysis in the given ecosystem."""
        return "{ecosystem}/{package}/{version}/{analysis}.json".format(ecosystem=ecosystem,
                                                                        package=package,
                                                                        version=version,
                                                                        analysis=analysis)

    @staticmethod
    def component_core_package_data_key(ecosystem, package):
        """Construct a key to the selected package in the given ecosystem."""
        return "{ecosystem}/{package}.json".format(ecosystem=ecosystem,
                                                   package=package)

    @staticmethod
    def component_core_package_data_analysis_key(ecosystem, package, analysis):
        """Construct a key to the selected package analysis in the given ecosystem."""
        return "{ecosystem}/{package}/{analysis}.json".format(ecosystem=ecosystem,
                                                              package=package,
                                                              analysis=analysis)

    @property
    def deployment_prefix(self):
        """Get the deployment prefix set up during initialization of this class."""
        return self.s3_configuration.deployment_prefix

    def does_bucket_exist(self, bucket_name):
        """Check if the given bucket exists in the S3 database.

        Return True only when bucket with given name exist and can be read
        by current AWS S3 database user.
        """
        try:
            s3 = self.s3_resource
            assert s3 is not None
            s3.meta.client.head_bucket(Bucket=self.full_bucket_name(bucket_name))
            return True
        except ClientError:
            return False

    def read_object(self, bucket_name, key):
        """Read byte stream from the S3 database, decode it into string, and parse as JSON."""
        s3 = self.s3_resource
        assert s3 is not None
        data = s3.Object(self.full_bucket_name(bucket_name), key).get()['Body'].read().decode()
        return json.loads(data)

    def read_object_metadata(self, bucket_name, key, attribute):
        """Read byte stream from the S3 database, decode it into string, and parse as JSON."""
        s3 = self.s3_resource
        assert s3 is not None
        data = s3.Object(self.full_bucket_name(bucket_name), key).get()[attribute]
        return data

    def read_ecosystems_from_bucket(self, bucket_name):
        """Return list of all ecosystems from selected bucket."""
        bucket = self.s3_resource.Bucket(self.full_bucket_name(bucket_name))
        result = bucket.meta.client.list_objects(Bucket=bucket.name, Delimiter='/')
        names = [o.get('Prefix') for o in result.get('CommonPrefixes')]
        # remove the / at the end of object name
        return [name[:-1] for name in names]

    def read_ecosystems_from_core_package_data(self):
        """Return list of all ecosystems from core-package-data bucket."""
        return self.read_ecosystems_from_bucket("bayesian-core-package-data")

    def read_ecosystems_from_core_data(self):
        """Return list of all ecosystems from core-data bucket."""
        return self.read_ecosystems_from_bucket("bayesian-core-data")

    def read_packages_from_bucket_for_ecosystem(self, ecosystem, bucket_name):
        """Return list of all packages found for the selected ecosystem."""
        if not ecosystem.endswith("/"):
            ecosystem += "/"
        bucket = self.s3_resource.Bucket(self.full_bucket_name(bucket_name))

        # parameters to be passed to list_objects_v2 method
        kwargs = {'Bucket': bucket.name, 'Delimiter': '/', 'Prefix': ecosystem}

        package_names = []

        # the S3 interface supports and requires 'pagination', so we need
        # to get list of package names in a loop
        while True:
            result = bucket.meta.client.list_objects_v2(**kwargs)
            names = [o.get('Prefix') for o in result.get('CommonPrefixes')]
            # names are returned in format "ecosystem/package/"
            # -> we need to get only the package part
            package_names.extend([name[name.find("/") + 1: name.find("/", -1)] for name in names])
            # perform 'pagination', but only when results were truncated
            if not result.get("IsTruncated"):
                break
            # token used by S3 to remember when the next page should begin
            kwargs['ContinuationToken'] = result['NextContinuationToken']

        return package_names

    def read_core_packages_for_ecosystem(self, ecosystem):
        """Return list of all core packages for the selected ecosystem."""
        return self.read_packages_from_bucket_for_ecosystem(ecosystem, "bayesian-core-package-data")

    def read_packages_for_ecosystem(self, ecosystem):
        """Return list of all packages for the selected ecosystem."""
        return self.read_packages_from_bucket_for_ecosystem(ecosystem, "bayesian-core-data")

    def read_object_list(self, bucket_name, ecosystem, package, update_names=True,
                         remove_prefix=False):
        """Read list of objects (JSON files) stored for the given E+P."""
        bucket_name = self.full_bucket_name(bucket_name)
        prefix = S3Interface.package_key_to_metadata(ecosystem, package)
        bucket = self.s3_resource.Bucket(bucket_name)
        result = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Prefix=prefix)
        contents = result["Contents"]
        json_files = [o["Key"] for o in contents]
        if update_names:
            return [json_file[json_file.rfind("/") + 1:] for json_file in json_files]
        elif remove_prefix:
            return [json_file[len(prefix) + 1:] for json_file in json_files]
        else:
            return json_files

    @staticmethod
    def selector_to_key(selector):
        """Construct a key from given selector (that is written in tests w/o underscores)."""
        return selector.lower().replace(" ", "_")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db.models import Sum
from django.db.models.functions import TruncDate
from django.utils.translation import gettext as _

import plotly.offline as plotly
import plotly.graph_objs as go

from reports import utils

def feeding_amounts(instances):
    """
    Create a graph showing daily feeding amounts over time.
    :param instances: a QuerySet of Feeding instances.
    :returns: a tuple of the the graph's html and javascript.
    """
    totals = instances.annotate(date=TruncDate('start')) \
        .values('date') \
        .annotate(sum=Sum('amount')) \
        .order_by('-date')

    dates = [value['date'] for value in totals.values('date')]
    amounts = [value['amount'] or 0 for value in totals.values('amount')]

    trace = go.Bar(
        name=_('Total feeding amount'),
        x=dates,
        y=amounts,
        hoverinfo='text',
        textposition='outside',
        text=amounts
    )

    layout_args = utils.default_graph_layout_options()
    layout_args['title'] = _('Total Feeding Amounts')
    layout_args['xaxis']['title'] = _('Date')
    layout_args['xaxis']['rangeselector'] = utils.rangeselector_date()
    layout_args['yaxis']['title'] = _('Feeding amount')

    fig = go.Figure({
        'data': [trace],
        'layout': go.Layout(**layout_args)
    })
    output = plotly.plot(fig, output_type='div', include_plotlyjs=False)
    return utils.split_graph_output(output)
<commit_msg>Sum amounts and account for timezone in "Feeding Amounts" report. (#68)<commit_after># -*- coding: utf-8 -*-
from django.utils import timezone
from django.utils.translation import gettext as _

import plotly.offline as plotly
import plotly.graph_objs as go

from reports import utils

def feeding_amounts(instances):
    """
    Create a graph showing daily feeding amounts over time.
    :param instances: a QuerySet of Feeding instances.
    :returns: a tuple of the the graph's html and javascript.
    """
    totals = {}
    for instance in instances:
        end = timezone.localtime(instance.end)
        date = end.date()
        if date not in totals.keys():
            totals[date] = 0
        totals[date] += instance.amount or 0

    amounts = [round(amount, 2) for amount in totals.values()]
    trace = go.Bar(
        name=_('Total feeding amount'),
        x=list(totals.keys()),
        y=amounts,
        hoverinfo='text',
        textposition='outside',
        text=amounts
    )

    layout_args = utils.default_graph_layout_options()
    layout_args['title'] = _('Total Feeding Amounts')
    layout_args['xaxis']['title'] = _('Date')
    layout_args['xaxis']['rangeselector'] = utils.rangeselector_date()
    layout_args['yaxis']['title'] = _('Feeding amount')

    fig = go.Figure({
        'data': [trace],
        'layout': go.Layout(**layout_args)
    })
    output = plotly.plot(fig, output_type='div', include_plotlyjs=False)
    return utils.split_graph_output(output)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import json
import urllib

class Traffic:

    """Wikipedia article statistics."""

    def __init__(self, title, site):
        self.site = site
        self.title = title

    def _make_url(self, year, month):
        """Make the URL to the JSON output of stats.grok.se service."""
        return "http://stats.grok.se/json/{0:s}/{1:d}{2:02d}/{3:s}".format(self.site, year, month, self.title)

    def _make_url_latest(self, latest):
        """Make URL to the JSON output of stats.grok.se service for lastest traffic.

        Args:
            latest (int): 30, 60 or 90
        """
        if latest not in [30, 60, 90]:
            raise ValueError("Expected 30, 60 or 90 instead of %s" % (latest))
        return "http://stats.grok.se/json/{0:s}/latest{1:d}/{2:s}".format(self.site, latest, self.title)

    def get_month_traffic(self, year, month):
        """Fetch the month traffic statistics."""
        return json.loads(urllib.urlopen(self._make_url(year, month)).read())

    def get_latest_traffic(self, latest):
        """Fetch the latest traffic statistics."""
        return json.loads(urllib.urlopen(self._make_url_latest(latest)).read())
<commit_msg>Module docstring, object style, and unix way<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Traffic statistics API to grok."""

import json
import urllib

class Traffic(object):

    """Wikipedia article statistics."""

    def __init__(self, title, site):
        self.site = site
        self.title = title

    def _make_url(self, year, month):
        """Make the URL to the JSON output of stats.grok.se service."""
        return "http://stats.grok.se/json/{0:s}/{1:d}{2:02d}/{3:s}".format(self.site, year, month, self.title)

    def _make_url_latest(self, latest):
        """Make URL to the JSON output of stats.grok.se service for lastest traffic.

        Args:
            latest (int): 30, 60 or 90
        """
        if latest not in [30, 60, 90]:
            raise ValueError("Expected 30, 60 or 90 instead of %s" % (latest))
        return "http://stats.grok.se/json/{0:s}/latest{1:d}/{2:s}".format(self.site, latest, self.title)

    def get_month_traffic(self, year, month):
        """Fetch the month traffic statistics."""
        return json.loads(urllib.urlopen(self._make_url(year, month)).read())

    def get_latest_traffic(self, latest):
        """Fetch the latest traffic statistics."""
        return json.loads(urllib.urlopen(self._make_url_latest(latest)).read())
<|endoftext|>


<commit_before>import uuid

from collections.abc import Mapping, Sequence
from logbook import Logger
from .runtime import behavior, Actor

logger = Logger('membrane')

class Membrane(object):

    def __init__(self, config, runtime):
        self.config = config
        if self.config['protocol'] == 'membrane':
            self.config['target'] = self
        self.runtime = runtime
        self.uid_to_proxy = {}
        self.proxy_to_uid = {}
        self.start_server()

    def get_proxy(self, uid):
        """Get proxy associated to ``uid``.

        Raise an exception is there is not associated proxy.

        """
        proxy = self.uid_to_proxy[uid]
        self.proxy_to_uid[proxy] = uid
        return proxy

    def create_proxy(self, uid, remote):
        """Create proxy for ``uid`` in the ``remote`` membrane."""
        proxy = self.runtime.create(self.proxy, uid, remote)
        self.uid_to_proxy[uid] = proxy
        self.proxy_to_uid[proxy] = uid
        return proxy

    def get_or_create_proxy(self, uid, remote):
        """Get or create a new proxy for ``uid`` in the ``remote`` membrane."""
        try:
            return self.get_proxy(uid)
        except KeyError:
            return self.create_proxy(uid, remote)

    def get_uid(self, actor):
        """Convert an actor to a uid."""
        uid = self.proxy_to_uid.setdefault(actor, uuid.uuid4().hex)
        self.uid_to_proxy[uid] = actor
        return uid

    @behavior
    def proxy(self, uid, remote, this, msg):
        protocol = remote['protocol']
        client = getattr(self, '{}_client'.format(protocol))
        client(self.marshall_message(msg), uid, remote)

    def local_delivery(self, uid, msg):
        """Deliver ``msg`` to actor with id ``uid``."""
        self.uid_to_proxy[uid] << self.unmarshall_message(msg)

    def marshall_actor(self, actor):
        uid = self.get_uid(actor)
        return {'_proxy': uid,
                '_config': self.config}

    def unmarshall_actor(self, obj):
        uid = obj['_proxy']
        return self.get_or_create_proxy(uid, obj['_config'])

    def is_marshalled_actor(self, obj):
        try:
            return set(obj.keys()) == {'_proxy', '_config'}
        except AttributeError:
            return False

    def unmarshall_message(self, msg):
        """Import a message.

        Uid identifiers of the form::

            {'_proxy': uid,
             '_transport': ...}

        are substituted for the proper actor to which they refer.

        """
        if self.is_marshalled_actor(msg):
            return self.unmarshall_actor(msg)
        if isinstance(msg, Mapping):
            return {key:self.unmarshall_message(value)
                    for key, value in msg.items()}
        if isinstance(msg, str):
            return msg
        if isinstance(msg, Sequence):
            return [self.unmarshall_message(value) for value in msg]
        return msg

    def marshall_message(self, msg):
        """Export a message.

        Convert any actor reference to a uid. Proceed recursively to
        convert references at any level.

        """
        if isinstance(msg, Actor):
            return self.marshall_actor(msg)
        if isinstance(msg, Mapping):
            return {key:self.marshall_message(value)
                    for key, value in msg.items()}
        if isinstance(msg, str):
            return msg
        if isinstance(msg, Sequence):
            return [self.marshall_message(value) for value in msg]
        return msg

    def start_server(self):
        pass

    def membrane_client(self, msg, uid, remote):
        logger.debug('membrane client')
        logger.debug(' target: uid: %s' %uid)
        logger.debug(' at membrane: %s' %remote['target'])
        logger.debug(' and message is %s' %msg)

        remote['target'].local_delivery(uid, msg)

def test():
    from .runtime import SimpleRuntime
    from .example import print_beh, echo_beh
    from .eventloop import EventLoop

    runtime = SimpleRuntime()

    @behavior
    def print_echo_beh(self, msg):
        print('echo got', msg)
        msg['reply_to'] << {'answer': msg}

    mb = Membrane({'protocol': 'membrane'}, runtime)
    print_echo = runtime.create(print_echo_beh)
    uid_at_mb = mb.get_uid(print_echo)

    mb2 = Membrane({'protocol': 'membrane'}, runtime)
    proxy_for_echo = mb2.create_proxy(uid_at_mb, {'protocol': 'membrane',
                                                  'target': mb})
    printer = runtime.create(print_beh)
    proxy_for_echo << {'tag': 5, 'reply_to': printer}

    evloop = EventLoop()

    return locals()<commit_msg>Implement start_server<commit_after>import uuid

from collections.abc import Mapping, Sequence
from logbook import Logger
from .runtime import behavior, Actor

logger = Logger('membrane')

class Membrane(object):

    def __init__(self, config, runtime):
        self.config = config
        if self.config['protocol'] == 'membrane':
            self.config['target'] = self
        self.runtime = runtime
        self.uid_to_proxy = {}
        self.proxy_to_uid = {}
        self.start_server()

    def get_proxy(self, uid):
        """Get proxy associated to ``uid``.

        Raise an exception is there is not associated proxy.

        """
        proxy = self.uid_to_proxy[uid]
        self.proxy_to_uid[proxy] = uid
        return proxy

    def create_proxy(self, uid, remote):
        """Create proxy for ``uid`` in the ``remote`` membrane."""
        proxy = self.runtime.create(self.proxy, uid, remote)
        self.uid_to_proxy[uid] = proxy
        self.proxy_to_uid[proxy] = uid
        return proxy

    def get_or_create_proxy(self, uid, remote):
        """Get or create a new proxy for ``uid`` in the ``remote`` membrane."""
        try:
            return self.get_proxy(uid)
        except KeyError:
            return self.create_proxy(uid, remote)

    def get_uid(self, actor):
        """Convert an actor to a uid."""
        uid = self.proxy_to_uid.setdefault(actor, uuid.uuid4().hex)
        self.uid_to_proxy[uid] = actor
        return uid

    @behavior
    def proxy(self, uid, remote, this, msg):
        protocol = remote['protocol']
        client = getattr(self, '{}_client'.format(protocol))
        client(self.marshall_message(msg), uid, remote)

    def start_server(self):
        protocol = self.config['protocol']
        server = getattr(self, '{}_server'.format(protocol))
        self.server_thread = threading.Thread(target=server)
        self.server_thread.daemon = True
        self.server_thread.start()

    def local_delivery(self, uid, msg):
        """Deliver ``msg`` to actor with id ``uid``."""
        self.uid_to_proxy[uid] << self.unmarshall_message(msg)

    def marshall_actor(self, actor):
        uid = self.get_uid(actor)
        return {'_proxy': uid,
                '_config': self.config}

    def unmarshall_actor(self, obj):
        uid = obj['_proxy']
        return self.get_or_create_proxy(uid, obj['_config'])

    def is_marshalled_actor(self, obj):
        try:
            return set(obj.keys()) == {'_proxy', '_config'}
        except AttributeError:
            return False

    def unmarshall_message(self, msg):
        """Import a message.

        Uid identifiers of the form::

            {'_proxy': uid,
             '_transport': ...}

        are substituted for the proper actor to which they refer.

        """
        if self.is_marshalled_actor(msg):
            return self.unmarshall_actor(msg)
        if isinstance(msg, Mapping):
            return {key:self.unmarshall_message(value)
                    for key, value in msg.items()}
        if isinstance(msg, str):
            return msg
        if isinstance(msg, Sequence):
            return [self.unmarshall_message(value) for value in msg]
        return msg

    def marshall_message(self, msg):
        """Export a message.

        Convert any actor reference to a uid. Proceed recursively to
        convert references at any level.

        """
        if isinstance(msg, Actor):
            return self.marshall_actor(msg)
        if isinstance(msg, Mapping):
            return {key:self.marshall_message(value)
                    for key, value in msg.items()}
        if isinstance(msg, str):
            return msg
        if isinstance(msg, Sequence):
            return [self.marshall_message(value) for value in msg]
        return msg

    def membrane_client(self, msg, uid, remote):
        logger.debug('membrane client')
        logger.debug(' target: uid: %s' %uid)
        logger.debug(' at membrane: %s' %remote['target'])
        logger.debug(' and message is %s' %msg)

        remote['target'].local_delivery(uid, msg)

    def membrane_server(self):
        pass

def test():
    from .runtime import SimpleRuntime
    from .example import print_beh, echo_beh
    from .eventloop import EventLoop

    runtime = SimpleRuntime()

    @behavior
    def print_echo_beh(self, msg):
        print('echo got', msg)
        msg['reply_to'] << {'answer': msg}

    mb = Membrane({'protocol': 'membrane'}, runtime)
    print_echo = runtime.create(print_echo_beh)
    uid_at_mb = mb.get_uid(print_echo)

    mb2 = Membrane({'protocol': 'membrane'}, runtime)
    proxy_for_echo = mb2.create_proxy(uid_at_mb, {'protocol': 'membrane',
                                                  'target': mb})
    printer = runtime.create(print_beh)
    proxy_for_echo << {'tag': 5, 'reply_to': printer}

    evloop = EventLoop()

    return locals()<|endoftext|>


<commit_before><commit_msg>update existing permission sets to be private<commit_after><|endoftext|>


<commit_before>from django.shortcuts import render_to_response, redirect
from django_auth_lti.decorators import lti_role_required

from ab_testing_tool_app.constants import ADMINS
from ab_testing_tool_app.models import Track, StageUrl, CourseSetting, Stage
from ab_testing_tool_app.canvas import get_lti_param
from ab_testing_tool_app.decorators import page
from ab_testing_tool_app.exceptions import (MISSING_TRACK, UNAUTHORIZED_ACCESS,
    COURSE_TRACKS_ALREADY_FINALIZED, NO_TRACKS_FOR_COURSE)
from django.http.response import HttpResponse
from ab_testing_tool_app.controllers import post_param

@lti_role_required(ADMINS)
@page
def create_track(request):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    if CourseSetting.get_is_finalized(course_id):
        raise COURSE_TRACKS_ALREADY_FINALIZED
    return render_to_response("edit_track.html")

@lti_role_required(ADMINS)
@page
def edit_track(request, track_id):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    t = Track.get_or_none(pk=track_id)
    if not t:
        raise MISSING_TRACK
    if course_id != t.course_id:
        raise UNAUTHORIZED_ACCESS
    is_finalized = CourseSetting.get_is_finalized(course_id)
    context = {"track": t,
               "is_finalized": is_finalized}
    return render_to_response("edit_track.html", context)

@lti_role_required(ADMINS)
@page
def submit_create_track(request):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    if CourseSetting.get_is_finalized(course_id):
        raise COURSE_TRACKS_ALREADY_FINALIZED
    name = post_param(request, "name")
    notes = post_param(request, "notes")
    Track.objects.create(name=name, notes=notes, course_id=course_id)
    return redirect("/")

@lti_role_required(ADMINS)
@page
def submit_edit_track(request):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    name = post_param(request, "name")
    notes = post_param(request, "notes")
    track_id = post_param(request, "id")
    track = Track.get_or_none(pk=track_id, course_id=course_id)
    if not track:
        raise MISSING_TRACK
    track.update(name=name, notes=notes)
    return redirect("/")

@lti_role_required(ADMINS)
@page
def delete_track(request, track_id):
    """
    NOTE: When a track gets deleted, urls for that track get deleted from all
          stages in that course as a result of cascading delete.
    """
    course_id = get_lti_param(request, "custom_canvas_course_id")
    if CourseSetting.get_is_finalized(course_id):
        raise COURSE_TRACKS_ALREADY_FINALIZED
    track = Track.get_or_none(pk=track_id, course_id=course_id)
    if not track:
        raise MISSING_TRACK
    track.delete()
    return redirect("/")

@lti_role_required(ADMINS)
@page
def finalize_tracks(request):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    if Track.objects.filter(course_id=course_id).count() == 0:
        raise NO_TRACKS_FOR_COURSE
    missing_urls = [stage.name for stage
                    in Stage.objects.filter(course_id=course_id)
                    if stage.is_missing_urls()]
    if missing_urls:
        #TODO: replace with better error display
        return HttpResponse("URLs missing for these tracks in these Stages: %s" % missing_urls)
    CourseSetting.set_finalized(course_id)
    return redirect("/")
<commit_msg>adds correct exception to unauthorized access<commit_after>from django.shortcuts import render_to_response, redirect
from django_auth_lti.decorators import lti_role_required

from ab_testing_tool_app.constants import ADMINS
from ab_testing_tool_app.models import Track, StageUrl, CourseSetting, Stage
from ab_testing_tool_app.canvas import get_lti_param
from ab_testing_tool_app.decorators import page
from ab_testing_tool_app.exceptions import (MISSING_TRACK, UNAUTHORIZED_ACCESS,
    COURSE_TRACKS_ALREADY_FINALIZED, NO_TRACKS_FOR_COURSE)
from django.http.response import HttpResponse
from ab_testing_tool_app.controllers import post_param

@lti_role_required(ADMINS)
@page
def create_track(request):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    if CourseSetting.get_is_finalized(course_id):
        raise COURSE_TRACKS_ALREADY_FINALIZED
    return render_to_response("edit_track.html")

@lti_role_required(ADMINS)
@page
def edit_track(request, track_id):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    t = Track.get_or_none(pk=track_id)
    if not t:
        raise MISSING_TRACK
    if course_id != t.course_id:
        raise UNAUTHORIZED_ACCESS
    is_finalized = CourseSetting.get_is_finalized(course_id)
    context = {"track": t,
               "is_finalized": is_finalized}
    return render_to_response("edit_track.html", context)

@lti_role_required(ADMINS)
@page
def submit_create_track(request):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    if CourseSetting.get_is_finalized(course_id):
        raise COURSE_TRACKS_ALREADY_FINALIZED
    name = post_param(request, "name")
    notes = post_param(request, "notes")
    Track.objects.create(name=name, notes=notes, course_id=course_id)
    return redirect("/")

@lti_role_required(ADMINS)
@page
def submit_edit_track(request):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    name = post_param(request, "name")
    notes = post_param(request, "notes")
    track_id = post_param(request, "id")
    track = Track.get_or_none(pk=track_id)
    if not track:
        raise MISSING_TRACK
    if course_id != track.course_id:
        raise UNAUTHORIZED_ACCESS
    track.update(name=name, notes=notes)
    return redirect("/")

@lti_role_required(ADMINS)
@page
def delete_track(request, track_id):
    """
    NOTE: When a track gets deleted, urls for that track get deleted from all
          stages in that course as a result of cascading delete.
    """
    course_id = get_lti_param(request, "custom_canvas_course_id")
    if CourseSetting.get_is_finalized(course_id):
        raise COURSE_TRACKS_ALREADY_FINALIZED
    track = Track.get_or_none(pk=track_id)
    if not track:
        raise MISSING_TRACK
    if course_id != track.course_id:
        raise UNAUTHORIZED_ACCESS
    track.delete()
    return redirect("/")

@lti_role_required(ADMINS)
@page
def finalize_tracks(request):
    course_id = get_lti_param(request, "custom_canvas_course_id")
    if Track.objects.filter(course_id=course_id).count() == 0:
        raise NO_TRACKS_FOR_COURSE
    missing_urls = [stage.name for stage
                    in Stage.objects.filter(course_id=course_id)
                    if stage.is_missing_urls()]
    if missing_urls:
        #TODO: replace with better error display
        return HttpResponse("URLs missing for these tracks in these Stages: %s" % missing_urls)
    CourseSetting.set_finalized(course_id)
    return redirect("/")
<|endoftext|>


<commit_before>"""
.. Copyright (c) 2015 Marshall Farrier, Robert Rodrigues, Mark Scappini
   license http://opensource.org/licenses/MIT

Quote puller service main
=========================
Start the service from the python interpreter in the appropriate virtual
environment:
http://stackoverflow.com/questions/7807315/daemonizing-a-python-script-in-debian-using-virtualenv
For example:

    $ /var/local/.virtualenvs/quotepuller/bin/python <path>/quotepuller.py
"""
import argparse
import ConfigParser
import logging
import os.path
from functools import partial
import signal
import sys
import time

import constants
import dbwrapper
import eqgetter

logger = logging.getLogger(constants.SERVICE_NAME)
test_mode = False

def stop_handler(_signal, frame):
    # http://stackoverflow.com/questions/1112343/how-do-i-capture-sigint-in-python/1112350#1112350
    msg = ('SIGINT' if _signal == signal.SIGINT else 'SIGTERM')
    logger.info('signal {} received. stopping'.format(msg))
    sys.exit(0)

def run(dbconn):
    logger.info('starting')
    logger.debug('dbconn: {}'.format(dbconn))
    _success = False
    while not _success:
        try:
            _equities = dbwrapper.job(dbconn, logger, partial(eqgetter.active, test_mode))
        except:
            logger.error('could not retrieve equities')
            time.sleep(300)
        else:
            _success = True
    signal.pause()

def init():
    # check for test mode
    _parser = argparse.ArgumentParser()
    _parser.add_argument('--test', action='store_true')
    test_mode = _parser.parse_args().test
    _config = ConfigParser.SafeConfigParser()
    _sec = constants.CFGSEC_MAIN
    _config.read(constants.CONFIGFILE)
    _logpath = _config.get(_sec, 'logpath', 1)
    _logfmt = _config.get(_sec, 'logfmt', 1)
    if test_mode:
        _logfmt += ' (TESTING)'
    _handler = logging.FileHandler(os.path.join(_logpath, 'service.log'))
    _formatter = logging.Formatter(_logfmt)
    _handler.setFormatter(_formatter)
    logger.addHandler(_handler)
    if test_mode:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)
    dbconn = _config.get(_sec, 'dbconn', 1)
    logger.debug('dbconn: {}'.format(dbconn))
    signal.signal(signal.SIGTERM, stop_handler)
    signal.signal(signal.SIGINT, stop_handler)
    return dbconn

if __name__ == '__main__':
    _dbconn = init()
    run(_dbconn)
<commit_msg>Improve logging.<commit_after>"""
.. Copyright (c) 2015 Marshall Farrier, Robert Rodrigues, Mark Scappini
   license http://opensource.org/licenses/MIT

Quote puller service main
=========================
Start the service from the python interpreter in the appropriate virtual
environment:
http://stackoverflow.com/questions/7807315/daemonizing-a-python-script-in-debian-using-virtualenv
For example:

    $ /var/local/.virtualenvs/quotepuller/bin/python <path>/quotepuller.py
"""
import argparse
import ConfigParser
import logging
import os.path
from functools import partial
import signal
import sys
import time

import constants
import dbwrapper
import eqgetter

logger = logging.getLogger(constants.SERVICE_NAME)
test_mode = False

def stop_handler(_signal, frame):
    # http://stackoverflow.com/questions/1112343/how-do-i-capture-sigint-in-python/1112350#1112350
    msg = ('SIGINT' if _signal == signal.SIGINT else 'SIGTERM')
    logger.info('signal {} received. stopping'.format(msg))
    sys.exit(0)

def run(dbconn):
    logger.info('starting')
    logger.debug('dbconn: {}'.format(dbconn))
    _success = False
    while not _success:
        try:
            _equities = dbwrapper.job(dbconn, logger, partial(eqgetter.active, test_mode))
        except:
            logger.exception('could not retrieve equities')
            time.sleep(300)
        else:
            _success = True
    signal.pause()

def init():
    # check for test mode
    _parser = argparse.ArgumentParser()
    _parser.add_argument('--test', action='store_true')
    test_mode = _parser.parse_args().test
    _config = ConfigParser.SafeConfigParser()
    _sec = constants.CFGSEC_MAIN
    _config.read(constants.CONFIGFILE)
    _logpath = _config.get(_sec, 'logpath', 1)
    _logfmt = _config.get(_sec, 'logfmt', 1)
    if test_mode:
        _logfmt += ' (TESTING)'
    _handler = logging.FileHandler(os.path.join(_logpath, 'service.log'))
    _formatter = logging.Formatter(_logfmt)
    _handler.setFormatter(_formatter)
    logger.addHandler(_handler)
    if test_mode:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)
    dbconn = _config.get(_sec, 'dbconn', 1)
    logger.debug('dbconn: {}'.format(dbconn))
    signal.signal(signal.SIGTERM, stop_handler)
    signal.signal(signal.SIGINT, stop_handler)
    return dbconn

if __name__ == '__main__':
    _dbconn = init()
    run(_dbconn)
<|endoftext|>


<commit_before>#!flask/bin/python
from datetime import timedelta

from flask import Flask, jsonify, request, make_response, render_template
from functools import update_wrapper
app = Flask(__name__)

from simulation import Simulation
simulation = Simulation(100.0)

def crossdomain(origin=None):
    def decorator(f):
        def wrapped_function(*args, **kwargs):
            resp = make_response(f(*args, **kwargs))
            h = resp.headers
            h['Access-Control-Allow-Origin'] = origin
            return resp
        return update_wrapper(wrapped_function, f)
    return decorator

@app.route('/')
@crossdomain(origin='*')
def index():
    return render_template('index.html')

@app.route('/api/data/', methods = ['GET'])
@crossdomain(origin='*')
def get_simulation_data():
    output = {}
    for i in simulation.devices:
        device = simulation.devices[i]
        device_data = {}
        for key, sensor in device.sensors.items():
            device_data[sensor.name] = sensor.value
        output[device.device_id] = device_data

    return jsonify( output )

@app.route('/api/info/', methods = ['GET'])
@crossdomain(origin='*')
def get_simulation_info():
    output = {}
    for i in simulation.devices:
        device = simulation.devices[i]
        device_data = {'name': device.name}
        for key, sensor in device.sensors.items():
            device_data[sensor.name] = sensor.unit
        output[device.device_id] = device_data

    return jsonify(output)

@app.route('/api/set/', methods = ['POST'])
@crossdomain(origin='*')
def set_data():
    # set heating temperature
    if "room_temperature" in request.form:
        temperature = float(request.form['room_temperature'])
        if temperature >= 10 and temperature <= 30:
            simulation.set_heating(temperature)
            return "1"
    # set electrical consumption
    if "electric_consumption" in request.form:
        power = float(request.form['electric_consumption'])
        simulation.set_electric_consumption(power)
        return "1"

    # set outside temperature
    if "outside_temperature" in request.form:
        temperature = float(request.form['outside_temperature'])
        simulation.set_outside_temperature(temperature)
        return "1"

    if "fast_forward" in request.form:
        time_interval = int(request.form['fast_forward'])
        return jsonify(simulation.fast_forward(time_interval, 2000.0))

    return "0"

@app.route('/api/device/<int:device_id>/get/', methods = ['GET'])
@crossdomain(origin='*')
def get_data(device_id):
    device = simulation.devices[device_id]
    device_data = {}
    for key,sensor in device.sensors.items():
        device_data[sensor.name] = sensor.value
    return jsonify( device_data )

@app.route('/api/device/<int:device_id>/info/', methods = ['GET'])
@crossdomain(origin='*')
def get_info(device_id):
    device = simulation.devices[device_id]
    device_data = {"device_name":device.name}
    for key, sensor in device.sensors.items():
        device_data[sensor.name] = sensor.unit
    return jsonify(device_data)

if __name__ == '__main__':
    simulation.start()
    app.run(host="0.0.0.0",debug = True, port = 9000, use_reloader=False)
<commit_msg>Remove multiple heating storages from /data hook<commit_after>#!flask/bin/python
from datetime import timedelta

from flask import Flask, jsonify, request, make_response, render_template
from functools import update_wrapper
app = Flask(__name__)

from simulation import Simulation
simulation = Simulation(100.0)

def crossdomain(origin=None):
    def decorator(f):
        def wrapped_function(*args, **kwargs):
            resp = make_response(f(*args, **kwargs))
            h = resp.headers
            h['Access-Control-Allow-Origin'] = origin
            return resp
        return update_wrapper(wrapped_function, f)
    return decorator

@app.route('/')
@crossdomain(origin='*')
def index():
    return render_template('index.html')

@app.route('/api/data/', methods = ['GET'])
@crossdomain(origin='*')
def get_simulation_data():
    output = {}
    for i in range(5):
        device = simulation.devices[i]
        device_data = {}
        for key, sensor in device.sensors.items():
            device_data[sensor.name] = sensor.value
        output[device.device_id] = device_data

    return jsonify( output )

@app.route('/api/info/', methods = ['GET'])
@crossdomain(origin='*')
def get_simulation_info():
    output = {}
    for i in simulation.devices:
        device = simulation.devices[i]
        device_data = {'name': device.name}
        for key, sensor in device.sensors.items():
            device_data[sensor.name] = sensor.unit
        output[device.device_id] = device_data

    return jsonify(output)

@app.route('/api/set/', methods = ['POST'])
@crossdomain(origin='*')
def set_data():
    # set heating temperature
    if "room_temperature" in request.form:
        temperature = float(request.form['room_temperature'])
        if temperature >= 10 and temperature <= 30:
            simulation.set_heating(temperature)
            return "1"
    # set electrical consumption
    if "electric_consumption" in request.form:
        power = float(request.form['electric_consumption'])
        simulation.set_electric_consumption(power)
        return "1"

    # set outside temperature
    if "outside_temperature" in request.form:
        temperature = float(request.form['outside_temperature'])
        simulation.set_outside_temperature(temperature)
        return "1"

    if "fast_forward" in request.form:
        time_interval = int(request.form['fast_forward'])
        return jsonify(simulation.fast_forward(time_interval, 2000.0))

    return "0"

@app.route('/api/device/<int:device_id>/get/', methods = ['GET'])
@crossdomain(origin='*')
def get_data(device_id):
    device = simulation.devices[device_id]
    device_data = {}
    for key,sensor in device.sensors.items():
        device_data[sensor.name] = sensor.value
    return jsonify( device_data )

@app.route('/api/device/<int:device_id>/info/', methods = ['GET'])
@crossdomain(origin='*')
def get_info(device_id):
    device = simulation.devices[device_id]
    device_data = {"device_name":device.name}
    for key, sensor in device.sensors.items():
        device_data[sensor.name] = sensor.unit
    return jsonify(device_data)

if __name__ == '__main__':
    simulation.start()
    app.run(host="0.0.0.0",debug = True, port = 9000, use_reloader=False)
<|endoftext|>


<commit_before>"""
The ``list`` module: Lists of traceables
===============================================================================

"""

from docutils import nodes
from docutils.parsers.rst import Directive, directives

from .infrastructure import ProcessorBase, TraceablesFilter

# =============================================================================
# Node types

class traceable_list(nodes.General, nodes.Element):
    """Placeholder node to be replaced by a list of traceables.

    Attributes:
        traceable-filter: The filter expression to determine which
            traceables to include in the output.

    """

    pass

# =============================================================================
# Processors

class ListProcessor(ProcessorBase):

    def __init__(self, app):
        ProcessorBase.__init__(self, app)

    def process_doctree(self, doctree, docname):
        traceables = sorted(self.storage.traceables_set,
                            key=lambda t: t.tag)
        filter = TraceablesFilter(traceables)
        for list_node in doctree.traverse(traceable_list):
            filter_expression = list_node["traceables-filter"]
            if filter_expression:
                filtered_traceables = filter.filter(filter_expression)
            else:
                filtered_traceables = traceables
            new_node = nodes.bullet_list()
            for traceable in filtered_traceables:
                item_node = nodes.list_item()
                new_node += item_node
                inline = nodes.inline()
                item_node += inline
                inline += traceable.make_reference_node(
                    self.app.builder, docname)
                if traceable.has_title:
                    title = u" " + traceable.title
                    inline += nodes.inline(title, title)
            list_node.replace_self(new_node)

# =============================================================================
# Directives

class TraceableListDirective(Directive):

    required_arguments = 0
    optional_arguments = 0
    option_spec = {
        "filter": directives.unchanged,
    }

    def run(self):
        env = self.state.document.settings.env
        node = traceable_list()
        node.docname = env.docname
        node.lineno = self.lineno
        node["traceables-filter"] = self.options.get("filter")
        return [node]

# =============================================================================
# Setup this extension part

def setup(app):
    app.add_node(traceable_list)
    app.add_directive("traceable-list", TraceableListDirective)
<commit_msg>Add pluggable formatters for traceable lists<commit_after>"""
The ``list`` module: Lists of traceables
===============================================================================

"""

from docutils import nodes
from docutils.parsers.rst import Directive, directives

from .infrastructure import FormatProcessorBase, TraceablesFilter

# =============================================================================
# Node types

class traceable_list(nodes.General, nodes.Element):
    """Placeholder node to be replaced by a list of traceables.

    Attributes:
        traceable-filter: The filter expression to determine which
            traceables to include in the output.

    """

    pass

# =============================================================================
# Processors

class ListProcessor(FormatProcessorBase):

    def __init__(self, app):
        FormatProcessorBase.__init__(self, app, traceable_list)

    def process_node_with_formatter(self, list_node, formatter,
                                    doctree, docname):
        traceables = sorted(self.storage.traceables_set,
                            key=lambda t: t.tag)

        filter = TraceablesFilter(traceables)
        filter_expression = list_node["traceables-filter"]
        if filter_expression:
            filtered_traceables = filter.filter(filter_expression)
        else:
            filtered_traceables = traceables

        options = {
            "format": list_node.get("traceables-format"),
            "filter": list_node.get("traceables-filter"),
            "attributes": list_node.get("traceables-attributes"),
        }

        new_nodes = formatter.format(self.app, docname, list_node,
                                     filtered_traceables, options)
        list_node.replace_self(new_nodes)

# =============================================================================
# Built-in formatters

class ListFormatterBase(object):

    def format(self, app, docname, node, traceables, options):
        raise NotImplementedError()

# -----------------------------------------------------------------------------

class TableListFormatter(ListFormatterBase):

    def format(self, app, docname, node, traceables, options):
        additional_attributes = options.get("attributes") or []
        columns = ["tag", "title"] + additional_attributes

        table = nodes.table()
        table["classes"].append("traceables-listtable")
        tgroup = nodes.tgroup(cols=len(columns), colwidths="auto")
        table += tgroup

        # Add column specifications.
        for attribute_name in columns:
            tgroup += nodes.colspec(colwidth=1)

        # Add heading row.
        thead = nodes.thead()
        tgroup += thead
        row = nodes.row()
        thead += row
        for attribute_name in columns:
            entry = nodes.entry()
            row += entry
            container = nodes.container()
            entry += container
            text = attribute_name.capitalize()
            inline = nodes.inline(text, text)
            container += inline

        # Add table body.
        tbody = nodes.tbody()
        tgroup += tbody
        for traceable in traceables:
            row = nodes.row()
            tbody += row

            for attribute_name in columns:
                entry = nodes.entry()
                row += entry
                if attribute_name == "tag":
                    inline = nodes.inline()
                    inline += traceable.make_reference_node(
                        app.builder, docname)
                elif attribute_name == "title":
                    text = traceable.title if traceable.has_title else ""
                    inline = nodes.inline(text, text)
                else:
                    text = traceable.attributes.get(attribute_name, "")
                    inline = nodes.inline(text, text)
                entry += inline

        return table

#        container = traceable_matrix_crosstable()
#        container += table
#        container["traceables-matrix"] = matrix
#        return container

# -----------------------------------------------------------------------------

class BulletListFormatter(ListFormatterBase):

    def format(self, app, docname, node, traceables, options):
        new_node = nodes.bullet_list()
        for traceable in traceables:
            item_node = nodes.list_item()
            new_node += item_node
            inline = nodes.inline()
            item_node += inline
            inline += traceable.make_reference_node(
                app.builder, docname)
            if traceable.has_title:
                title = u" " + traceable.title
                inline += nodes.inline(title, title)

        return new_node

# =============================================================================
# Directives

def list_of_attributes(argument):
    if not argument:
        return []
    return filter(None, (attribute.strip()
                         for attribute in argument.split(",")))

class TraceableListDirective(Directive):

    required_arguments = 0
    optional_arguments = 0
    option_spec = {
        "format": ListProcessor.directive_format_choice,
        "filter": directives.unchanged,
        "attributes": list_of_attributes,
    }

    def run(self):
        env = self.state.document.settings.env

        node = traceable_list()
        node["source"] = env.docname
        node["line"] = self.lineno
        for option in self.option_spec.keys():
            node["traceables-" + option] = self.options.get(option)

        return [node]

# =============================================================================
# Setup this extension part

ListProcessor.register_formatter("table", TableListFormatter(), default=True)
ListProcessor.register_formatter("list", BulletListFormatter())

def setup(app):
    app.add_node(traceable_list)
    app.add_directive("traceable-list", TraceableListDirective)
<|endoftext|>


<commit_before>import os
import re
import shlex
import string
import subprocess
from collections import defaultdict
from ConfigParser import ConfigParser
from datetime import date
from urlparse import urljoin

from allura.config import middleware
from allura.lib import rest_api

DEBUG=1
CP = ConfigParser()

re_ticket_ref = re.compile(r'\[#\d+\]')

CRED={}

def main():
    CP.read(os.path.join(os.environ['HOME'], '.forgepushrc'))
    engineer = option('re', 'engineer', 'Name of engineer pushing: ')
    api_key = option('re', 'api_key', 'Forge API Key:')
    secret_key = option('re', 'secret_key', 'Forge Secret Key:')
    CRED['api_key'] = api_key
    CRED['secret_key'] = secret_key
    text, tag = make_ticket_text(engineer)
    print '*** Create a ticket on SourceForge (https://sourceforge.net/p/allura/tickets/new/) with the following contents:'
    print '*** Summary: Production Push (R:%s, D:%s) - allura' % (
        tag, date.today().strftime('%Y%m%d'))
    print '---BEGIN---'
    print text
    print '---END---'
    raw_input("Verify that there are no new dependencies, or RPM's are build for all deps...")
    raw_input("Verify that a new sandbox builds starts without engr help...")
    raw_input('When this is done, create a SOG Trac ticket'
              ' (https://control.sog.geek.net/sog/trac/newticket?keywords=LIAISON) with the'
              ' same contents...')
    raw_input('Now link the two tickets...')
    newforge_num = raw_input('What is the newforge ticket number? ')
    command('git', 'tag', '-a', '-m', '[#%s] - Push to RE' % newforge_num, tag, 'master')
    command('git', 'push', 'origin', 'master')
    command('git', 'push', 'live', 'master')
    command('git', 'push', '--tags', 'origin')
    command('git', 'push', '--tags', 'live')
    raw_input('Now go to the sog-engr channel and let them know that %s is ready'
              ' for pushing (include the JIRA ticket #' % tag)
    raw_input('Make sure SOG restarted reactors and web services.')
    CP.write(open(os.path.join(os.environ['HOME'], '.forgepushrc'), 'w'))
    print "You're done!"

def make_ticket_text(engineer):
    tag_prefix = date.today().strftime('release_%Y%m%d')
    # get release tag
    existing_tags_today = command('git tag -l %s*' % tag_prefix)
    if existing_tags_today:
        tag = '%s.%.2d' % (tag_prefix, len(existing_tags_today))
    else:
        tag = tag_prefix
    last_release = command('git tag -l release_*')
    if last_release: last_release = last_release[-1]
    else: last_release = ''
    changes = command(
            'git', 'log', "--format=* %h %s", last_release.strip() + '..')
    assert changes, 'There were no commits found; maybe you forgot to merge dev->master?'
    changelog = ''.join(changes)
    changes = ''.join(format_changes(changes))
    print 'Changelog:\n%s' % changelog
    print 'Tickets:\n%s' % changes
    prelaunch = []
    postlaunch = []
    needs_reactor_setup = raw_input('Does this release require a reactor_setup? [n]')
    needs_flyway = raw_input('Does this release require a migration? [y]')
    if needs_reactor_setup[:1].lower() in ('y', '1'):
        postlaunch.append('* service reactor stop')
        postlaunch.append('* allurapaste reactor_setup /var/local/config/production.ini')
        postlaunch.append('* service reactor start')
    if needs_flyway[:1].lower() in ('', 'y', '1'):
        prelaunch.append('* dump the database in case we need to roll back')
        postlaunch.append('* allurapaste flyway --url mongo://sfn-mongo:27017/')
    postlaunch.append('* allurapaste ensure_index /var/local/config/production.ini')
    if postlaunch:
        postlaunch = [ 'From sfu-scmprocess-1 do the following:\n' ] + postlaunch
        postlaunch = '\n'.join(postlaunch)
    else:
        postlaunch = '-none-'
    if prelaunch:
        prelaunch = [ 'From sfn-mongo do the following:\n' ] + prelaunch
        prelaunch = '\n'.join(prelaunch)
    else:
        prelaunch = '-none-'
    return TICKET_TEMPLATE.substitute(locals()), tag

def format_changes(changes):
    ticket_groups = defaultdict(list)
    for change in changes:
        for m in re_ticket_ref.finditer(change):
            ticket_groups[m.group(0)].append(change)
    cli = rest_api.RestClient(
        base_uri='http://sourceforge.net', **CRED)
    for ref, commits in sorted(ticket_groups.iteritems()):
        ticket_num = ref[2:-1]
        ticket = cli.request(
            'GET',
            urljoin('/rest/p/allura/tickets/', str(ticket_num)) + '/')['ticket']
        verb = {
            'validation': 'Fix',
            'closed': 'Fix' }.get(ticket['status'], 'Address')
        yield ' * %s %s: %s\n' % (verb, ref, ticket['summary'])

def command(*args):
    if len(args) == 1 and isinstance(args[0], basestring):
        argv = shlex.split(args[0])
    else:
        argv = list(args)
    if DEBUG:
        print ' '.join(argv)
        raw_input('Press enter to run this command...')
    p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    rc = p.wait()
    if rc != 0:
        print 'Error running %s' % ' '.join(argv)
        import pdb; pdb.set_trace()
    return p.stdout.readlines()

def option(section, key, prompt=None):
    if not CP.has_section(section):
        CP.add_section(section)
    if CP.has_option(section, key):
        value = CP.get(section, key)
    else:
        value = raw_input(prompt or ('%s: ' % key))
        CP.set(section, key, value)
    return value

TICKET_TEMPLATE=string.Template('''{{{
#!push

(engr) Name of Engineer pushing: $engineer
(engr) Which code tree(s): allura
(engr) Is configtree to be pushed?: no
(engr) Which release/revision is going to be synced?: $tag
(engr) Itemized list of changes to be launched with sync:

$changes

Pre-launch dependencies:

$prelaunch

Post-launch dependencies:

$postlaunch

(engr) Approved for release (Dean/Dave/John): None
(sog) Outcome of sync:
}}}''')

if __name__ == '__main__':
    main()
<commit_msg>[#878] - Semi-gracefully handly the Forge being unavailable during a push<commit_after>import os
import re
import shlex
import string
import subprocess
from collections import defaultdict
from ConfigParser import ConfigParser
from datetime import date
from urlparse import urljoin

from allura.config import middleware
from allura.lib import rest_api

DEBUG=1
CP = ConfigParser()

re_ticket_ref = re.compile(r'\[#\d+\]')

CRED={}

def main():
    CP.read(os.path.join(os.environ['HOME'], '.forgepushrc'))
    engineer = option('re', 'engineer', 'Name of engineer pushing: ')
    api_key = option('re', 'api_key', 'Forge API Key:')
    secret_key = option('re', 'secret_key', 'Forge Secret Key:')
    CRED['api_key'] = api_key
    CRED['secret_key'] = secret_key
    text, tag = make_ticket_text(engineer)
    print '*** Create a ticket on SourceForge (https://sourceforge.net/p/allura/tickets/new/) with the following contents:'
    print '*** Summary: Production Push (R:%s, D:%s) - allura' % (
        tag, date.today().strftime('%Y%m%d'))
    print '---BEGIN---'
    print text
    print '---END---'
    raw_input("Verify that there are no new dependencies, or RPM's are build for all deps...")
    raw_input("Verify that a new sandbox builds starts without engr help...")
    raw_input('When this is done, create a SOG Trac ticket'
              ' (https://control.sog.geek.net/sog/trac/newticket?keywords=LIAISON) with the'
              ' same contents...')
    raw_input('Now link the two tickets...')
    newforge_num = raw_input('What is the newforge ticket number? ')
    command('git', 'tag', '-a', '-m', '[#%s] - Push to RE' % newforge_num, tag, 'master')
    command('git', 'push', 'origin', 'master')
    command('git', 'push', 'live', 'master')
    command('git', 'push', '--tags', 'origin')
    command('git', 'push', '--tags', 'live')
    raw_input('Now go to the sog-engr channel and let them know that %s is ready'
              ' for pushing (include the JIRA ticket #' % tag)
    raw_input('Make sure SOG restarted reactors and web services.')
    CP.write(open(os.path.join(os.environ['HOME'], '.forgepushrc'), 'w'))
    print "You're done!"

def make_ticket_text(engineer):
    tag_prefix = date.today().strftime('release_%Y%m%d')
    # get release tag
    existing_tags_today = command('git tag -l %s*' % tag_prefix)
    if existing_tags_today:
        tag = '%s.%.2d' % (tag_prefix, len(existing_tags_today))
    else:
        tag = tag_prefix
    last_release = command('git tag -l release_*')
    if last_release: last_release = last_release[-1]
    else: last_release = ''
    changes = command(
            'git', 'log', "--format=* %h %s", last_release.strip() + '..')
    assert changes, 'There were no commits found; maybe you forgot to merge dev->master?'
    changelog = ''.join(changes)
    changes = ''.join(format_changes(changes))
    print 'Changelog:\n%s' % changelog
    print 'Tickets:\n%s' % changes
    prelaunch = []
    postlaunch = []
    needs_reactor_setup = raw_input('Does this release require a reactor_setup? [n]')
    needs_flyway = raw_input('Does this release require a migration? [y]')
    if needs_reactor_setup[:1].lower() in ('y', '1'):
        postlaunch.append('* service reactor stop')
        postlaunch.append('* allurapaste reactor_setup /var/local/config/production.ini')
        postlaunch.append('* service reactor start')
    if needs_flyway[:1].lower() in ('', 'y', '1'):
        prelaunch.append('* dump the database in case we need to roll back')
        postlaunch.append('* allurapaste flyway --url mongo://sfn-mongo:27017/')
    postlaunch.append('* allurapaste ensure_index /var/local/config/production.ini')
    if postlaunch:
        postlaunch = [ 'From sfu-scmprocess-1 do the following:\n' ] + postlaunch
        postlaunch = '\n'.join(postlaunch)
    else:
        postlaunch = '-none-'
    if prelaunch:
        prelaunch = [ 'From sfn-mongo do the following:\n' ] + prelaunch
        prelaunch = '\n'.join(prelaunch)
    else:
        prelaunch = '-none-'
    return TICKET_TEMPLATE.substitute(locals()), tag

def format_changes(changes):
    ticket_groups = defaultdict(list)
    for change in changes:
        for m in re_ticket_ref.finditer(change):
            ticket_groups[m.group(0)].append(change)
    try:
        cli = rest_api.RestClient(
            base_uri='http://sourceforge.net', **CRED)
        for ref, commits in sorted(ticket_groups.iteritems()):
            ticket_num = ref[2:-1]
            ticket = cli.request(
                'GET',
                urljoin('/rest/p/allura/tickets/', str(ticket_num)) + '/')['ticket']
            verb = {
                'validation': 'Fix',
                'closed': 'Fix' }.get(ticket['status'], 'Address')
            yield ' * %s %s: %s\n' % (verb, ref, ticket['summary'])
    except:
        print '*** ERROR CONTACTING FORGE FOR TICKET SUMMARIES ***'
        for ci in changes:
            yield ci

def command(*args):
    if len(args) == 1 and isinstance(args[0], basestring):
        argv = shlex.split(args[0])
    else:
        argv = list(args)
    if DEBUG:
        print ' '.join(argv)
        raw_input('Press enter to run this command...')
    p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    rc = p.wait()
    if rc != 0:
        print 'Error running %s' % ' '.join(argv)
        import pdb; pdb.set_trace()
    return p.stdout.readlines()

def option(section, key, prompt=None):
    if not CP.has_section(section):
        CP.add_section(section)
    if CP.has_option(section, key):
        value = CP.get(section, key)
    else:
        value = raw_input(prompt or ('%s: ' % key))
        CP.set(section, key, value)
    return value

TICKET_TEMPLATE=string.Template('''{{{
#!push

(engr) Name of Engineer pushing: $engineer
(engr) Which code tree(s): allura
(engr) Is configtree to be pushed?: no
(engr) Which release/revision is going to be synced?: $tag
(engr) Itemized list of changes to be launched with sync:

$changes

Pre-launch dependencies:

$prelaunch

Post-launch dependencies:

$postlaunch

(engr) Approved for release (Dean/Dave/John): None
(sog) Outcome of sync:
}}}''')

if __name__ == '__main__':
    main()
<|endoftext|>