<commit_before>#!/usr/bin/env python
import json
import names # Use 'pip install names'
import random
import pymongo

'''
Updates the DB for a dev instance, making the subject subdocuments rich and interesting.
'''

# Connect to the DB
db = pymongo.MongoClient('mongodb://docker.local.flywheel.io:9001/scitran').get_default_database()

# Update DB: Set group names to use the group '_id' (Should be done by the UI)
for g in db.groups.find({},['_id']):
    group_name = g['_id'].capitalize() + ' Group'
    db.groups.update_one({'_id': g['_id']},{'$set':{'name': group_name}})

# Configure tags for the groups
tags = ['Good Subject', 'Bad Subject', 'To Process', 'Processed', 'To Analyze', 'Analyzed',  'Control', 'Patient', 'NSF Grant', 'NIH Grant', 'NIMH Grant']
tags1 = tags[:2]
tags2 = tags[2:6]
tags3 = tags[6:8]
tags4 = tags[8:]
groups = list(db.groups.find())
for g in groups:
    db.groups.update_one({'_id':g['_id']}, {'$set':{'tags':tags}})

# Get a list of the subjects/sessions
subjects = list(db.sessions.find({}, ['subject']))

# Get unique subject codes (some have multiple sessions)
codes = set([s['subject']['code'] for s in subjects])
total_sessions = len(codes)

# Update the subject subdocument only once per unique subject code
idx = 0
for c in codes:
    # Set half to male, rest female, using gender specific names
    if idx <= total_sessions * 0.5:
        first_name = names.get_first_name(gender='male')
        sex = 'male'
    else:
        first_name = names.get_first_name(gender='female')
        sex = 'female'
    last_name = names.get_last_name()

    # Set the age (in seconds) from a normal distrubition
    age = int(random.normalvariate(35, 10) * 31536000)

    # Build some fun subject metadata for each subject
    metadata = {}
    metadata['IQ'] = int(random.normalvariate(100, 15))
    metadata['Education'] = int(random.normalvariate(14, 2))
    ses = ['High', 'Middle', 'Low']
    metadata['SES'] = ses[random.randrange(0, len(ses))]
    metadata['TOWRE'] = int(random.normalvariate(100, 15))
    metadata['GORT'] = int(random.normalvariate(100, 15))
    metadata['SAT'] = int(random.normalvariate(1490, 100))
    metadata['Verbal_Reasoning'] = {}
    metadata['Verbal_Reasoning']['Score_1'] = random.randrange(130, 170)
    metadata['Verbal_Reasoning']['Score_2'] = random.randrange(130, 170)
    metadata['Quantitative_Reasoning'] = {}
    metadata['Quantitative_Reasoning']['Score_1'] = random.randrange(130, 170)
    metadata['Quantitative_Reasoning']['Score_2'] = random.randrange(130, 170)
    metadata['Analytical_Writing'] = {}
    metadata['Analytical_Writing']['Score_1'] = random.randrange(2, 6)
    metadata['Analytical_Writing']['Score_2'] = random.randrange(2, 6)

    db.sessions.update_many({'subject.code': c},{'$set':{'subject.lastname': last_name}})
    db.sessions.update_many({'subject.code': c},{'$set':{'subject.firstname': first_name}})
    db.sessions.update_many({'subject.code': c},{'$set':{'subject.sex': sex}})
    db.sessions.update_many({'subject.code': c},{'$set':{'subject.age': age}})
    db.sessions.update_many({'subject.code': c},{'$set':{'subject.metadata': metadata}})

    # Increment the index
    idx +=1
<commit_msg>random assignment of session tags from group tags.<commit_after>#!/usr/bin/env python
import json
import names # Use 'pip install names'
import random
import pymongo

'''
Updates the DB for a dev instance, making the subject subdocuments rich and interesting.
'''

# Connect to the DB
db = pymongo.MongoClient('mongodb://docker.local.flywheel.io:9001/scitran').get_default_database()

# Update DB: Set group names to use the group '_id' (Should be done by the UI)
for g in db.groups.find({},['_id']):
    group_name = g['_id'].capitalize() + ' Group'
    db.groups.update_one({'_id': g['_id']},{'$set':{'name': group_name}})

# Configure tags for the groups
tags = ['Good Subject', 'Bad Subject', 'To Process', 'Processed', 'To Analyze', 'Analyzed',  'Control', 'Patient', 'NSF Grant', 'NIH Grant', 'NIMH Grant']
groups = list(db.groups.find())
for g in groups:
    db.groups.update_one({'_id':g['_id']}, {'$set':{'tags':tags}})

# Randomly assign tags from each group to each session
tags1 = tags[:2]
tags2 = tags[2:6]
tags3 = tags[6:8]
tags4 = tags[8:]
sessions = list(db.sessions.find())
for ses in sessions:
    tag_set = []
    tag_set.append(random.choice(tags1))
    tag_set.append(random.choice(tags2))
    tag_set.append(random.choice(tags3))
    tag_set.append(random.choice(tags4))
    db.sessions.update_one({'label': ses['label']}, {'$set':{'tags':tag_set}})

# Get a list of the subjects/sessions
subjects = list(db.sessions.find({}, ['subject']))

# Get unique subject codes (some have multiple sessions)
codes = set([s['subject']['code'] for s in subjects])
total_sessions = len(codes)

# Update the subject subdocument only once per unique subject code
idx = 0
for c in codes:
    # Set half to male, rest female, using gender specific names
    if idx <= total_sessions * 0.5:
        first_name = names.get_first_name(gender='male')
        sex = 'male'
    else:
        first_name = names.get_first_name(gender='female')
        sex = 'female'
    last_name = names.get_last_name()

    # Set the age (in seconds) from a normal distrubition
    age = int(random.normalvariate(35, 10) * 31536000)

    # Build some fun subject metadata for each subject
    metadata = {}
    metadata['IQ'] = int(random.normalvariate(100, 15))
    metadata['Education'] = int(random.normalvariate(14, 2))
    ses = ['High', 'Middle', 'Low']
    metadata['SES'] = ses[random.randrange(0, len(ses))]
    metadata['TOWRE'] = int(random.normalvariate(100, 15))
    metadata['GORT'] = int(random.normalvariate(100, 15))
    metadata['SAT'] = int(random.normalvariate(1490, 100))
    metadata['Verbal_Reasoning'] = {}
    metadata['Verbal_Reasoning']['Score_1'] = random.randrange(130, 170)
    metadata['Verbal_Reasoning']['Score_2'] = random.randrange(130, 170)
    metadata['Quantitative_Reasoning'] = {}
    metadata['Quantitative_Reasoning']['Score_1'] = random.randrange(130, 170)
    metadata['Quantitative_Reasoning']['Score_2'] = random.randrange(130, 170)
    metadata['Analytical_Writing'] = {}
    metadata['Analytical_Writing']['Score_1'] = random.randrange(2, 6)
    metadata['Analytical_Writing']['Score_2'] = random.randrange(2, 6)

    db.sessions.update_many({'subject.code': c},{'$set':{'subject.lastname': last_name}})
    db.sessions.update_many({'subject.code': c},{'$set':{'subject.firstname': first_name}})
    db.sessions.update_many({'subject.code': c},{'$set':{'subject.sex': sex}})
    db.sessions.update_many({'subject.code': c},{'$set':{'subject.age': age}})
    db.sessions.update_many({'subject.code': c},{'$set':{'subject.metadata': metadata}})

    # Increment the index
    idx +=1
<|endoftext|>


<commit_before>from chainer.functions.connection import convolution_2d
from chainer import initializers
from chainer import link
from chainer.utils import argument
from chainer import variable

class Convolution2D(link.Link):

    """__init__(self, in_channels, out_channels, ksize=None, stride=1, pad=0, nobias=False, initialW=None, initial_bias=None, *, dilate=1, groups=1)

    Two-dimensional convolutional layer.

    This link wraps the :func:`~chainer.functions.convolution_2d` function and
    holds the filter weight and bias vector as parameters.

    The output of this function can be non-deterministic when it uses cuDNN.
    If ``chainer.configuration.config.deterministic`` is ``True`` and
    cuDNN version is >= v3, it forces cuDNN to use a deterministic algorithm.

    Convolution links can use a feature of cuDNN called autotuning, which
    selects the most efficient CNN algorithm for images of fixed-size,
    can provide a significant performance boost for fixed neural nets.
    To enable, set `chainer.using_config('autotune', True)`

    .. warning::

        ``deterministic`` argument is not supported anymore since v2.
        Instead, use ``chainer.using_config('cudnn_deterministic', value``
        (value is either ``True`` or ``False``).
        See :func:`chainer.using_config`.

    Args:
        in_channels (int or None): Number of channels of input arrays.
            If ``None``, parameter initialization will be deferred until the
            first forward data pass at which time the size will be determined.
        out_channels (int): Number of channels of output arrays.
        ksize (int or pair of ints): Size of filters (a.k.a. kernels).
            ``ksize=k`` and ``ksize=(k, k)`` are equivalent.
        stride (int or pair of ints): Stride of filter applications.
            ``stride=s`` and ``stride=(s, s)`` are equivalent.
        pad (int or pair of ints): Spatial padding width for input arrays.
            ``pad=p`` and ``pad=(p, p)`` are equivalent.
        nobias (bool): If ``True``, then this link does not use the bias term.
        initialW (:ref:`initializer <initializer>`): Initializer to
            initialize the weight. When it is :class:`numpy.ndarray`,
            its ``ndim`` should be 4.
        initial_bias (:ref:`initializer <initializer>`): Initializer to
            initialize the bias. If ``None``, the bias will be initialized to
            zero. When it is :class:`numpy.ndarray`, its ``ndim`` should be 1.
        dilate (int or pair of ints):
            Dilation factor of filter applications.
            ``dilate=d`` and ``dilate=(d, d)`` are equivalent.
        groups (:class:`int`): Number of groups of channels. If the number
            is greater than 1, input tensor :math:`W` is divided into some
            blocks by this value channel-wise. For each tensor blocks,
            convolution operation will be executed independently. Input channel
            size ``in_channels`` and output channel size ``out_channels`` must
            be exactly divisible by this value.

    .. seealso::
       See :func:`chainer.functions.convolution_2d` for the definition of
       two-dimensional convolution.

    Attributes:
        W (~chainer.Variable): Weight parameter.
        b (~chainer.Variable): Bias parameter.

    .. admonition:: Example

        There are several ways to make a Convolution2D link.

        Let an input vector ``x`` be:

        >>> x = np.arange(1 * 3 * 10 * 10, dtype=np.float32).reshape(1, 3, 10, 10)

        1. Give the first three arguments explicitly:

            >>> l = L.Convolution2D(3, 7, 5)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

        2. Omit ``in_channels`` or fill it with ``None``:

            The below two cases are the same.

            >>> l = L.Convolution2D(7, 5)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

            >>> l = L.Convolution2D(None, 7, 5)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

            When you omit the first argument, you need to specify the other
            subsequent arguments from ``stride`` as keyword auguments. So the
            below two cases are the same.

            >>> l = L.Convolution2D(7, 5, stride=1, pad=0)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

            >>> l = L.Convolution2D(None, 7, 5, 1, 0)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

    """  # NOQA

    def __init__(self, in_channels, out_channels, ksize=None, stride=1, pad=0,
                 nobias=False, initialW=None, initial_bias=None, **kwargs):
        super(Convolution2D, self).__init__()

        dilate, groups = argument.parse_kwargs(
            kwargs, ('dilate', 1), ('groups', 1),
            deterministic="deterministic argument is not supported anymore. "
            "Use chainer.using_config('cudnn_deterministic', value) "
            "context where value is either `True` or `False`.")

        if ksize is None:
            out_channels, ksize, in_channels = in_channels, out_channels, None

        self.ksize = ksize
        self.stride = _pair(stride)
        self.pad = _pair(pad)
        self.dilate = _pair(dilate)
        self.out_channels = out_channels
        self.groups = int(groups)

        with self.init_scope():
            W_initializer = initializers._get_initializer(initialW)
            self.W = variable.Parameter(W_initializer)
            if in_channels is not None:
                self._initialize_params(in_channels)

            if nobias:
                self.b = None
            else:
                if initial_bias is None:
                    initial_bias = 0
                bias_initializer = initializers._get_initializer(initial_bias)
                self.b = variable.Parameter(bias_initializer, out_channels)

    def _initialize_params(self, in_channels):
        kh, kw = _pair(self.ksize)
        if self.out_channels % self.groups != 0:
            raise ValueError('the number of output channels must be'
                             'divisible by the number of groups')
        if in_channels % self.groups != 0:
            raise ValueError('the number of input channels must be'
                             'divisible by the number of groups')
        W_shape = (self.out_channels, int(in_channels / self.groups), kh, kw)
        self.W.initialize(W_shape)

    def forward(self, x):
        """Applies the convolution layer.

        Args:
            x (~chainer.Variable): Input image.

        Returns:
            ~chainer.Variable: Output of the convolution.

        """
        if self.W.data is None:
            self._initialize_params(x.shape[1])
        return convolution_2d.convolution_2d(
            x, self.W, self.b, self.stride, self.pad, dilate=self.dilate,
            groups=self.groups)

def _pair(x):
    if hasattr(x, '__getitem__'):
        return x
    return x, x
<commit_msg>add space between words<commit_after>from chainer.functions.connection import convolution_2d
from chainer import initializers
from chainer import link
from chainer.utils import argument
from chainer import variable

class Convolution2D(link.Link):

    """__init__(self, in_channels, out_channels, ksize=None, stride=1, pad=0, nobias=False, initialW=None, initial_bias=None, *, dilate=1, groups=1)

    Two-dimensional convolutional layer.

    This link wraps the :func:`~chainer.functions.convolution_2d` function and
    holds the filter weight and bias vector as parameters.

    The output of this function can be non-deterministic when it uses cuDNN.
    If ``chainer.configuration.config.deterministic`` is ``True`` and
    cuDNN version is >= v3, it forces cuDNN to use a deterministic algorithm.

    Convolution links can use a feature of cuDNN called autotuning, which
    selects the most efficient CNN algorithm for images of fixed-size,
    can provide a significant performance boost for fixed neural nets.
    To enable, set `chainer.using_config('autotune', True)`

    .. warning::

        ``deterministic`` argument is not supported anymore since v2.
        Instead, use ``chainer.using_config('cudnn_deterministic', value``
        (value is either ``True`` or ``False``).
        See :func:`chainer.using_config`.

    Args:
        in_channels (int or None): Number of channels of input arrays.
            If ``None``, parameter initialization will be deferred until the
            first forward data pass at which time the size will be determined.
        out_channels (int): Number of channels of output arrays.
        ksize (int or pair of ints): Size of filters (a.k.a. kernels).
            ``ksize=k`` and ``ksize=(k, k)`` are equivalent.
        stride (int or pair of ints): Stride of filter applications.
            ``stride=s`` and ``stride=(s, s)`` are equivalent.
        pad (int or pair of ints): Spatial padding width for input arrays.
            ``pad=p`` and ``pad=(p, p)`` are equivalent.
        nobias (bool): If ``True``, then this link does not use the bias term.
        initialW (:ref:`initializer <initializer>`): Initializer to
            initialize the weight. When it is :class:`numpy.ndarray`,
            its ``ndim`` should be 4.
        initial_bias (:ref:`initializer <initializer>`): Initializer to
            initialize the bias. If ``None``, the bias will be initialized to
            zero. When it is :class:`numpy.ndarray`, its ``ndim`` should be 1.
        dilate (int or pair of ints):
            Dilation factor of filter applications.
            ``dilate=d`` and ``dilate=(d, d)`` are equivalent.
        groups (:class:`int`): Number of groups of channels. If the number
            is greater than 1, input tensor :math:`W` is divided into some
            blocks by this value channel-wise. For each tensor blocks,
            convolution operation will be executed independently. Input channel
            size ``in_channels`` and output channel size ``out_channels`` must
            be exactly divisible by this value.

    .. seealso::
       See :func:`chainer.functions.convolution_2d` for the definition of
       two-dimensional convolution.

    Attributes:
        W (~chainer.Variable): Weight parameter.
        b (~chainer.Variable): Bias parameter.

    .. admonition:: Example

        There are several ways to make a Convolution2D link.

        Let an input vector ``x`` be:

        >>> x = np.arange(1 * 3 * 10 * 10, dtype=np.float32).reshape(1, 3, 10, 10)

        1. Give the first three arguments explicitly:

            >>> l = L.Convolution2D(3, 7, 5)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

        2. Omit ``in_channels`` or fill it with ``None``:

            The below two cases are the same.

            >>> l = L.Convolution2D(7, 5)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

            >>> l = L.Convolution2D(None, 7, 5)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

            When you omit the first argument, you need to specify the other
            subsequent arguments from ``stride`` as keyword auguments. So the
            below two cases are the same.

            >>> l = L.Convolution2D(7, 5, stride=1, pad=0)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

            >>> l = L.Convolution2D(None, 7, 5, 1, 0)
            >>> y = l(x)
            >>> y.shape
            (1, 7, 6, 6)

    """  # NOQA

    def __init__(self, in_channels, out_channels, ksize=None, stride=1, pad=0,
                 nobias=False, initialW=None, initial_bias=None, **kwargs):
        super(Convolution2D, self).__init__()

        dilate, groups = argument.parse_kwargs(
            kwargs, ('dilate', 1), ('groups', 1),
            deterministic="deterministic argument is not supported anymore. "
            "Use chainer.using_config('cudnn_deterministic', value) "
            "context where value is either `True` or `False`.")

        if ksize is None:
            out_channels, ksize, in_channels = in_channels, out_channels, None

        self.ksize = ksize
        self.stride = _pair(stride)
        self.pad = _pair(pad)
        self.dilate = _pair(dilate)
        self.out_channels = out_channels
        self.groups = int(groups)

        with self.init_scope():
            W_initializer = initializers._get_initializer(initialW)
            self.W = variable.Parameter(W_initializer)
            if in_channels is not None:
                self._initialize_params(in_channels)

            if nobias:
                self.b = None
            else:
                if initial_bias is None:
                    initial_bias = 0
                bias_initializer = initializers._get_initializer(initial_bias)
                self.b = variable.Parameter(bias_initializer, out_channels)

    def _initialize_params(self, in_channels):
        kh, kw = _pair(self.ksize)
        if self.out_channels % self.groups != 0:
            raise ValueError('the number of output channels must be'
                             ' divisible by the number of groups')
        if in_channels % self.groups != 0:
            raise ValueError('the number of input channels must be'
                             ' divisible by the number of groups')
        W_shape = (self.out_channels, int(in_channels / self.groups), kh, kw)
        self.W.initialize(W_shape)

    def forward(self, x):
        """Applies the convolution layer.

        Args:
            x (~chainer.Variable): Input image.

        Returns:
            ~chainer.Variable: Output of the convolution.

        """
        if self.W.data is None:
            self._initialize_params(x.shape[1])
        return convolution_2d.convolution_2d(
            x, self.W, self.b, self.stride, self.pad, dilate=self.dilate,
            groups=self.groups)

def _pair(x):
    if hasattr(x, '__getitem__'):
        return x
    return x, x
<|endoftext|>


<commit_before>from chainer import reporter
import chainer.serializer as serializer_module
from chainer.training import extension
import chainer.training.trigger as trigger_module
import json
import numpy as np
from os import path
import six

try:
    import matplotlib

    matplotlib.use('Agg')
    from matplotlib import pyplot as plot

    _available = True

except ImportError:
    _available = False

def _check_available():
    if not _available:
        msg = '''matplotlib is not installed on your environment.
Please install matplotlib to plot figure.

  $ pip install matplotlib'''
        raise RuntimeError(msg)

class PlotReport(extension.Extension):

    def __init__(self, y_keys, x_key='iteration', trigger=(1, 'epoch'),
                 postprocess=None, file_name='graph.png'):
        """Trainer extension to output graph.

        This extension accumulates the observations of the trainer to
        :class:`~chainer.DictSummary` at a regular interval specified by a
        supplied trigger, and plot a graph with using them.

        There are two triggers to handle this extension. One is the trigger
        to invoke this extension, which is used to handle the timing of
        accumulating the results. It is set to ``1, 'iteration'`` by default.
        The other is the trigger to determine when to emit the result. When
        this trigger returns True, this extension appends the summary of
        accumulated values to the list of past summaries, and writes the list
        to the log file. Then, this extension makes a new fresh summary
        object which is used until the next time that the trigger fires.

        It also adds ``'epoch'`` and ``'iteration'`` entries to each result
        dictionary, which are the epoch and iteration counts at the output.

        Args:
            y_keys (iterable of strs): Keys of values regarded as y. If this is
                None, nothing is output to the graph.
            x_key (str): Keys of values regarded as x.
            trigger: Trigger that decides when to aggregate the result and
                output the values. This is distinct from the trigger of this
                extension itself. If it is a tuple in the form
                ``<int>, 'epoch'`` or ``<int>, 'iteration'``, it is passed to
                :class:`IntervalTrigger`.
            postprocess: Callback to postprocess the result dictionaries.
                Figure object, Axes object, and all plot data is passed to this
                callback on the output. This callback can modify the figure.
            file_name (str): Name of the figure file under the output
                directory. It can be a format string.

        """

        _check_available()

        self._x_key = x_key
        if isinstance(y_keys, str):
            y_keys = (y_keys,)

        self._y_keys = y_keys
        self._trigger = trigger_module.get_trigger(trigger)
        self._file_name = file_name
        self._postprocess = postprocess
        self._init_summary()
        self._data = {k: [] for k in y_keys}

    def __call__(self, trainer):
        keys = self._y_keys
        observation = trainer.observation
        summary = self._summary

        if keys is None:
            summary.add(observation)
        else:
            summary.add({k: observation[k] for k in keys if k in observation})

        if self._trigger(trainer):
            stats = self._summary.compute_mean()
            stats_cpu = {}
            for name, value in six.iteritems(stats):
                stats_cpu[name] = float(value)  # copy to CPU

            updater = trainer.updater
            stats_cpu['epoch'] = updater.epoch
            stats_cpu['iteration'] = updater.iteration
            x = stats_cpu[self._x_key]
            data = self._data

            for k in keys:
                if k in stats_cpu:
                    data[k].append((x, stats_cpu[k]))

            f = plot.figure()
            a = f.add_subplot(111)

            flag_any_item = False
            for k in keys:
                xy = data[k]
                if len(xy) == 0:
                    continue

                flag_any_item = True
                xy = np.array(xy)
                a.plot(xy[:, 0], xy[:, 1], label=k)

            if flag_any_item:
                if self._postprocess is not None:
                    self._postprocess(f, a, summary)

                f.savefig(path.join(trainer.out, self._file_name))

            plot.close()
            self._init_summary()

    def serialize(self, serializer):
        if isinstance(serializer, serializer_module.Serializer):
            serializer('_graph_{}'.format(self._file_name),
                       json.dumps(self._data))

        else:
            self._data = json.loads(
                serializer('_graph_{}'.format(self._file_name), ''))

    def _init_summary(self):
        self._summary = reporter.DictSummary()
<commit_msg>fix for coding guidelines<commit_after>import json
import numpy
from os import path

import six

from chainer import reporter
import chainer.serializer as serializer_module
from chainer.training import extension
import chainer.training.trigger as trigger_module

try:
    import matplotlib

    matplotlib.use('Agg')
    from matplotlib import pyplot as plot

    _available = True

except ImportError:
    _available = False

def _check_available():
    if not _available:
        msg = '''matplotlib is not installed on your environment.
Please install matplotlib to plot figure.

  $ pip install matplotlib'''
        raise RuntimeError(msg)

class PlotReport(extension.Extension):
    """Trainer extension to output plots.

    This extension accumulates the observations of the trainer to
    :class:`~chainer.DictSummary` at a regular interval specified by a
    supplied trigger, and plot a graph with using them.

    There are two triggers to handle this extension. One is the trigger
    to invoke this extension, which is used to handle the timing of
    accumulating the results. It is set to ``1, 'iteration'`` by default.
    The other is the trigger to determine when to emit the result. When
    this trigger returns True, this extension appends the summary of
    accumulated values to the list of past summaries, and writes the list
    to the log file. Then, this extension makes a new fresh summary
    object which is used until the next time that the trigger fires.

    It also adds ``'epoch'`` and ``'iteration'`` entries to each result
    dictionary, which are the epoch and iteration counts at the output.

    Args:
        y_keys (iterable of strs): Keys of values regarded as y. If this is
            None, nothing is output to the graph.
        x_key (str): Keys of values regarded as x. The default value is
            'iteration'
        trigger: Trigger that decides when to aggregate the result and
            output the values. This is distinct from the trigger of this
            extension itself. If it is a tuple in the form
            ``<int>, 'epoch'`` or ``<int>, 'iteration'``, it is passed to
            :class:`IntervalTrigger`.
        postprocess: Callback to postprocess the result dictionaries.
            Figure object, Axes object, and all plot data is passed to this
            callback on the output. This callback can modify the figure.
        file_name (str): Name of the figure file under the output
            directory. It can be a format string.

    """

    def __init__(self, y_keys, x_key='iteration', trigger=(1, 'epoch'),
                 postprocess=None, file_name='graph.png'):

        _check_available()

        self._x_key = x_key
        if isinstance(y_keys, str):
            y_keys = (y_keys,)

        self._y_keys = y_keys
        self._trigger = trigger_module.get_trigger(trigger)
        self._file_name = file_name
        self._postprocess = postprocess
        self._init_summary()
        self._data = {k: [] for k in y_keys}

    def __call__(self, trainer):
        keys = self._y_keys
        observation = trainer.observation
        summary = self._summary

        if keys is None:
            summary.add(observation)
        else:
            summary.add({k: observation[k] for k in keys if k in observation})

        if self._trigger(trainer):
            stats = self._summary.compute_mean()
            stats_cpu = {}
            for name, value in six.iteritems(stats):
                stats_cpu[name] = float(value)  # copy to CPU

            updater = trainer.updater
            stats_cpu['epoch'] = updater.epoch
            stats_cpu['iteration'] = updater.iteration
            x = stats_cpu[self._x_key]
            data = self._data

            for k in keys:
                if k in stats_cpu:
                    data[k].append((x, stats_cpu[k]))

            f = plot.figure()
            a = f.add_subplot(111)

            flag_any_item = False
            for k in keys:
                xy = data[k]
                if len(xy) == 0:
                    continue

                flag_any_item = True
                xy = numpy.array(xy)
                a.plot(xy[:, 0], xy[:, 1], label=k)

            if flag_any_item:
                if self._postprocess is not None:
                    self._postprocess(f, a, summary)
                f.savefig(path.join(trainer.out, self._file_name))

            plot.close()
            self._init_summary()

    def serialize(self, serializer):
        if isinstance(serializer, serializer_module.Serializer):
            serializer('_plot_{}'.format(self._file_name),
                       json.dumps(self._data))

        else:
            self._data = json.loads(
                serializer('_plot_{}'.format(self._file_name), ''))

    def _init_summary(self):
        self._summary = reporter.DictSummary()
<|endoftext|>


<commit_before># Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import argparse
import collections
import importlib
import os.path
import logging.config

import babel.dates
import babel.numbers
import babel.support

import guard

import jinja2

import redis as redispy

import sqlalchemy
import yaml

from raven import Client
from raven.middleware import Sentry
from werkzeug.exceptions import HTTPException
from werkzeug.wsgi import SharedDataMiddleware, responder

import warehouse
import warehouse.cli

from warehouse import urls
from warehouse import db
from warehouse.http import Request
from warehouse.middleware import PoweredBy
from warehouse.packaging import helpers as packaging_helpers
from warehouse.packaging.search import ProjectMapping
from warehouse.search.indexes import Index
from warehouse.utils import AttributeDict, merge_dict, convert_to_attr_dict

# Register the SQLAlchemy tables by importing them
import warehouse.accounts.tables
import warehouse.download_statistics.tables
import warehouse.packaging.tables

# Get the various models
import warehouse.accounts.models
import warehouse.packaging.models

class Warehouse(object):

    model_classes = {
        "accounts": warehouse.accounts.models.Model,
        "packaging": warehouse.packaging.models.Model,
    }

    def __init__(self, config, engine=None, redis=None):
        self.config = convert_to_attr_dict(config)

        self.metadata = db.metadata

        # Connect to the database
        if engine is None and self.config.get("database", {}).get("url"):
            engine = sqlalchemy.create_engine(self.config.database.url)
        self.engine = engine

        # Create our redis connection
        if redis is None and self.config.get("redis", {}).get("url"):
            redis = redispy.StrictRedis.from_url(self.config.redis.url)
        self.redis = redis

        # Create our Store instance and associate our store modules with it
        self.models = AttributeDict()
        for name, klass in self.model_classes.items():
            self.models[name] = klass(
                self,
                self.metadata,
                self.engine,
                self.redis,
            )

        # Create our Search Index instance and associate our mappings with it
        self.search = Index(self.models, self.config.search)
        self.search.register(ProjectMapping)

        # Set up our URL routing
        self.urls = urls.urls

        # Initialize our Translations engine
        self.trans = babel.support.NullTranslations()

        # Setup our Jinja2 Environment
        self.templates = jinja2.Environment(
            autoescape=True,
            auto_reload=self.config.debug,
            extensions=[
                "jinja2.ext.i18n",
            ],
            loader=jinja2.PackageLoader("warehouse"),
        )

        # Install Babel
        self.templates.filters.update({
            "package_type_display": packaging_helpers.package_type_display,
            "format_number": babel.numbers.format_number,
            "format_decimal": babel.numbers.format_decimal,
            "format_percent": babel.numbers.format_percent,
            "format_date": babel.dates.format_date,
            "format_datetime": babel.dates.format_datetime,
            "format_time": babel.dates.format_time,
        })

        # Install our translations
        self.templates.install_gettext_translations(self.trans, newstyle=True)

        # Add our Powered By Middleware
        self.wsgi_app = PoweredBy(self.wsgi_app, "Warehouse {} ({})".format(
            warehouse.__version__,
            warehouse.__build__,
        ))

        # Add our Content Security Policy Middleware
        if not self.config.theme_debug:
            self.wsgi_app = guard.ContentSecurityPolicy(
                self.wsgi_app,
                self.config.security.csp,
            )

        if "sentry" in self.config:
            self.wsgi_app = Sentry(self.wsgi_app, Client(**self.config.sentry))

        # Serve the static files that are packaged as part of Warehouse
        self.wsgi_app = SharedDataMiddleware(
            self.wsgi_app,
            {
                "/static/": os.path.abspath(
                    os.path.join(
                        os.path.dirname(warehouse.__file__),
                        "static",
                        "compiled",
                    ),
                ),
            },
        )

        # configure logging
        logging.config.dictConfig(self.config.logging)

    def __call__(self, environ, start_response):
        """
        Shortcut for :attr:`wsgi_app`.
        """
        return self.wsgi_app(environ, start_response)

    @classmethod
    def from_yaml(cls, *paths, **kwargs):
        # Pull out other keyword arguments
        override = kwargs.pop("override", None)

        default = os.path.abspath(os.path.join(
            os.path.dirname(warehouse.__file__),
            "config.yml",
        ))

        paths = [default] + list(paths)

        config = {}
        for path in paths:
            with open(path) as configfile:
                # Use no cover to work around a coverage bug
                config = merge_dict(  # pragma: no cover
                    config,
                    yaml.safe_load(configfile)
                )

        if override:
            config = merge_dict(config, override)

        return cls(config=config, **kwargs)

    @classmethod
    def from_cli(cls, argv):
        def _generate_parser(parser, commands):
            # Generate our commands
            subparsers = parser.add_subparsers()
            for name, command in commands.items():
                cmd_parser = subparsers.add_parser(name)

                if hasattr(command, "create_parser"):
                    command.create_parser(cmd_parser)

                if isinstance(command, collections.Mapping):
                    _generate_parser(cmd_parser, command)
                else:
                    cmd_parser.set_defaults(_cmd=command)

        parser = argparse.ArgumentParser(prog="warehouse")
        parser.add_argument("-c", "--config", action="append", dest="_configs")

        _generate_parser(parser, warehouse.cli.__commands__)

        args = parser.parse_args(argv)

        configs = args._configs if args._configs is not None else []
        app = cls.from_yaml(*configs)

        return args._cmd(
            app,
            *args._get_args(),
            **{k: v for k, v in args._get_kwargs() if not k.startswith("_")}
        )

    @responder
    def wsgi_app(self, environ, start_response):
        """
        The actual WSGI application.  This is not implemented in
        `__call__` so that middlewares can be applied without losing a
        reference to the class.  So instead of doing this::

            app = MyMiddleware(app)

        It's a better idea to do this instead::

            app.wsgi_app = MyMiddleware(app.wsgi_app)

        Then you still have the original application object around and
        can continue to call methods on it.

        :param environ: a WSGI environment
        :param start_response: a callable accepting a status code,
                               a list of headers and an optional
                               exception context to start the response
        """
        try:
            # Figure out what endpoint to call
            urls = self.urls.bind_to_environ(environ)
            endpoint, kwargs = urls.match()

            # Load our view function
            modname, viewname = endpoint.rsplit(".", 1)
            module = importlib.import_module(modname)
            view = getattr(module, viewname)

            # Create our request object
            request = Request(environ)
            request.url_adapter = urls

            # Dispatch to our view
            return view(self, request, **kwargs)
        except HTTPException as exc:
            return exc
<commit_msg>Always use our CSP Policy<commit_after># Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import argparse
import collections
import importlib
import os.path
import logging.config

import babel.dates
import babel.numbers
import babel.support

import guard

import jinja2

import redis as redispy

import sqlalchemy
import yaml

from raven import Client
from raven.middleware import Sentry
from werkzeug.exceptions import HTTPException
from werkzeug.wsgi import SharedDataMiddleware, responder

import warehouse
import warehouse.cli

from warehouse import urls
from warehouse import db
from warehouse.http import Request
from warehouse.middleware import PoweredBy
from warehouse.packaging import helpers as packaging_helpers
from warehouse.packaging.search import ProjectMapping
from warehouse.search.indexes import Index
from warehouse.utils import AttributeDict, merge_dict, convert_to_attr_dict

# Register the SQLAlchemy tables by importing them
import warehouse.accounts.tables
import warehouse.download_statistics.tables
import warehouse.packaging.tables

# Get the various models
import warehouse.accounts.models
import warehouse.packaging.models

class Warehouse(object):

    model_classes = {
        "accounts": warehouse.accounts.models.Model,
        "packaging": warehouse.packaging.models.Model,
    }

    def __init__(self, config, engine=None, redis=None):
        self.config = convert_to_attr_dict(config)

        self.metadata = db.metadata

        # Connect to the database
        if engine is None and self.config.get("database", {}).get("url"):
            engine = sqlalchemy.create_engine(self.config.database.url)
        self.engine = engine

        # Create our redis connection
        if redis is None and self.config.get("redis", {}).get("url"):
            redis = redispy.StrictRedis.from_url(self.config.redis.url)
        self.redis = redis

        # Create our Store instance and associate our store modules with it
        self.models = AttributeDict()
        for name, klass in self.model_classes.items():
            self.models[name] = klass(
                self,
                self.metadata,
                self.engine,
                self.redis,
            )

        # Create our Search Index instance and associate our mappings with it
        self.search = Index(self.models, self.config.search)
        self.search.register(ProjectMapping)

        # Set up our URL routing
        self.urls = urls.urls

        # Initialize our Translations engine
        self.trans = babel.support.NullTranslations()

        # Setup our Jinja2 Environment
        self.templates = jinja2.Environment(
            autoescape=True,
            auto_reload=self.config.debug,
            extensions=[
                "jinja2.ext.i18n",
            ],
            loader=jinja2.PackageLoader("warehouse"),
        )

        # Install Babel
        self.templates.filters.update({
            "package_type_display": packaging_helpers.package_type_display,
            "format_number": babel.numbers.format_number,
            "format_decimal": babel.numbers.format_decimal,
            "format_percent": babel.numbers.format_percent,
            "format_date": babel.dates.format_date,
            "format_datetime": babel.dates.format_datetime,
            "format_time": babel.dates.format_time,
        })

        # Install our translations
        self.templates.install_gettext_translations(self.trans, newstyle=True)

        # Add our Powered By Middleware
        self.wsgi_app = PoweredBy(self.wsgi_app, "Warehouse {} ({})".format(
            warehouse.__version__,
            warehouse.__build__,
        ))

        # Add our Content Security Policy Middleware
        self.wsgi_app = guard.ContentSecurityPolicy(
            self.wsgi_app,
            self.config.security.csp,
        )

        if "sentry" in self.config:
            self.wsgi_app = Sentry(self.wsgi_app, Client(**self.config.sentry))

        # Serve the static files that are packaged as part of Warehouse
        self.wsgi_app = SharedDataMiddleware(
            self.wsgi_app,
            {
                "/static/": os.path.abspath(
                    os.path.join(
                        os.path.dirname(warehouse.__file__),
                        "static",
                        "compiled",
                    ),
                ),
            },
        )

        # configure logging
        logging.config.dictConfig(self.config.logging)

    def __call__(self, environ, start_response):
        """
        Shortcut for :attr:`wsgi_app`.
        """
        return self.wsgi_app(environ, start_response)

    @classmethod
    def from_yaml(cls, *paths, **kwargs):
        # Pull out other keyword arguments
        override = kwargs.pop("override", None)

        default = os.path.abspath(os.path.join(
            os.path.dirname(warehouse.__file__),
            "config.yml",
        ))

        paths = [default] + list(paths)

        config = {}
        for path in paths:
            with open(path) as configfile:
                # Use no cover to work around a coverage bug
                config = merge_dict(  # pragma: no cover
                    config,
                    yaml.safe_load(configfile)
                )

        if override:
            config = merge_dict(config, override)

        return cls(config=config, **kwargs)

    @classmethod
    def from_cli(cls, argv):
        def _generate_parser(parser, commands):
            # Generate our commands
            subparsers = parser.add_subparsers()
            for name, command in commands.items():
                cmd_parser = subparsers.add_parser(name)

                if hasattr(command, "create_parser"):
                    command.create_parser(cmd_parser)

                if isinstance(command, collections.Mapping):
                    _generate_parser(cmd_parser, command)
                else:
                    cmd_parser.set_defaults(_cmd=command)

        parser = argparse.ArgumentParser(prog="warehouse")
        parser.add_argument("-c", "--config", action="append", dest="_configs")

        _generate_parser(parser, warehouse.cli.__commands__)

        args = parser.parse_args(argv)

        configs = args._configs if args._configs is not None else []
        app = cls.from_yaml(*configs)

        return args._cmd(
            app,
            *args._get_args(),
            **{k: v for k, v in args._get_kwargs() if not k.startswith("_")}
        )

    @responder
    def wsgi_app(self, environ, start_response):
        """
        The actual WSGI application.  This is not implemented in
        `__call__` so that middlewares can be applied without losing a
        reference to the class.  So instead of doing this::

            app = MyMiddleware(app)

        It's a better idea to do this instead::

            app.wsgi_app = MyMiddleware(app.wsgi_app)

        Then you still have the original application object around and
        can continue to call methods on it.

        :param environ: a WSGI environment
        :param start_response: a callable accepting a status code,
                               a list of headers and an optional
                               exception context to start the response
        """
        try:
            # Figure out what endpoint to call
            urls = self.urls.bind_to_environ(environ)
            endpoint, kwargs = urls.match()

            # Load our view function
            modname, viewname = endpoint.rsplit(".", 1)
            module = importlib.import_module(modname)
            view = getattr(module, viewname)

            # Create our request object
            request = Request(environ)
            request.url_adapter = urls

            # Dispatch to our view
            return view(self, request, **kwargs)
        except HTTPException as exc:
            return exc
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import logging
from pathlib import Path
import subprocess

import synthtool as s
from synthtool.languages import php
from synthtool import _tracked_paths

logging.basicConfig(level=logging.DEBUG)

src = Path(f"../{php.STAGING_DIR}/Monitoring").resolve()
dest = Path().resolve()

# Added so that we can pass copy_excludes in the owlbot_main() call
_tracked_paths.add(src)

php.owlbot_main(
    src=src,
    dest=dest,
    copy_excludes=[
        src / "*/src/V3/AlertPolicyServiceClient.php",
        src / "*/src/V3/GroupServiceClient.php",
        src / "*/src/V3/NotificationChannelServiceClient.php",
        src / "*/src/V3/ServiceMonitoringServiceClient.php",
        src / "*/src/V3/UptimeCheckServiceGapicClient.php",
    ]
)

# document and utilize apiEndpoint instead of serviceAddress
s.replace(
    "**/Gapic/*GapicClient.php",
    r"'serviceAddress' =>",
    r"'apiEndpoint' =>")
s.replace(
    "**/Gapic/*GapicClient.php",
    r"@type string \$serviceAddress\n\s+\*\s+The address",
    r"""@type string $serviceAddress
     *           **Deprecated**. This option will be removed in a future major release. Please
     *           utilize the `$apiEndpoint` option instead.
     *     @type string $apiEndpoint
     *           The address""")
s.replace(
    "**/Gapic/*GapicClient.php",
    r"\$transportConfig, and any \$serviceAddress",
    r"$transportConfig, and any `$apiEndpoint`")

# V3 is GA, so remove @experimental tags
s.replace(
    'src/V3/**/*Client.php',
    r'^(\s+\*\n)?\s+\*\s@experimental\n',
    '')

for client in ['AlertPolicyService', 'NotificationChannelService']:
    s.replace(
        f'**/Gapic/{client}GapicClient.php',
        r'Copyright \d{4}',
        'Copyright 2018')
    s.replace(
        f'**/V3/{client}Client.php',
        r'Copyright \d{4}',
        'Copyright 2018')

s.replace(
    'tests/**/V3/*Test.php',
    r'Copyright \d{4}',
    'Copyright 2018')

# Fix class references in gapic samples
for version in ['V3']:
    pathExpr = 'src/' + version + '/Gapic/*GapicClient.php'

    types = {
        '= new Alert': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\Alert',
        '= new Group': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\Group',
        '= new Metric': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\Metric',
        '= new TimeInterval': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\TimeInterval',
        '= TimeSeriesView::': r'= Google\\Cloud\\Monitoring\\'+ version + r'\\ListTimeSeriesRequest\\TimeSeriesView::',
        '= new Notification': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\Notification',
        '= new UptimeCheck': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\UptimeCheck',

    }

    for search, replace in types.items():
        s.replace(
            pathExpr,
            search,
            replace
)

### [START] protoc backwards compatibility fixes

# roll back to private properties.
s.replace(
    "src/**/V*/**/*.php",
    r"Generated from protobuf field ([^\n]{0,})\n\s{5}\*/\n\s{4}protected \$",
    r"""Generated from protobuf field \1
     */
    private $""")

# prevent proto messages from being marked final
s.replace(
    "src/**/V*/**/*.php",
    r"final class",
    r"class")

# Replace "Unwrapped" with "Value" for method names.
s.replace(
    "src/**/V*/**/*.php",
    r"public function ([s|g]\w{3,})Unwrapped",
    r"public function \1Value"
)

### [END] protoc backwards compatibility fixes

# fix relative cloud.google.com links
s.replace(
    "src/**/V*/**/*.php",
    r"(.{0,})\]\((/.{0,})\)",
    r"\1](https://cloud.google.com\2)"
)
<commit_msg>chore: fix monitoring owlbot excluse (#5417)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import logging
from pathlib import Path
import subprocess

import synthtool as s
from synthtool.languages import php
from synthtool import _tracked_paths

logging.basicConfig(level=logging.DEBUG)

src = Path(f"../{php.STAGING_DIR}/Monitoring").resolve()
dest = Path().resolve()

# Added so that we can pass copy_excludes in the owlbot_main() call
_tracked_paths.add(src)

php.owlbot_main(
    src=src,
    dest=dest,
    copy_excludes=[
        src / "*/src/V3/AlertPolicyServiceClient.php",
        src / "*/src/V3/GroupServiceClient.php",
        src / "*/src/V3/NotificationChannelServiceClient.php",
        src / "*/src/V3/ServiceMonitoringServiceClient.php",
        src / "*/src/V3/UptimeCheckServiceClient.php",
    ]
)

# document and utilize apiEndpoint instead of serviceAddress
s.replace(
    "**/Gapic/*GapicClient.php",
    r"'serviceAddress' =>",
    r"'apiEndpoint' =>")
s.replace(
    "**/Gapic/*GapicClient.php",
    r"@type string \$serviceAddress\n\s+\*\s+The address",
    r"""@type string $serviceAddress
     *           **Deprecated**. This option will be removed in a future major release. Please
     *           utilize the `$apiEndpoint` option instead.
     *     @type string $apiEndpoint
     *           The address""")
s.replace(
    "**/Gapic/*GapicClient.php",
    r"\$transportConfig, and any \$serviceAddress",
    r"$transportConfig, and any `$apiEndpoint`")

# V3 is GA, so remove @experimental tags
s.replace(
    'src/V3/**/*Client.php',
    r'^(\s+\*\n)?\s+\*\s@experimental\n',
    '')

for client in ['AlertPolicyService', 'NotificationChannelService']:
    s.replace(
        f'**/Gapic/{client}GapicClient.php',
        r'Copyright \d{4}',
        'Copyright 2018')
    s.replace(
        f'**/V3/{client}Client.php',
        r'Copyright \d{4}',
        'Copyright 2018')

s.replace(
    'tests/**/V3/*Test.php',
    r'Copyright \d{4}',
    'Copyright 2018')

# Fix class references in gapic samples
for version in ['V3']:
    pathExpr = 'src/' + version + '/Gapic/*GapicClient.php'

    types = {
        '= new Alert': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\Alert',
        '= new Group': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\Group',
        '= new Metric': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\Metric',
        '= new TimeInterval': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\TimeInterval',
        '= TimeSeriesView::': r'= Google\\Cloud\\Monitoring\\'+ version + r'\\ListTimeSeriesRequest\\TimeSeriesView::',
        '= new Notification': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\Notification',
        '= new UptimeCheck': r'= new Google\\Cloud\\Monitoring\\'+ version + r'\\UptimeCheck',

    }

    for search, replace in types.items():
        s.replace(
            pathExpr,
            search,
            replace
)

### [START] protoc backwards compatibility fixes

# roll back to private properties.
s.replace(
    "src/**/V*/**/*.php",
    r"Generated from protobuf field ([^\n]{0,})\n\s{5}\*/\n\s{4}protected \$",
    r"""Generated from protobuf field \1
     */
    private $""")

# prevent proto messages from being marked final
s.replace(
    "src/**/V*/**/*.php",
    r"final class",
    r"class")

# Replace "Unwrapped" with "Value" for method names.
s.replace(
    "src/**/V*/**/*.php",
    r"public function ([s|g]\w{3,})Unwrapped",
    r"public function \1Value"
)

### [END] protoc backwards compatibility fixes

# fix relative cloud.google.com links
s.replace(
    "src/**/V*/**/*.php",
    r"(.{0,})\]\((/.{0,})\)",
    r"\1](https://cloud.google.com\2)"
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    silo.exceptions
    ~~~~~~~~~~~~~~~

    This module contains Silo-specific exceptions.

    :copyright: (c) 2014 by Janne Vanhala.
    :license: BSD, see LICENSE for more details.
"""

from __future__ import unicode_literals

from ._compat import force_text, unicode_compatible

class SiloError(Exception):
    """
    Base class for all Silo exceptions.
    """

@unicode_compatible
class FileNotFoundError(SiloError):
    """
    Raised when attempting to access a file that does not exist.

    :param name: name of the file
    """
    def __init__(self, name):
        self.name = force_text(name, 'utf-8')

    def __str__(self):
        return 'The file "{name}" was not found.'.format(name=self.name)

@unicode_compatible
class FileNotWithinStorageError(SiloError):
    """
    Raised when a suspicious filename is supplied to an storage.

    This error can occur when using the
    :class:`silo.storages.filesystem.FileSystemStorage` storage and the
    filename is not within the base directory.

    :param name: name of the file
    :type name: str
    """
    def __init__(self, name):
        self.name = force_text(name, 'utf-8')

    def __str__(self):
        return 'The file "{name}" is not within the storage.'.format(
            name=self.name
        )
<commit_msg>Improve FileNotWithinStorageError documentation<commit_after># -*- coding: utf-8 -*-
"""
    silo.exceptions
    ~~~~~~~~~~~~~~~

    This module contains Silo-specific exceptions.

    :copyright: (c) 2014 by Janne Vanhala.
    :license: BSD, see LICENSE for more details.
"""

from __future__ import unicode_literals

from ._compat import force_text, unicode_compatible

class SiloError(Exception):
    """
    Base class for all Silo exceptions.
    """

@unicode_compatible
class FileNotFoundError(SiloError):
    """
    Raised when attempting to access a file that does not exist.

    :param name: name of the file
    """
    def __init__(self, name):
        self.name = force_text(name, 'utf-8')

    def __str__(self):
        return 'The file "{name}" was not found.'.format(name=self.name)

@unicode_compatible
class FileNotWithinStorageError(SiloError):
    """
    Raised when a suspicious filename is supplied to a storage.

    This error occurs when using :class:`.FileSystemStorage` and trying
    to access a file not within the base directory::

        >>> from silo.storages.filesystem import FileSystemStorage
        >>> storage = FileSystemStorage(directory='/path/to/storage/root')
        >>> storage.open('/etc/passwd')
        Traceback (most recent call last):
        FileNotWithinStorageError: The file "/etc/passwd" is not within the storage.

    :param name: name of the file
    :type name: str
    """
    def __init__(self, name):
        self.name = force_text(name, 'utf-8')

    def __str__(self):
        return 'The file "{name}" is not within the storage.'.format(
            name=self.name
        )
<|endoftext|>


<commit_before># Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import argparse
import collections
import importlib
import os.path
import logging.config

import babel.dates
import babel.numbers
import babel.support
import guard
import jinja2

import redis as redispy

import sqlalchemy
import yaml

from raven import Client
from raven.middleware import Sentry
from webassets import Environment as AssetsEnvironment
from werkzeug.exceptions import HTTPException
from werkzeug.wsgi import SharedDataMiddleware, responder

import warehouse
import warehouse.cli

from warehouse import urls
from warehouse.http import Request
from warehouse.middleware import PoweredBy
from warehouse.packaging import helpers as packaging_helpers
from warehouse.utils import AttributeDict, merge_dict, convert_to_attr_dict

class Warehouse(object):

    metadata = sqlalchemy.MetaData()

    model_names = {
        "accounts": "warehouse.accounts.models:Model",
        "packaging": "warehouse.packaging.models:Model",
    }

    def __init__(self, config, engine=None, redis=None):
        self.config = convert_to_attr_dict(config)

        # Connect to the database
        if engine is None and self.config.get("database", {}).get("url"):
            engine = sqlalchemy.create_engine(self.config.database.url)
        self.engine = engine

        # Create our redis connection
        if redis is None and self.config.get("redis", {}).get("url"):
            redis = redispy.StrictRedis.from_url(self.config.redis.url)
        self.redis = redis

        # Create our Store instance and associate our store modules with it
        self.models = AttributeDict()
        for name, mod_path in self.model_names.items():
            mod_name, klass = mod_path.rsplit(":", 1)
            mod = importlib.import_module(mod_name)
            self.models[name] = getattr(mod, klass)(
                self,
                self.metadata,
                self.engine,
                self.redis,
            )

        # Set up our URL routing
        self.urls = urls.urls

        # Initialize our Translations engine
        self.trans = babel.support.NullTranslations()

        # Setup our Jinja2 Environment
        self.templates = jinja2.Environment(
            autoescape=True,
            auto_reload=self.config.debug,
            extensions=[
                "jinja2.ext.i18n",
                "webassets.ext.jinja2.AssetsExtension",
            ],
            loader=jinja2.PackageLoader("warehouse"),
        )

        # Install Babel
        self.templates.filters.update({
            "package_type_display": packaging_helpers.package_type_display,
            "format_number": babel.numbers.format_number,
            "format_decimal": babel.numbers.format_decimal,
            "format_percent": babel.numbers.format_percent,
            "format_date": babel.dates.format_date,
            "format_datetime": babel.dates.format_datetime,
            "format_time": babel.dates.format_time,
        })

        # Install our translations
        self.templates.install_gettext_translations(self.trans, newstyle=True)

        # Setup our web assets environment
        asset_config = self.config.assets
        asset_config.setdefault("debug", self.config.debug)
        asset_config.setdefault("auto_build", self.config.debug)

        self.templates.assets_environment = AssetsEnvironment(**asset_config)

        # Load our static directories
        static_path = os.path.abspath(
            os.path.join(os.path.dirname(warehouse.__file__), "static"),
        )
        self.templates.assets_environment.append_path(
            static_path,
            self.config.assets.url,
        )

        # Add our Powered By Middleware
        self.wsgi_app = PoweredBy(self.wsgi_app, "Warehouse {} ({})".format(
            warehouse.__version__,
            warehouse.__build__,
        ))

        # Add our Content Security Policy Middleware
        self.wsgi_app = guard.ContentSecurityPolicy(
            self.wsgi_app,
            self.config.security.csp,
        )

        if "sentry" in self.config:
            self.wsgi_app = Sentry(self.wsgi_app, Client(**self.config.sentry))

        # Serve the static files if we're in debug
        if self.config.debug:
            self.wsgi_app = SharedDataMiddleware(
                self.wsgi_app,
                {"/static/": static_path},
            )
            self.wsgi_app = SharedDataMiddleware(
                self.wsgi_app,
                {"/static/": self.config.assets.directory},
            )

        # configure logging
        logging.config.dictConfig(self.config.logging)

    def __call__(self, environ, start_response):
        """
        Shortcut for :attr:`wsgi_app`.
        """
        return self.wsgi_app(environ, start_response)

    @classmethod
    def from_yaml(cls, *paths, **kwargs):
        # Pull out other keyword arguments
        override = kwargs.pop("override", None)

        default = os.path.abspath(os.path.join(
            os.path.dirname(warehouse.__file__),
            "config.yml",
        ))

        paths = [default] + list(paths)

        config = {}
        for path in paths:
            with open(path) as configfile:
                # Use no cover to work around a coverage bug
                config = merge_dict(  # pragma: no cover
                    config,
                    yaml.safe_load(configfile)
                )

        if override:
            config = merge_dict(config, override)

        return cls(config=config, **kwargs)

    @classmethod
    def from_cli(cls, argv):
        def _generate_parser(parser, commands):
            # Generate our commands
            subparsers = parser.add_subparsers()
            for name, command in commands.items():
                cmd_parser = subparsers.add_parser(name)

                if hasattr(command, "create_parser"):
                    command.create_parser(cmd_parser)

                if isinstance(command, collections.Mapping):
                    _generate_parser(cmd_parser, command)
                else:
                    cmd_parser.set_defaults(_cmd=command)

        parser = argparse.ArgumentParser(prog="warehouse")
        parser.add_argument("-c", "--config", action="append", dest="_configs")

        _generate_parser(parser, warehouse.cli.__commands__)

        args = parser.parse_args(argv)

        configs = args._configs if args._configs is not None else []
        app = cls.from_yaml(*configs)

        return args._cmd(
            app,
            *args._get_args(),
            **{k: v for k, v in args._get_kwargs() if not k.startswith("_")}
        )

    @responder
    def wsgi_app(self, environ, start_response):
        """
        The actual WSGI application.  This is not implemented in
        `__call__` so that middlewares can be applied without losing a
        reference to the class.  So instead of doing this::

            app = MyMiddleware(app)

        It's a better idea to do this instead::

            app.wsgi_app = MyMiddleware(app.wsgi_app)

        Then you still have the original application object around and
        can continue to call methods on it.

        :param environ: a WSGI environment
        :param start_response: a callable accepting a status code,
                               a list of headers and an optional
                               exception context to start the response
        """
        try:
            # Figure out what endpoint to call
            urls = self.urls.bind_to_environ(environ)
            endpoint, kwargs = urls.match()

            # Load our view function
            modname, viewname = endpoint.rsplit(".", 1)
            module = importlib.import_module(modname)
            view = getattr(module, viewname)

            # Create our request object
            request = Request(environ)
            request.url_adapter = urls

            # Dispatch to our view
            return view(self, request, **kwargs)
        except HTTPException as exc:
            return exc
<commit_msg>Serve static files regardless of debug<commit_after># Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import argparse
import collections
import importlib
import os.path
import logging.config

import babel.dates
import babel.numbers
import babel.support
import guard
import jinja2

import redis as redispy

import sqlalchemy
import yaml

from raven import Client
from raven.middleware import Sentry
from webassets import Environment as AssetsEnvironment
from werkzeug.exceptions import HTTPException
from werkzeug.wsgi import SharedDataMiddleware, responder

import warehouse
import warehouse.cli

from warehouse import urls
from warehouse.http import Request
from warehouse.middleware import PoweredBy
from warehouse.packaging import helpers as packaging_helpers
from warehouse.utils import AttributeDict, merge_dict, convert_to_attr_dict

class Warehouse(object):

    metadata = sqlalchemy.MetaData()

    model_names = {
        "accounts": "warehouse.accounts.models:Model",
        "packaging": "warehouse.packaging.models:Model",
    }

    def __init__(self, config, engine=None, redis=None):
        self.config = convert_to_attr_dict(config)

        # Connect to the database
        if engine is None and self.config.get("database", {}).get("url"):
            engine = sqlalchemy.create_engine(self.config.database.url)
        self.engine = engine

        # Create our redis connection
        if redis is None and self.config.get("redis", {}).get("url"):
            redis = redispy.StrictRedis.from_url(self.config.redis.url)
        self.redis = redis

        # Create our Store instance and associate our store modules with it
        self.models = AttributeDict()
        for name, mod_path in self.model_names.items():
            mod_name, klass = mod_path.rsplit(":", 1)
            mod = importlib.import_module(mod_name)
            self.models[name] = getattr(mod, klass)(
                self,
                self.metadata,
                self.engine,
                self.redis,
            )

        # Set up our URL routing
        self.urls = urls.urls

        # Initialize our Translations engine
        self.trans = babel.support.NullTranslations()

        # Setup our Jinja2 Environment
        self.templates = jinja2.Environment(
            autoescape=True,
            auto_reload=self.config.debug,
            extensions=[
                "jinja2.ext.i18n",
                "webassets.ext.jinja2.AssetsExtension",
            ],
            loader=jinja2.PackageLoader("warehouse"),
        )

        # Install Babel
        self.templates.filters.update({
            "package_type_display": packaging_helpers.package_type_display,
            "format_number": babel.numbers.format_number,
            "format_decimal": babel.numbers.format_decimal,
            "format_percent": babel.numbers.format_percent,
            "format_date": babel.dates.format_date,
            "format_datetime": babel.dates.format_datetime,
            "format_time": babel.dates.format_time,
        })

        # Install our translations
        self.templates.install_gettext_translations(self.trans, newstyle=True)

        # Setup our web assets environment
        asset_config = self.config.assets
        asset_config.setdefault("debug", self.config.debug)
        asset_config.setdefault("auto_build", self.config.debug)

        self.templates.assets_environment = AssetsEnvironment(**asset_config)

        # Load our static directories
        static_path = os.path.abspath(
            os.path.join(os.path.dirname(warehouse.__file__), "static"),
        )
        self.templates.assets_environment.append_path(
            static_path,
            self.config.assets.url,
        )

        # Add our Powered By Middleware
        self.wsgi_app = PoweredBy(self.wsgi_app, "Warehouse {} ({})".format(
            warehouse.__version__,
            warehouse.__build__,
        ))

        # Add our Content Security Policy Middleware
        self.wsgi_app = guard.ContentSecurityPolicy(
            self.wsgi_app,
            self.config.security.csp,
        )

        if "sentry" in self.config:
            self.wsgi_app = Sentry(self.wsgi_app, Client(**self.config.sentry))

        # Serve the static files that are packaged as part of Warehouse
        self.wsgi_app = SharedDataMiddleware(
            self.wsgi_app,
            {"/static/": static_path},
        )

        # Serve the static files that webassets creates
        self.wsgi_app = SharedDataMiddleware(
            self.wsgi_app,
            {"/static/": self.config.assets.directory},
        )

        # configure logging
        logging.config.dictConfig(self.config.logging)

    def __call__(self, environ, start_response):
        """
        Shortcut for :attr:`wsgi_app`.
        """
        return self.wsgi_app(environ, start_response)

    @classmethod
    def from_yaml(cls, *paths, **kwargs):
        # Pull out other keyword arguments
        override = kwargs.pop("override", None)

        default = os.path.abspath(os.path.join(
            os.path.dirname(warehouse.__file__),
            "config.yml",
        ))

        paths = [default] + list(paths)

        config = {}
        for path in paths:
            with open(path) as configfile:
                # Use no cover to work around a coverage bug
                config = merge_dict(  # pragma: no cover
                    config,
                    yaml.safe_load(configfile)
                )

        if override:
            config = merge_dict(config, override)

        return cls(config=config, **kwargs)

    @classmethod
    def from_cli(cls, argv):
        def _generate_parser(parser, commands):
            # Generate our commands
            subparsers = parser.add_subparsers()
            for name, command in commands.items():
                cmd_parser = subparsers.add_parser(name)

                if hasattr(command, "create_parser"):
                    command.create_parser(cmd_parser)

                if isinstance(command, collections.Mapping):
                    _generate_parser(cmd_parser, command)
                else:
                    cmd_parser.set_defaults(_cmd=command)

        parser = argparse.ArgumentParser(prog="warehouse")
        parser.add_argument("-c", "--config", action="append", dest="_configs")

        _generate_parser(parser, warehouse.cli.__commands__)

        args = parser.parse_args(argv)

        configs = args._configs if args._configs is not None else []
        app = cls.from_yaml(*configs)

        return args._cmd(
            app,
            *args._get_args(),
            **{k: v for k, v in args._get_kwargs() if not k.startswith("_")}
        )

    @responder
    def wsgi_app(self, environ, start_response):
        """
        The actual WSGI application.  This is not implemented in
        `__call__` so that middlewares can be applied without losing a
        reference to the class.  So instead of doing this::

            app = MyMiddleware(app)

        It's a better idea to do this instead::

            app.wsgi_app = MyMiddleware(app.wsgi_app)

        Then you still have the original application object around and
        can continue to call methods on it.

        :param environ: a WSGI environment
        :param start_response: a callable accepting a status code,
                               a list of headers and an optional
                               exception context to start the response
        """
        try:
            # Figure out what endpoint to call
            urls = self.urls.bind_to_environ(environ)
            endpoint, kwargs = urls.match()

            # Load our view function
            modname, viewname = endpoint.rsplit(".", 1)
            module = importlib.import_module(modname)
            view = getattr(module, viewname)

            # Create our request object
            request = Request(environ)
            request.url_adapter = urls

            # Dispatch to our view
            return view(self, request, **kwargs)
        except HTTPException as exc:
            return exc
<|endoftext|>