<commit_before>from __future__ import print_function

from genome_handler import GenomeHandler
import numpy as np
from keras.models import Sequential
from keras.utils import np_utils
from keras.datasets import mnist, cifar10
from keras.callbacks import EarlyStopping
from datetime import datetime
import random as rand
import csv
import sys
import operator

METRIC_OPS = [operator.__lt__, operator.__gt__]
METRIC_OBJECTIVES = [min, max]

class DEvol:

    def __init__(self, genome_handler, data_path=""):
        self.genome_handler = genome_handler
        self.datafile = data_path or (datetime.now().ctime() + '.csv')

        print("Genome encoding and accuracy data stored at", self.datafile, "\n")
        with open(self.datafile, 'a') as csvfile:
            writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            genome = genome_handler.genome_representation() + ["Val Loss", "Val Accuracy"]
            writer.writerow(genome)

    def set_objective(self, metric):
        """set the metric and objective for this search  should be 'accuracy' or 'loss'"""
        if metric is 'acc':
            metric = 'accuracy'
        if not metric in ['loss', 'accuracy']:
            raise ValueError(
                'Invalid metric name {} provided - should be "accuracy" or "loss"'.format(metric))
        self.metric = metric
        self.objective = "max" if self.metric is "accuracy" else "min"
        self.metric_index = 1 if self.metric is 'loss' else -1
        self.metric_op = METRIC_OPS[self.objective is 'max']
        self.metric_objective = METRIC_OBJECTIVES[self.objective is 'max']

    def run(self, dataset, num_generations, pop_size, epochs, fitness=None, metric='accuracy'):
        """run genetic search on dataset given number of generations and population size

        Args:
            dataset : tuple or list of numpy arrays in form ((train_data, train_labels), (validation_data, validation_labels))
            num_generations (int): number of generations to search
            pop_size (int): initial population size
            epochs (int): epochs to run each search, passed to keras model.fit -currently searches are
                            curtailed if no improvement is seen in 1 epoch
            fitness (None, optional): scoring function to be applied to population scores, will be called on a numpy array
                                      which is a  min/max scaled version of evaluated model metrics, so
                                      It should accept a real number including 0. If left as default just the min/max
                                      scaled values will be used.
            metric (str, optional): must be "accuracy" or "loss" , defines what to optimize during search

        Returns:
            (keras model, float, float ): best model found in the form of (model, loss, accuracy)
        """
        self.set_objective(metric)
        (self.x_train, self.y_train), (self.x_test, self.y_test) = dataset
        # Generate initial random population
        members = [self.genome_handler.generate() for _ in range(pop_size)]
        fit = []
        metric_index = 1 if self.metric is 'loss' else -1
        for i in range(len(members)):
            print("\nmodel {0}/{1} - generation {2}/{3}:\n"\
                    .format(i + 1, len(members), 1, num_generations))
            res = self.evaluate(members[i], epochs)
            v = res[metric_index]
            del res
            fit.append(v)

        fit = np.array(fit)
        pop = Population(members, fit, fitness, obj=self.objective)
        print("Generation {3}:\t\tbest {4}: {0:0.4f}\t\taverage: {1:0.4f}\t\tstd: {2:0.4f}"\
                .format(self.metric_objective(fit), np.mean(fit), np.std(fit), 1, self.metric))

        # Evolve over
        for gen in range(1, num_generations):
            members = []
            for i in range(int(pop_size * 0.95)):  # Crossover
                members.append(self.crossover(pop.select(), pop.select()))
            members += pop.getBest(pop_size - int(pop_size * 0.95))
            for i in range(len(members)):  # Mutation
                members[i] = self.mutate(members[i], gen)
            fit = []
            for i in range(len(members)):
                print("\nmodel {0}/{1} - generation {2}/{3}:\n"
                        .format(i + 1, len(members), gen + 1, num_generations))
                res = self.evaluate(members[i], epochs)
                v = res[metric_index]
                del res
                fit.append(v)

            fit = np.array(fit)
            pop = Population(members, fit, fitness, obj=self.objective)
            print("Generation {3}:\t\tbest {4}: {0:0.4f}\t\taverage: {1:0.4f}\t\tstd: {2:0.4f}"\
                    .format(self.metric_objective(fit), np.mean(fit), np.std(fit), gen + 1, self.metric))

        return self.genome_handler.decode_best(self.datafile)

    def evaluate(self, genome, epochs):
        model = self.genome_handler.decode(genome)
        loss, accuracy = None, None
        try:
            model.fit(self.x_train, self.y_train, validation_data=(self.x_test, self.y_test),
                      epochs=epochs,
                      verbose=1,
                      callbacks=[EarlyStopping(monitor='val_loss', patience=1, verbose=1)])
            loss, accuracy = model.evaluate(self.x_test, self.y_test, verbose=0)
        except:
            loss = 1.5
            accuracy = 1 / self.genome_handler.n_classes
        # Record the stats
        with open(self.datafile, 'a') as csvfile:
            writer = csv.writer(csvfile, delimiter=',',
                                quotechar='"', quoting=csv.QUOTE_MINIMAL)
            row = list(genome) + [loss, accuracy]
            writer.writerow(row)
        return model, loss, accuracy

    def crossover(self, genome1, genome2):
        crossIndexA = rand.randint(0, len(genome1))
        child = genome1[:crossIndexA] + genome2[crossIndexA:]
        return child

    def mutate(self, genome, generation):
        # increase mutations as program continues
        num_mutations = max(3, generation / 4)
        return self.genome_handler.mutate(genome, num_mutations)

class Population:

    def __len__(self):
        return len(self.members)

    def __init__(self, members, fitnesses, score, obj='max'):
        self.members = members
        scores = fitnesses - fitnesses.min()
        scores /= scores.max()
        if obj is 'min':
            scores = 1 - scores
        if score:
            self.scores = score(scores)
        else:
            self.scores = scores
        self.s_fit = sum(self.scores)

    def getBest(self, n):
        combined = [(self.members[i], self.scores[i])
                    for i in range(len(self.members))]
        sorted(combined, key=(lambda x: x[1]), reverse=True)
        return [x[0] for x in combined[:n]]

    def select(self):
        dart = rand.uniform(0, self.s_fit)
        sum_fits = 0
        for i in range(len(self.members)):
            sum_fits += self.scores[i]
            if sum_fits > dart:
                return self.members[i]
<commit_msg>fix some circumstantial errors and memory issues<commit_after>from __future__ import print_function

from genome_handler import GenomeHandler
import numpy as np
from keras.models import Sequential
from keras.utils import np_utils
from keras.datasets import mnist, cifar10
from keras.callbacks import EarlyStopping
from datetime import datetime
import random as rand
import csv
import sys
import operator
import gc

METRIC_OPS = [operator.__lt__, operator.__gt__]
METRIC_OBJECTIVES = [min, max]

class DEvol:

    def __init__(self, genome_handler, data_path=""):
        self.genome_handler = genome_handler
        self.datafile = data_path or (datetime.now().ctime() + '.csv')

        print("Genome encoding and accuracy data stored at", self.datafile, "\n")
        with open(self.datafile, 'a') as csvfile:
            writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            genome = genome_handler.genome_representation() + ["Val Loss", "Val Accuracy"]
            writer.writerow(genome)

    def set_objective(self, metric):
        """set the metric and objective for this search  should be 'accuracy' or 'loss'"""
        if metric is 'acc':
            metric = 'accuracy'
        if not metric in ['loss', 'accuracy']:
            raise ValueError(
                'Invalid metric name {} provided - should be "accuracy" or "loss"'.format(metric))
        self.metric = metric
        self.objective = "max" if self.metric is "accuracy" else "min"
        self.metric_index = 1 if self.metric is 'loss' else -1
        self.metric_op = METRIC_OPS[self.objective is 'max']
        self.metric_objective = METRIC_OBJECTIVES[self.objective is 'max']

    def run(self, dataset, num_generations, pop_size, epochs, fitness=None, metric='accuracy'):
        """run genetic search on dataset given number of generations and population size

        Args:
            dataset : tuple or list of numpy arrays in form ((train_data, train_labels), (validation_data, validation_labels))
            num_generations (int): number of generations to search
            pop_size (int): initial population size
            epochs (int): epochs to run each search, passed to keras model.fit -currently searches are
                            curtailed if no improvement is seen in 1 epoch
            fitness (None, optional): scoring function to be applied to population scores, will be called on a numpy array
                                      which is a  min/max scaled version of evaluated model metrics, so
                                      It should accept a real number including 0. If left as default just the min/max
                                      scaled values will be used.
            metric (str, optional): must be "accuracy" or "loss" , defines what to optimize during search

        Returns:
            (keras model, float, float ): best model found in the form of (model, loss, accuracy)
        """
        self.set_objective(metric)
        (self.x_train, self.y_train), (self.x_test, self.y_test) = dataset
        # Generate initial random population
        members = [self.genome_handler.generate() for _ in range(pop_size)]
        fit = []
        metric_index = 1 if self.metric is 'loss' else -1
        for i in range(len(members)):
            print("\nmodel {0}/{1} - generation {2}/{3}:\n"\
                    .format(i + 1, len(members), 1, num_generations))
            res = self.evaluate(members[i], epochs)
            v = res[metric_index]
            del res
            fit.append(v)

        fit = np.array(fit)
        pop = Population(members, fit, fitness, obj=self.objective)
        print("Generation {3}:\t\tbest {4}: {0:0.4f}\t\taverage: {1:0.4f}\t\tstd: {2:0.4f}"\
                .format(self.metric_objective(fit), np.mean(fit), np.std(fit), 1, self.metric))

        # Evolve over
        for gen in range(1, num_generations):
            members = []
            for i in range(int(pop_size * 0.95)):  # Crossover
                members.append(self.crossover(pop.select(), pop.select()))
            members += pop.getBest(pop_size - int(pop_size * 0.95))
            for i in range(len(members)):  # Mutation
                members[i] = self.mutate(members[i], gen)
            fit = []
            for i in range(len(members)):
                print("\nmodel {0}/{1} - generation {2}/{3}:\n"
                        .format(i + 1, len(members), gen + 1, num_generations))
                res = self.evaluate(members[i], epochs)
                v = res[metric_index]
                del res
                fit.append(v)

            fit = np.array(fit)
            pop = Population(members, fit, fitness, obj=self.objective)
            print("Generation {3}:\t\tbest {4}: {0:0.4f}\t\taverage: {1:0.4f}\t\tstd: {2:0.4f}"\
                    .format(self.metric_objective(fit), np.mean(fit), np.std(fit), gen + 1, self.metric))

        return self.genome_handler.decode_best(self.datafile)

    def evaluate(self, genome, epochs):
        model = self.genome_handler.decode(genome)
        loss, accuracy = None, None
        try:
            model.fit(self.x_train, self.y_train, validation_data=(self.x_test, self.y_test),
                      epochs=epochs,
                      verbose=1,
                      callbacks=[EarlyStopping(monitor='val_loss', patience=1, verbose=1)])
            loss, accuracy = model.evaluate(self.x_test, self.y_test, verbose=0)
        except:
            loss = 1.5
            accuracy = 1 / self.genome_handler.n_classes
            gc.collect()
            print("Model exhausted too much memory. Assigned poor score.")
        # Record the stats
        with open(self.datafile, 'a') as csvfile:
            writer = csv.writer(csvfile, delimiter=',',
                                quotechar='"', quoting=csv.QUOTE_MINIMAL)
            row = list(genome) + [loss, accuracy]
            writer.writerow(row)
        return model, loss, accuracy

    def crossover(self, genome1, genome2):
        crossIndexA = rand.randint(0, len(genome1))
        child = genome1[:crossIndexA] + genome2[crossIndexA:]
        return child

    def mutate(self, genome, generation):
        # increase mutations as program continues
        num_mutations = max(3, generation // 4)
        return self.genome_handler.mutate(genome, num_mutations)

class Population:

    def __len__(self):
        return len(self.members)

    def __init__(self, members, fitnesses, score, obj='max'):
        self.members = members
        scores = fitnesses - fitnesses.min()
        if scores.max() > 0:
            scores /= scores.max()
        if obj is 'min':
            scores = 1 - scores
        if score:
            self.scores = score(scores)
        else:
            self.scores = scores
        self.s_fit = sum(self.scores)

    def getBest(self, n):
        combined = [(self.members[i], self.scores[i])
                    for i in range(len(self.members))]
        sorted(combined, key=(lambda x: x[1]), reverse=True)
        return [x[0] for x in combined[:n]]

    def select(self):
        dart = rand.uniform(0, self.s_fit)
        sum_fits = 0
        for i in range(len(self.members)):
            sum_fits += self.scores[i]
            if sum_fits >= dart:
                return self.members[i]
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2010 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Utility methods for working with WSGI servers
"""

import json
import logging
import sys
from xml.dom import minidom

import eventlet
import eventlet.wsgi
eventlet.patcher.monkey_patch(all=False, socket=True)
import routes
import routes.middleware
import webob
import webob.dec
import webob.exc

logging.getLogger("routes.middleware").addHandler(logging.StreamHandler())

def run_server(application, port):
    """Run a WSGI server with the given application."""
    sock = eventlet.listen(('0.0.0.0', port))
    eventlet.wsgi.server(sock, application)

class Application(object):
# TODO(gundlach): I think we should toss this class, now that it has no
# purpose.
    """Base WSGI application wrapper. Subclasses need to implement __call__."""

    def __call__(self, environ, start_response):
        r"""Subclasses will probably want to implement __call__ like this:

        @webob.dec.wsgify
        def __call__(self, req):
          # Any of the following objects work as responses:

          # Option 1: simple string
          res = 'message\n'

          # Option 2: a nicely formatted HTTP exception page
          res = exc.HTTPForbidden(detail='Nice try')

          # Option 3: a webob Response object (in case you need to play with
          # headers, or you want to be treated like an iterable, or or or)
          res = Response();
          res.app_iter = open('somefile')

          # Option 4: any wsgi app to be run next
          res = self.application

          # Option 5: you can get a Response object for a wsgi app, too, to
          # play with headers etc
          res = req.get_response(self.application)

          # You can then just return your response...
          return res
          # ... or set req.response and return None.
          req.response = res

        See the end of http://pythonpaste.org/webob/modules/dec.html
        for more info.
        """
        raise NotImplementedError("You must implement __call__")

class Middleware(Application):
    """
    Base WSGI middleware wrapper. These classes require an application to be
    initialized that will be called next.  By default the middleware will
    simply call its wrapped app, or you can override __call__ to customize its
    behavior.
    """

    def __init__(self, application): # pylint: disable-msg=W0231
        self.application = application

    @webob.dec.wsgify
    def __call__(self, req): # pylint: disable-msg=W0221
        """Override to implement middleware behavior."""
        return self.application

class Debug(Middleware):
    """Helper class that can be inserted into any WSGI application chain
    to get information about the request and response."""

    @webob.dec.wsgify
    def __call__(self, req):
        print ("*" * 40) + " REQUEST ENVIRON"
        for key, value in req.environ.items():
            print key, "=", value
        print
        resp = req.get_response(self.application)

        print ("*" * 40) + " RESPONSE HEADERS"
        for (key, value) in resp.headers.iteritems():
            print key, "=", value
        print

        resp.app_iter = self.print_generator(resp.app_iter)

        return resp

    @staticmethod
    def print_generator(app_iter):
        """
        Iterator that prints the contents of a wrapper string iterator
        when iterated.
        """
        print ("*" * 40) + " BODY"
        for part in app_iter:
            sys.stdout.write(part)
            sys.stdout.flush()
            yield part
        print

class Router(object):
    """
    WSGI middleware that maps incoming requests to WSGI apps.
    """

    def __init__(self, mapper):
        """
        Create a router for the given routes.Mapper.

        Each route in `mapper` must specify a 'controller', which is a
        WSGI app to call.  You'll probably want to specify an 'action' as
        well and have your controller be a wsgi.Controller, who will route
        the request to the action method.

        Examples:
          mapper = routes.Mapper()
          sc = ServerController()

          # Explicit mapping of one route to a controller+action
          mapper.connect(None, "/svrlist", controller=sc, action="list")

          # Actions are all implicitly defined
          mapper.resource("server", "servers", controller=sc)

          # Pointing to an arbitrary WSGI app.  You can specify the
          # {path_info:.*} parameter so the target app can be handed just that
          # section of the URL.
          mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp())
        """
        self.map = mapper
        self._router = routes.middleware.RoutesMiddleware(self._dispatch,
                                                          self.map)

    @webob.dec.wsgify
    def __call__(self, req):
        """
        Route the incoming request to a controller based on self.map.
        If no match, return a 404.
        """
        return self._router

    @staticmethod
    @webob.dec.wsgify
    def _dispatch(req):
        """
        Called by self._router after matching the incoming request to a route
        and putting the information into req.environ.  Either returns 404
        or the routed WSGI app's response.
        """
        match = req.environ['wsgiorg.routing_args'][1]
        if not match:
            return webob.exc.HTTPNotFound()
        app = match['controller']
        return app

class Controller(object):
    """
    WSGI app that reads routing information supplied by RoutesMiddleware
    and calls the requested action method upon itself.  All action methods
    must, in addition to their normal parameters, accept a 'req' argument
    which is the incoming webob.Request.  They raise a webob.exc exception,
    or return a dict which will be serialized by requested content type.
    """

    @webob.dec.wsgify
    def __call__(self, req):
        """
        Call the method specified in req.environ by RoutesMiddleware.
        """
        arg_dict = req.environ['wsgiorg.routing_args'][1]
        action = arg_dict['action']
        method = getattr(self, action)
        del arg_dict['controller']
        del arg_dict['action']
        arg_dict['req'] = req
        result = method(**arg_dict)
        if type(result) is dict:
            return self._serialize(result, req)
        else:
            return result

    def _serialize(self, data, request):
        """
        Serialize the given dict to the response type requested in request.
        Uses self._serialization_metadata if it exists, which is a dict mapping
        MIME types to information needed to serialize to that type.
        """
        _metadata = getattr(type(self), "_serialization_metadata", {})
        serializer = Serializer(request.environ, _metadata)
        return serializer.to_content_type(data)

    def _deserialize(self, data, request):
        """
        Deserialize the request body to the response type requested in request.
        Uses self._serialization_metadata if it exists, which is a dict mapping
        MIME types to information needed to serialize to that type.
        """
        _metadata = getattr(type(self), "_serialization_metadata", {})
        serializer = Serializer(request.environ, _metadata)
        return serializer.deserialize(data)

class Serializer(object):
    """
    Serializes and deserializes dictionaries to certain MIME types.
    """

    def __init__(self, environ, metadata=None):
        """
        Create a serializer based on the given WSGI environment.
        'metadata' is an optional dict mapping MIME types to information
        needed to serialize a dictionary to that type.
        """
        self.metadata = metadata or {}
        req = webob.Request(environ)
        suffix = req.path_info.split('.')[-1].lower()
        if suffix == 'json':
            self.handler = self._to_json
        elif suffix == 'xml':
            self.handler = self._to_xml
        elif 'application/json' in req.accept:
            self.handler = self._to_json
        elif 'application/xml' in req.accept:
            self.handler = self._to_xml
        else:
            self.handler = self._to_json # default

    def to_content_type(self, data):
        """
        Serialize a dictionary into a string.

        The format of the string will be decided based on the Content Type
        requested in self.environ: by Accept: header, or by URL suffix.
        """
        return self.handler(data)

    def deserialize(self, datastring):
        """
        Deserialize a string to a dictionary.

        The string must be in the format of a supported MIME type.
        """
        datastring = datastring.strip()
        is_xml = (datastring[0] == '<')
        if not is_xml:
            return json.loads(datastring)
        return self._from_xml(datastring)

    def _from_xml(self, datastring):
        xmldata = self.metadata.get('application/xml', {})
        plurals = set(xmldata.get('plurals', {}))
        node = minidom.parseString(datastring).childNodes[0]
        return {node.nodeName: self._from_xml_node(node, plurals)}

    def _from_xml_node(self, node, listnames):
        """
        Convert a minidom node to a simple Python type.

        listnames is a collection of names of XML nodes whose subnodes should
        be considered list items.
        """
        if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3:
            return node.childNodes[0].nodeValue
        elif node.nodeName in listnames:
            return [self._from_xml_node(n, listnames) for n in node.childNodes]
        else:
            result = dict()
            for attr in node.attributes.keys():
                result[attr] = node.attributes[attr].nodeValue
            for child in node.childNodes:
                if child.nodeType != node.TEXT_NODE:
                    result[child.nodeName] = self._from_xml_node(child, listnames)
            return result

    def _to_json(self, data):
        return json.dumps(data)

    def _to_xml(self, data):
        metadata = self.metadata.get('application/xml', {})
        # We expect data to contain a single key which is the XML root.
        root_key = data.keys()[0]
        doc = minidom.Document()
        node = self._to_xml_node(doc, metadata, root_key, data[root_key])
        return node.toprettyxml(indent='    ')

    def _to_xml_node(self, doc, metadata, nodename, data):
        """Recursive method to convert data members to XML nodes."""
        result = doc.createElement(nodename)
        if type(data) is list:
            singular = metadata.get('plurals', {}).get(nodename, None)
            if singular is None:
                if nodename.endswith('s'):
                    singular = nodename[:-1]
                else:
                    singular = 'item'
            for item in data:
                node = self._to_xml_node(doc, metadata, singular, item)
                result.appendChild(node)
        elif type(data) is dict:
            attrs = metadata.get('attributes', {}).get(nodename, {})
            for k, v in data.items():
                if k in attrs:
                    result.setAttribute(k, str(v))
                else:
                    node = self._to_xml_node(doc, metadata, k, v)
                    result.appendChild(node)
        else: # atom
            node = doc.createTextNode(str(data))
            result.appendChild(node)
        return result
<commit_msg>Server creation up to, but not including, network configuration<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2010 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Utility methods for working with WSGI servers
"""

import json
import logging
import sys
from xml.dom import minidom

import eventlet
import eventlet.wsgi
eventlet.patcher.monkey_patch(all=False, socket=True)
import routes
import routes.middleware
import webob
import webob.dec
import webob.exc

logging.getLogger("routes.middleware").addHandler(logging.StreamHandler())

def run_server(application, port):
    """Run a WSGI server with the given application."""
    sock = eventlet.listen(('0.0.0.0', port))
    eventlet.wsgi.server(sock, application)

class Application(object):
# TODO(gundlach): I think we should toss this class, now that it has no
# purpose.
    """Base WSGI application wrapper. Subclasses need to implement __call__."""

    def __call__(self, environ, start_response):
        r"""Subclasses will probably want to implement __call__ like this:

        @webob.dec.wsgify
        def __call__(self, req):
          # Any of the following objects work as responses:

          # Option 1: simple string
          res = 'message\n'

          # Option 2: a nicely formatted HTTP exception page
          res = exc.HTTPForbidden(detail='Nice try')

          # Option 3: a webob Response object (in case you need to play with
          # headers, or you want to be treated like an iterable, or or or)
          res = Response();
          res.app_iter = open('somefile')

          # Option 4: any wsgi app to be run next
          res = self.application

          # Option 5: you can get a Response object for a wsgi app, too, to
          # play with headers etc
          res = req.get_response(self.application)

          # You can then just return your response...
          return res
          # ... or set req.response and return None.
          req.response = res

        See the end of http://pythonpaste.org/webob/modules/dec.html
        for more info.
        """
        raise NotImplementedError("You must implement __call__")

class Middleware(Application):
    """
    Base WSGI middleware wrapper. These classes require an application to be
    initialized that will be called next.  By default the middleware will
    simply call its wrapped app, or you can override __call__ to customize its
    behavior.
    """

    def __init__(self, application): # pylint: disable-msg=W0231
        self.application = application

    @webob.dec.wsgify
    def __call__(self, req): # pylint: disable-msg=W0221
        """Override to implement middleware behavior."""
        return self.application

class Debug(Middleware):
    """Helper class that can be inserted into any WSGI application chain
    to get information about the request and response."""

    @webob.dec.wsgify
    def __call__(self, req):
        print ("*" * 40) + " REQUEST ENVIRON"
        for key, value in req.environ.items():
            print key, "=", value
        print
        resp = req.get_response(self.application)

        print ("*" * 40) + " RESPONSE HEADERS"
        for (key, value) in resp.headers.iteritems():
            print key, "=", value
        print

        resp.app_iter = self.print_generator(resp.app_iter)

        return resp

    @staticmethod
    def print_generator(app_iter):
        """
        Iterator that prints the contents of a wrapper string iterator
        when iterated.
        """
        print ("*" * 40) + " BODY"
        for part in app_iter:
            sys.stdout.write(part)
            sys.stdout.flush()
            yield part
        print

class Router(object):
    """
    WSGI middleware that maps incoming requests to WSGI apps.
    """

    def __init__(self, mapper):
        """
        Create a router for the given routes.Mapper.

        Each route in `mapper` must specify a 'controller', which is a
        WSGI app to call.  You'll probably want to specify an 'action' as
        well and have your controller be a wsgi.Controller, who will route
        the request to the action method.

        Examples:
          mapper = routes.Mapper()
          sc = ServerController()

          # Explicit mapping of one route to a controller+action
          mapper.connect(None, "/svrlist", controller=sc, action="list")

          # Actions are all implicitly defined
          mapper.resource("server", "servers", controller=sc)

          # Pointing to an arbitrary WSGI app.  You can specify the
          # {path_info:.*} parameter so the target app can be handed just that
          # section of the URL.
          mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp())
        """
        self.map = mapper
        self._router = routes.middleware.RoutesMiddleware(self._dispatch,
                                                          self.map)

    @webob.dec.wsgify
    def __call__(self, req):
        """
        Route the incoming request to a controller based on self.map.
        If no match, return a 404.
        """
        return self._router

    @staticmethod
    @webob.dec.wsgify
    def _dispatch(req):
        """
        Called by self._router after matching the incoming request to a route
        and putting the information into req.environ.  Either returns 404
        or the routed WSGI app's response.
        """
        match = req.environ['wsgiorg.routing_args'][1]
        if not match:
            return webob.exc.HTTPNotFound()
        app = match['controller']
        return app

class Controller(object):
    """
    WSGI app that reads routing information supplied by RoutesMiddleware
    and calls the requested action method upon itself.  All action methods
    must, in addition to their normal parameters, accept a 'req' argument
    which is the incoming webob.Request.  They raise a webob.exc exception,
    or return a dict which will be serialized by requested content type.
    """

    @webob.dec.wsgify
    def __call__(self, req):
        """
        Call the method specified in req.environ by RoutesMiddleware.
        """
        arg_dict = req.environ['wsgiorg.routing_args'][1]
        action = arg_dict['action']
        method = getattr(self, action)
        del arg_dict['controller']
        del arg_dict['action']
        arg_dict['req'] = req
        result = method(**arg_dict)
        if type(result) is dict:
            return self._serialize(result, req)
        else:
            return result

    def _serialize(self, data, request):
        """
        Serialize the given dict to the response type requested in request.
        Uses self._serialization_metadata if it exists, which is a dict mapping
        MIME types to information needed to serialize to that type.
        """
        _metadata = getattr(type(self), "_serialization_metadata", {})
        serializer = Serializer(request.environ, _metadata)
        return serializer.to_content_type(data)

    def _deserialize(self, data, request):
        """
        Deserialize the request body to the response type requested in request.
        Uses self._serialization_metadata if it exists, which is a dict mapping
        MIME types to information needed to serialize to that type.
        """
        _metadata = getattr(type(self), "_serialization_metadata", {})
        serializer = Serializer(request.environ, _metadata)
        return serializer.deserialize(data)

class Serializer(object):
    """
    Serializes and deserializes dictionaries to certain MIME types.
    """

    def __init__(self, environ, metadata=None):
        """
        Create a serializer based on the given WSGI environment.
        'metadata' is an optional dict mapping MIME types to information
        needed to serialize a dictionary to that type.
        """
        self.metadata = metadata or {}
        req = webob.Request(environ)
        suffix = req.path_info.split('.')[-1].lower()
        if suffix == 'json':
            self.handler = self._to_json
        elif suffix == 'xml':
            self.handler = self._to_xml
        elif 'application/json' in req.accept:
            self.handler = self._to_json
        elif 'application/xml' in req.accept:
            self.handler = self._to_xml
        else:
            self.handler = self._to_json # default

    def to_content_type(self, data):
        """
        Serialize a dictionary into a string.

        The format of the string will be decided based on the Content Type
        requested in self.environ: by Accept: header, or by URL suffix.
        """
        return self.handler(data)

    def deserialize(self, datastring):
        """
        Deserialize a string to a dictionary.

        The string must be in the format of a supported MIME type.
        """
        datastring = datastring.strip()
        try:
            is_xml = (datastring[0] == '<')
            if not is_xml:
                return json.loads(datastring)
            return self._from_xml(datastring)
        except:
            return None

    def _from_xml(self, datastring):
        xmldata = self.metadata.get('application/xml', {})
        plurals = set(xmldata.get('plurals', {}))
        node = minidom.parseString(datastring).childNodes[0]
        return {node.nodeName: self._from_xml_node(node, plurals)}

    def _from_xml_node(self, node, listnames):
        """
        Convert a minidom node to a simple Python type.

        listnames is a collection of names of XML nodes whose subnodes should
        be considered list items.
        """
        if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3:
            return node.childNodes[0].nodeValue
        elif node.nodeName in listnames:
            return [self._from_xml_node(n, listnames) for n in node.childNodes]
        else:
            result = dict()
            for attr in node.attributes.keys():
                result[attr] = node.attributes[attr].nodeValue
            for child in node.childNodes:
                if child.nodeType != node.TEXT_NODE:
                    result[child.nodeName] = self._from_xml_node(child, listnames)
            return result

    def _to_json(self, data):
        return json.dumps(data)

    def _to_xml(self, data):
        metadata = self.metadata.get('application/xml', {})
        # We expect data to contain a single key which is the XML root.
        root_key = data.keys()[0]
        doc = minidom.Document()
        node = self._to_xml_node(doc, metadata, root_key, data[root_key])
        return node.toprettyxml(indent='    ')

    def _to_xml_node(self, doc, metadata, nodename, data):
        """Recursive method to convert data members to XML nodes."""
        result = doc.createElement(nodename)
        if type(data) is list:
            singular = metadata.get('plurals', {}).get(nodename, None)
            if singular is None:
                if nodename.endswith('s'):
                    singular = nodename[:-1]
                else:
                    singular = 'item'
            for item in data:
                node = self._to_xml_node(doc, metadata, singular, item)
                result.appendChild(node)
        elif type(data) is dict:
            attrs = metadata.get('attributes', {}).get(nodename, {})
            for k, v in data.items():
                if k in attrs:
                    result.setAttribute(k, str(v))
                else:
                    node = self._to_xml_node(doc, metadata, k, v)
                    result.appendChild(node)
        else: # atom
            node = doc.createTextNode(str(data))
            result.appendChild(node)
        return result
<|endoftext|>


<commit_before>#! /usr/bin/env python

import argparse
import json
import requests
import sys

def create_rule(url, rule):
    headers = {'created-from': 'Action: ' + __name__}
    # TODO: Figure out AUTH
    resp = requests.post(url, data=json.dumps(rule), headers=headers)
    if resp.status_code not in [200, 201]:
        raise Exception('Failed creating rule in st2. status code: %s' % resp.status_code)

def _get_st2_rules_url(base_url):
    if base_url.endswith('/'):
        return base_url + 'rules'
    else:
        return base_url + '/rules'

def main(args):
    parser = argparse.ArgumentParser(description='Create a rule to that watches ' +
                                                 'for github events on a branch.')
    parser.add_argument('--branch', help='Branch to use.',
                        required=True)
    parser.add_argument('--st2-base-url', help='st2 base url.',
                        required=True)
    args = parser.parse_args()

    if args.branch in ['master']:
        sys.stderr.write('Master is not allowed branch for release.')
        sys.exit(1)

    if not args.st2_base_url:
        sys.stderr.write('st2 URL needed to create a rule.')
        sys.exit(2)

    rule_meta = {
        'name': 'st2_pytests_' + args.branch,
        'description': 'Run pytests on each commit to branch.',
        'enabled': True,
        'trigger': {
            'type': 'webhooks.github_event'
        },
        'criteria': {
            'trigger.body.ref': {
                'pattern': 'refs/heads/' + args.branch,
                'type': 'equals'
            },
            'trigger.body.repository.full_name': {
                'pattern': 'DoriftoShoes/st2',
                'type': 'equals'
            }
        },
        'action': {
            'ref': 'st2cd.pytests',
            'parameters': {
                'repo': '{{trigger.body.repository.clone_url}}',
                'revision': '{{trigger.body.head_commit.id}}',
                'branch': args.branch,
                'author': '{{trigger.body.head_commit.author.username}}'
            }
        }
    }

    try:
        create_rule(_get_st2_rules_url(args.st2_base_url), rule_meta)
        sys.stdout.write('Successfully created rule %s' % rule_meta['name'])
    except Exception as e:
        sys.stderr.write('Failed creating rule %s: %s' % (rule_meta['name'], str(e)))

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>Update a rule if it already exists<commit_after>#! /usr/bin/env python

import argparse
import json
import requests
import sys

def create_rule(url, rule):
    headers = {'created-from': 'Action: ' + __name__}
    # TODO: Figure out AUTH
    resp = requests.post(url, data=json.dumps(rule), headers=headers)

    if resp.status_code in [409]:
        resp = requests.put(url, data=json.dumps(rule), headers=headers)

    if resp.status_code not in [200, 201]:
        raise Exception('Failed creating rule in st2. status code: %s' % resp.status_code)

def _get_st2_rules_url(base_url):
    if base_url.endswith('/'):
        return base_url + 'rules'
    else:
        return base_url + '/rules'

def main(args):
    parser = argparse.ArgumentParser(description='Create a rule to that watches ' +
                                                 'for github events on a branch.')
    parser.add_argument('--branch', help='Branch to use.',
                        required=True)
    parser.add_argument('--st2-base-url', help='st2 base url.',
                        required=True)
    args = parser.parse_args()

    if args.branch in ['master']:
        sys.stderr.write('Master is not allowed branch for release.')
        sys.exit(1)

    if not args.st2_base_url:
        sys.stderr.write('st2 URL needed to create a rule.')
        sys.exit(2)

    rule_meta = {
        'name': 'st2_pytests_' + args.branch,
        'description': 'Run pytests on each commit to branch.',
        'enabled': True,
        'trigger': {
            'type': 'webhooks.github_event'
        },
        'criteria': {
            'trigger.body.ref': {
                'pattern': 'refs/heads/' + args.branch,
                'type': 'equals'
            },
            'trigger.body.repository.full_name': {
                'pattern': 'DoriftoShoes/st2',
                'type': 'equals'
            }
        },
        'action': {
            'ref': 'st2cd.pytests',
            'parameters': {
                'repo': '{{trigger.body.repository.clone_url}}',
                'revision': '{{trigger.body.head_commit.id}}',
                'branch': args.branch,
                'author': '{{trigger.body.head_commit.author.username}}'
            }
        }
    }

    try:
        create_rule(_get_st2_rules_url(args.st2_base_url), rule_meta)
        sys.stdout.write('Successfully created rule %s' % rule_meta['name'])
    except Exception as e:
        sys.stderr.write('Failed creating rule %s: %s' % (rule_meta['name'], str(e)))
        sys.exit(1)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>from .RiotWatcher import RiotWatcher

__all__ = [
    'RiotWatcher',
    'Handlers',
]
<commit_msg>changed import to be lower case, as apparently thats the case in git<commit_after>from .riotwatcher import RiotWatcher

__all__ = [
    'RiotWatcher',
    'Handlers',
]
<|endoftext|>


<commit_before>from firedrake import *
from firedrake.petsc import PETSc
from firedrake.utils import cached_property
from pyop2.profiling import timed_stage

class W5Problem(object):
    """Williamson test case 5 Problem class."""

    def __init__(self, refinement_level, R,
                 H, Dt, method="BDM",
                 hybridization=False, model_degree=2):
        super(W5Problem, self).__init__()

        self.refinement_level = refinement_level
        self.dt = Constant(Dt)
        self.Dt = Dt
        self.method = method
        self.model_degree = model_degree
        self.hybridization = hybridization

        # Mesh radius
        self.R = R

        # Earth-sized mesh
        mesh_degree = 2
        if self.method == "RTCF":
            from firedrake import CubedSphereMesh
            mesh = CubedSphereMesh(self.R, self.refinement_level,
                                   degree=mesh_degree)
        else:
            from firedrake import OctahedralSphereMesh
            # from firedrake import IcosahedralSphereMesh
            mesh = OctahedralSphereMesh(self.R, self.refinement_level,
                                        degree=mesh_degree,
                                        hemisphere="both")
            # mesh = IcosahedralSphereMesh(self.R, self.refinement_level,
            #                              degree=mesh_degree)

        global_normal = Expression(("x[0]", "x[1]", "x[2]"))
        mesh.init_cell_orientations(global_normal)
        self.mesh = mesh

        # Compatible FE spaces for velocity and depth
        if self.method == "RT":
            Vu = FunctionSpace(self.mesh, "RT", self.model_degree)
        elif self.method == "RTCF":
            Vu = FunctionSpace(self.mesh, "RTCF", self.model_degree)
        elif self.method == "BDM":
            Vu = FunctionSpace(self.mesh, "BDM", self.model_degree)
        else:
            raise ValueError("Unrecognized method '%s'" % self.method)

        VD = FunctionSpace(self.mesh, "DG", self.model_degree - 1)

        self.function_spaces = (Vu, VD)
        self.Vm = FunctionSpace(self.mesh, "CG", mesh_degree)

        # Mean depth
        self.H = Constant(H)

        # Build initial conditions and parameters
        self._build_initial_conditions()

        # Setup solvers
        self._build_DU_solvers()
        self._build_picard_solver()

        self.ksp_outer_its = []
        self.ksp_inner_its = []
        self.sim_time = []
        self.picard_seq = []
        self.reductions = []

    def _build_initial_conditions(self):

        x = SpatialCoordinate(self.mesh)
        _, VD = self.function_spaces

        # Initial conditions for velocity and depth (in geostrophic balance)
        u_0 = 20.0
        u_max = Constant(u_0)
        R0 = Constant(self.R)
        uexpr = as_vector([-u_max*x[1]/R0, u_max*x[0]/R0, 0.0])
        h0 = Constant(self.H)
        Omega = Constant(7.292e-5)
        g = Constant(9.810616)
        Dexpr = h0 - ((R0*Omega*u_max + u_max*u_max/2.0)*(x[2]*x[2]/(R0*R0)))/g

        bexpr = Expression("2000*(1 - sqrt(fmin(pow(pi/9.0,2), pow(atan2(x[1]/R0,x[0]/R0)+1.0*pi/2.0,2) + pow(asin(x[2]/R0)-pi/6.0,2)))/(pi/9.0))", R0=self.R)
        self.b = Function(VD, name="Topography").interpolate(bexpr)

        # Coriolis expression (1/s)
        fexpr = 2*Omega*x[2]/R0
        self.f = Function(self.Vm).interpolate(fexpr)

        self.g = g
        self.uexpr = uexpr
        self.Dexpr = Dexpr

    def _build_DU_solvers(self):

        Vu, VD = self.function_spaces
        un, Dn = self.state
        up, Dp = self.updates
        dt = self.dt

        # Stage 1: Depth advection
        # DG upwinded advection for depth
        self.Dps = Function(VD, name="stabilized depth")
        D = TrialFunction(VD)
        phi = TestFunction(VD)
        Dh = 0.5*(Dn + D)
        uh = 0.5*(un + up)
        n = FacetNormal(self.Vm.mesh())
        uup = 0.5*(dot(uh, n) + abs(dot(uh, n)))

        Deqn = (
            (D - Dn)*phi*dx - dt*inner(grad(phi), uh*Dh)*dx
            + dt*jump(phi)*(uup('+')*Dh('+')-uup('-')*Dh('-'))*dS
        )

        Dparams = {'ksp_type': 'cg',
                   'pc_type': 'bjacobi',
                   'sub_pc_type': 'ilu'}
        Dproblem = LinearVariationalProblem(lhs(Deqn), rhs(Deqn),
                                            self.Dps)
        Dsolver = LinearVariationalSolver(Dproblem,
                                          solver_parameters=Dparams,
                                          options_prefix="D-advection")
        self.Dsolver = Dsolver

        # Stage 2: U update
        self.Ups = Function(Vu, name="stabilized velocity")
        u = TrialFunction(Vu)
        v = TestFunction(Vu)
        Dh = 0.5*(Dn + Dp)
        ubar = 0.5*(un + up)
        uup = 0.5*(dot(ubar, n) + abs(dot(ubar, n)))
        uh = 0.5*(un + u)
        Upwind = 0.5*(sign(dot(ubar, n)) + 1)

        # Kinetic energy term (implicit midpoint)
        K = 0.5*(inner(0.5*(un + up), 0.5*(un + up)))

        both = lambda u: 2*avg(u)
        # u_t + gradperp.u + f)*perp(ubar) + grad(g*D + K)
        # <w, gradperp.u * perp(ubar)> = <perp(ubar).w, gradperp(u)>
        #                                = <-gradperp(w.perp(ubar))), u>
        #                                  +<< [[perp(n)(w.perp(ubar))]], u>>
        ueqn = (
            inner(u - un, v)*dx + dt*inner(self.perp(uh)*self.f, v)*dx
            - dt*inner(self.perp(grad(inner(v, self.perp(ubar)))), uh)*dx
            + dt*inner(both(self.perp(n)*inner(v, self.perp(ubar))),
                       both(Upwind*uh))*dS
            - dt*div(v)*(self.g*(Dh + self.b) + K)*dx
        )

        uparams = {'ksp_type': 'gmres',
                   'pc_type': 'bjacobi',
                   'sub_pc_type': 'ilu'}
        Uproblem = LinearVariationalProblem(lhs(ueqn), rhs(ueqn),
                                            self.Ups)
        Usolver = LinearVariationalSolver(Uproblem,
                                          solver_parameters=uparams,
                                          options_prefix="U-advection")
        self.Usolver = Usolver

    def _build_picard_solver(self):
        Vu, VD = self.function_spaces
        un, Dn = self.state
        up, Dp = self.updates
        dt = self.dt
        f = self.f
        g = self.g
        H = self.H

        Dps = self.Dps
        Ups = self.Ups

        # Stage 3: Implicit linear solve for u, D increments
        W = MixedFunctionSpace((Vu, VD))
        self.DU = Function(W, name="linear updates")
        w, phi = TestFunctions(W)
        du, dD = TrialFunctions(W)

        uDlhs = (
            inner(w, du + 0.5*dt*f*self.perp(du)) - 0.5*dt*div(w)*g*dD +
            phi*(dD + 0.5*dt*H*div(du))
        )*dx

        uDrhs = -(
            inner(w, up - Ups)*dx
            + phi*(Dp - Dps)*dx
        )

        self.FuD = action(uDlhs, self.DU) - uDrhs
        DUproblem = LinearVariationalProblem(uDlhs, uDrhs, self.DU,
                                             constant_jacobian=True)

        gamg_params = {'ksp_type': 'cg',
                       'pc_type': 'gamg',
                       # 'pc_gamg_reuse_interpolation': True,
                       # 'pc_gamg_sym_graph': True,
                       'ksp_rtol': 1e-8,
                       'mg_levels': {'ksp_type': 'chebyshev',
                                     # 'ksp_chebyshev_esteig': True,
                                     'ksp_max_it': 2,
                                     'pc_type': 'bjacobi',
                                     'sub_pc_type': 'ilu'}}
        if self.hybridization:
            parameters = {'ksp_type': 'preonly',
                          'mat_type': 'matfree',
                          'pmat_type': 'matfree',
                          'pc_type': 'python',
                          'pc_python_type': 'scpc.HybridizationPC',
                          'hybridization': gamg_params}

        else:
            parameters = {'ksp_type': 'gmres',
                          'pc_type': 'fieldsplit',
                          'pc_fieldsplit_type': 'schur',
                          'ksp_type': 'gmres',
                          'ksp_rtol': 1e-8,
                          'ksp_max_it': 100,
                          'ksp_gmres_restart': 50,
                          'pc_fieldsplit_schur_fact_type': 'FULL',
                          'pc_fieldsplit_schur_precondition': 'selfp',
                          'fieldsplit_0': {'ksp_type': 'preonly',
                                           'pc_type': 'bjacobi',
                                           'sub_pc_type': 'ilu'},
                          'fieldsplit_1': gamg_params}

        DUsolver = LinearVariationalSolver(DUproblem,
                                           solver_parameters=parameters,
                                           options_prefix="implicit-solve")
        self.DUsolver = DUsolver

    @cached_property
    def num_cells(self):
        return self.mesh.cell_set.size

    @cached_property
    def comm(self):
        return self.mesh.comm

    @cached_property
    def outward_normals(self):
        return CellNormal(self.mesh)

    def perp(self, u):
        return cross(self.outward_normals, u)

    @cached_property
    def eta(self):
        _, VD = self.function_spaces
        eta = Function(VD, name="Surface height")
        return eta

    @cached_property
    def state(self):
        Vu, VD = self.function_spaces
        un = Function(Vu, name="Velocity")
        Dn = Function(VD, name="Depth")
        return un, Dn

    @cached_property
    def updates(self):
        Vu, VD = self.function_spaces
        up = Function(Vu)
        Dp = Function(VD)
        return up, Dp

    @cached_property
    def output_file(self):
        dirname = "results/"
        if self.hybridization:
            dirname += "hybrid_%s_ref%d/" % (self.method,
                                             self.refinement_level)
        else:
            dirname += "gmres_%s_ref%d/" % (self.method,
                                            self.refinement_level)
        return File(dirname + "w5_" + str(self.refinement_level) + ".pvd")

    def write(self, dumpcount, dumpfreq):
        dumpcount += 1
        un, Dn = self.state
        if(dumpcount > dumpfreq):
            eta = self.eta
            eta.assign(Dn + self.b)
            self.output_file.write(un, Dn, eta, self.b)
            dumpcount -= dumpfreq
        return dumpcount

    def initialize(self):
        self.DU.assign(0.0)
        self.Ups.assign(0.0)
        self.Dps.assign(0.0)
        un, Dn = self.state
        un.project(self.uexpr)
        Dn.interpolate(self.Dexpr)
        Dn -= self.b

    def warmup(self):
        un, Dn = self.state
        up, Dp = self.updates
        up.assign(un)
        Dp.assign(Dn)

        with timed_stage("Warm up: DU Residuals"):
            self.Dsolver.solve()
            self.Usolver.solve()

        with timed_stage("Warm up: Linear solve"):
            self.DUsolver.solve()

    def run_simulation(self, tmax, write=False, dumpfreq=100):
        PETSc.Sys.Print("""
Running Williamson 5 test with parameters:\n
method: %s,\n
model degree: %d,\n
refinements: %d,\n
hybridization: %s,\n
tmax: %s
"""
                        % (self.method, self.model_degree,
                           self.refinement_level, self.hybridization, tmax))
        t = 0.0
        un, Dn = self.state
        up, Dp = self.updates
        deltau, deltaD = self.DU.split()
        dumpcount = dumpfreq
        if write:
            dumpcount = self.write(dumpcount, dumpfreq)

        self.Dsolver.snes.setConvergenceHistory()
        self.Dsolver.snes.ksp.setConvergenceHistory()
        self.Usolver.snes.setConvergenceHistory()
        self.Usolver.snes.ksp.setConvergenceHistory()
        self.DUsolver.snes.setConvergenceHistory()
        self.DUsolver.snes.ksp.setConvergenceHistory()

        while t < tmax - self.Dt/2:
            t += self.Dt

            up.assign(un)
            Dp.assign(Dn)

            for i in range(4):
                self.sim_time.append(t)
                self.picard_seq.append(i+1)

                with timed_stage("DU Residuals"):
                    self.Dsolver.solve()
                    self.Usolver.solve()

                with timed_stage("Linear solve"):
                    self.DUsolver.solve()

                # Here we collect the reductions in the linear residual.
                # Get rhs from ksp
                r0 = self.DUsolver.snes.ksp.getRhs()
                # Assemble the problem residual (b - Ax)
                res = assemble(self.FuD, mat_type="aij")
                bnorm = r0.norm()
                rnorm = res.dat.norm
                r_factor = rnorm/bnorm
                self.reductions.append(r_factor)

                up += deltau
                Dp += deltaD

                outer_ksp = self.DUsolver.snes.ksp
                if self.hybridization:
                    ctx = outer_ksp.getPC().getPythonContext()
                    inner_ksp = ctx.trace_ksp
                else:
                    ksps = outer_ksp.getPC().getFieldSplitSubKSP()
                    _, inner_ksp = ksps

                # Collect ksp iterations
                self.ksp_outer_its.append(outer_ksp.getIterationNumber())
                self.ksp_inner_its.append(inner_ksp.getIterationNumber())

            un.assign(up)
            Dn.assign(Dp)

            if write:
                with timed_stage("Dump output"):
                    dumpcount = self.write(dumpcount, dumpfreq)
<commit_msg>Delete commented lines<commit_after>from firedrake import *
from firedrake.petsc import PETSc
from firedrake.utils import cached_property
from pyop2.profiling import timed_stage

class W5Problem(object):
    """Williamson test case 5 Problem class."""

    def __init__(self, refinement_level, R,
                 H, Dt, method="BDM",
                 hybridization=False, model_degree=2):
        super(W5Problem, self).__init__()

        self.refinement_level = refinement_level
        self.dt = Constant(Dt)
        self.Dt = Dt
        self.method = method
        self.model_degree = model_degree
        self.hybridization = hybridization

        # Mesh radius
        self.R = R

        # Earth-sized mesh
        mesh_degree = 2
        if self.method == "RTCF":
            from firedrake import CubedSphereMesh
            mesh = CubedSphereMesh(self.R, self.refinement_level,
                                   degree=mesh_degree)
        else:
            from firedrake import OctahedralSphereMesh
            mesh = OctahedralSphereMesh(self.R, self.refinement_level,
                                        degree=mesh_degree,
                                        hemisphere="both")

        global_normal = Expression(("x[0]", "x[1]", "x[2]"))
        mesh.init_cell_orientations(global_normal)
        self.mesh = mesh

        # Compatible FE spaces for velocity and depth
        if self.method == "RT":
            Vu = FunctionSpace(self.mesh, "RT", self.model_degree)
        elif self.method == "RTCF":
            Vu = FunctionSpace(self.mesh, "RTCF", self.model_degree)
        elif self.method == "BDM":
            Vu = FunctionSpace(self.mesh, "BDM", self.model_degree)
        else:
            raise ValueError("Unrecognized method '%s'" % self.method)

        VD = FunctionSpace(self.mesh, "DG", self.model_degree - 1)

        self.function_spaces = (Vu, VD)
        self.Vm = FunctionSpace(self.mesh, "CG", mesh_degree)

        # Mean depth
        self.H = Constant(H)

        # Build initial conditions and parameters
        self._build_initial_conditions()

        # Setup solvers
        self._build_DU_solvers()
        self._build_picard_solver()

        self.ksp_outer_its = []
        self.ksp_inner_its = []
        self.sim_time = []
        self.picard_seq = []
        self.reductions = []

    def _build_initial_conditions(self):

        x = SpatialCoordinate(self.mesh)
        _, VD = self.function_spaces

        # Initial conditions for velocity and depth (in geostrophic balance)
        u_0 = 20.0
        u_max = Constant(u_0)
        R0 = Constant(self.R)
        uexpr = as_vector([-u_max*x[1]/R0, u_max*x[0]/R0, 0.0])
        h0 = Constant(self.H)
        Omega = Constant(7.292e-5)
        g = Constant(9.810616)
        Dexpr = h0 - ((R0*Omega*u_max + u_max*u_max/2.0)*(x[2]*x[2]/(R0*R0)))/g

        bexpr = Expression("2000*(1 - sqrt(fmin(pow(pi/9.0,2), pow(atan2(x[1]/R0,x[0]/R0)+1.0*pi/2.0,2) + pow(asin(x[2]/R0)-pi/6.0,2)))/(pi/9.0))", R0=self.R)
        self.b = Function(VD, name="Topography").interpolate(bexpr)

        # Coriolis expression (1/s)
        fexpr = 2*Omega*x[2]/R0
        self.f = Function(self.Vm).interpolate(fexpr)

        self.g = g
        self.uexpr = uexpr
        self.Dexpr = Dexpr

    def _build_DU_solvers(self):

        Vu, VD = self.function_spaces
        un, Dn = self.state
        up, Dp = self.updates
        dt = self.dt

        # Stage 1: Depth advection
        # DG upwinded advection for depth
        self.Dps = Function(VD, name="stabilized depth")
        D = TrialFunction(VD)
        phi = TestFunction(VD)
        Dh = 0.5*(Dn + D)
        uh = 0.5*(un + up)
        n = FacetNormal(self.Vm.mesh())
        uup = 0.5*(dot(uh, n) + abs(dot(uh, n)))

        Deqn = (
            (D - Dn)*phi*dx - dt*inner(grad(phi), uh*Dh)*dx
            + dt*jump(phi)*(uup('+')*Dh('+')-uup('-')*Dh('-'))*dS
        )

        Dparams = {'ksp_type': 'cg',
                   'pc_type': 'bjacobi',
                   'sub_pc_type': 'ilu'}
        Dproblem = LinearVariationalProblem(lhs(Deqn), rhs(Deqn),
                                            self.Dps)
        Dsolver = LinearVariationalSolver(Dproblem,
                                          solver_parameters=Dparams,
                                          options_prefix="D-advection")
        self.Dsolver = Dsolver

        # Stage 2: U update
        self.Ups = Function(Vu, name="stabilized velocity")
        u = TrialFunction(Vu)
        v = TestFunction(Vu)
        Dh = 0.5*(Dn + Dp)
        ubar = 0.5*(un + up)
        uup = 0.5*(dot(ubar, n) + abs(dot(ubar, n)))
        uh = 0.5*(un + u)
        Upwind = 0.5*(sign(dot(ubar, n)) + 1)

        # Kinetic energy term (implicit midpoint)
        K = 0.5*(inner(0.5*(un + up), 0.5*(un + up)))

        both = lambda u: 2*avg(u)
        # u_t + gradperp.u + f)*perp(ubar) + grad(g*D + K)
        # <w, gradperp.u * perp(ubar)> = <perp(ubar).w, gradperp(u)>
        #                                = <-gradperp(w.perp(ubar))), u>
        #                                  +<< [[perp(n)(w.perp(ubar))]], u>>
        ueqn = (
            inner(u - un, v)*dx + dt*inner(self.perp(uh)*self.f, v)*dx
            - dt*inner(self.perp(grad(inner(v, self.perp(ubar)))), uh)*dx
            + dt*inner(both(self.perp(n)*inner(v, self.perp(ubar))),
                       both(Upwind*uh))*dS
            - dt*div(v)*(self.g*(Dh + self.b) + K)*dx
        )

        uparams = {'ksp_type': 'gmres',
                   'pc_type': 'bjacobi',
                   'sub_pc_type': 'ilu'}
        Uproblem = LinearVariationalProblem(lhs(ueqn), rhs(ueqn),
                                            self.Ups)
        Usolver = LinearVariationalSolver(Uproblem,
                                          solver_parameters=uparams,
                                          options_prefix="U-advection")
        self.Usolver = Usolver

    def _build_picard_solver(self):
        Vu, VD = self.function_spaces
        un, Dn = self.state
        up, Dp = self.updates
        dt = self.dt
        f = self.f
        g = self.g
        H = self.H

        Dps = self.Dps
        Ups = self.Ups

        # Stage 3: Implicit linear solve for u, D increments
        W = MixedFunctionSpace((Vu, VD))
        self.DU = Function(W, name="linear updates")
        w, phi = TestFunctions(W)
        du, dD = TrialFunctions(W)

        uDlhs = (
            inner(w, du + 0.5*dt*f*self.perp(du)) - 0.5*dt*div(w)*g*dD +
            phi*(dD + 0.5*dt*H*div(du))
        )*dx

        uDrhs = -(
            inner(w, up - Ups)*dx
            + phi*(Dp - Dps)*dx
        )

        self.FuD = action(uDlhs, self.DU) - uDrhs
        DUproblem = LinearVariationalProblem(uDlhs, uDrhs, self.DU,
                                             constant_jacobian=True)

        gamg_params = {'ksp_type': 'cg',
                       'pc_type': 'gamg',
                       'pc_gamg_reuse_interpolation': True,
                       'ksp_rtol': 1e-8,
                       'mg_levels': {'ksp_type': 'chebyshev',
                                     'ksp_max_it': 2,
                                     'pc_type': 'bjacobi',
                                     'sub_pc_type': 'ilu'}}
        if self.hybridization:
            parameters = {'ksp_type': 'preonly',
                          'mat_type': 'matfree',
                          'pmat_type': 'matfree',
                          'pc_type': 'python',
                          'pc_python_type': 'scpc.HybridizationPC',
                          'hybridization': gamg_params}

        else:
            parameters = {'ksp_type': 'gmres',
                          'pc_type': 'fieldsplit',
                          'pc_fieldsplit_type': 'schur',
                          'ksp_type': 'gmres',
                          'ksp_rtol': 1e-8,
                          'ksp_max_it': 100,
                          'ksp_gmres_restart': 50,
                          'pc_fieldsplit_schur_fact_type': 'FULL',
                          'pc_fieldsplit_schur_precondition': 'selfp',
                          'fieldsplit_0': {'ksp_type': 'preonly',
                                           'pc_type': 'bjacobi',
                                           'sub_pc_type': 'ilu'},
                          'fieldsplit_1': gamg_params}

        DUsolver = LinearVariationalSolver(DUproblem,
                                           solver_parameters=parameters,
                                           options_prefix="implicit-solve")
        self.DUsolver = DUsolver

    @cached_property
    def num_cells(self):
        return self.mesh.cell_set.size

    @cached_property
    def comm(self):
        return self.mesh.comm

    @cached_property
    def outward_normals(self):
        return CellNormal(self.mesh)

    def perp(self, u):
        return cross(self.outward_normals, u)

    @cached_property
    def eta(self):
        _, VD = self.function_spaces
        eta = Function(VD, name="Surface height")
        return eta

    @cached_property
    def state(self):
        Vu, VD = self.function_spaces
        un = Function(Vu, name="Velocity")
        Dn = Function(VD, name="Depth")
        return un, Dn

    @cached_property
    def updates(self):
        Vu, VD = self.function_spaces
        up = Function(Vu)
        Dp = Function(VD)
        return up, Dp

    @cached_property
    def output_file(self):
        dirname = "results/"
        if self.hybridization:
            dirname += "hybrid_%s_ref%d/" % (self.method,
                                             self.refinement_level)
        else:
            dirname += "gmres_%s_ref%d/" % (self.method,
                                            self.refinement_level)
        return File(dirname + "w5_" + str(self.refinement_level) + ".pvd")

    def write(self, dumpcount, dumpfreq):
        dumpcount += 1
        un, Dn = self.state
        if(dumpcount > dumpfreq):
            eta = self.eta
            eta.assign(Dn + self.b)
            self.output_file.write(un, Dn, eta, self.b)
            dumpcount -= dumpfreq
        return dumpcount

    def initialize(self):
        self.DU.assign(0.0)
        self.Ups.assign(0.0)
        self.Dps.assign(0.0)
        un, Dn = self.state
        un.project(self.uexpr)
        Dn.interpolate(self.Dexpr)
        Dn -= self.b

    def warmup(self):
        un, Dn = self.state
        up, Dp = self.updates
        up.assign(un)
        Dp.assign(Dn)

        with timed_stage("Warm up: DU Residuals"):
            self.Dsolver.solve()
            self.Usolver.solve()

        with timed_stage("Warm up: Linear solve"):
            self.DUsolver.solve()

    def run_simulation(self, tmax, write=False, dumpfreq=100):
        PETSc.Sys.Print("""
Running Williamson 5 test with parameters:\n
method: %s,\n
model degree: %d,\n
refinements: %d,\n
hybridization: %s,\n
tmax: %s
"""
                        % (self.method, self.model_degree,
                           self.refinement_level, self.hybridization, tmax))
        t = 0.0
        un, Dn = self.state
        up, Dp = self.updates
        deltau, deltaD = self.DU.split()
        dumpcount = dumpfreq
        if write:
            dumpcount = self.write(dumpcount, dumpfreq)

        self.Dsolver.snes.setConvergenceHistory()
        self.Dsolver.snes.ksp.setConvergenceHistory()
        self.Usolver.snes.setConvergenceHistory()
        self.Usolver.snes.ksp.setConvergenceHistory()
        self.DUsolver.snes.setConvergenceHistory()
        self.DUsolver.snes.ksp.setConvergenceHistory()

        while t < tmax - self.Dt/2:
            t += self.Dt

            up.assign(un)
            Dp.assign(Dn)

            for i in range(4):
                self.sim_time.append(t)
                self.picard_seq.append(i+1)

                with timed_stage("DU Residuals"):
                    self.Dsolver.solve()
                    self.Usolver.solve()

                with timed_stage("Linear solve"):
                    self.DUsolver.solve()

                # Here we collect the reductions in the linear residual.
                # Get rhs from ksp
                r0 = self.DUsolver.snes.ksp.getRhs()
                # Assemble the problem residual (b - Ax)
                res = assemble(self.FuD, mat_type="aij")
                bnorm = r0.norm()
                rnorm = res.dat.norm
                r_factor = rnorm/bnorm
                self.reductions.append(r_factor)

                up += deltau
                Dp += deltaD

                outer_ksp = self.DUsolver.snes.ksp
                if self.hybridization:
                    ctx = outer_ksp.getPC().getPythonContext()
                    inner_ksp = ctx.trace_ksp
                else:
                    ksps = outer_ksp.getPC().getFieldSplitSubKSP()
                    _, inner_ksp = ksps

                # Collect ksp iterations
                self.ksp_outer_its.append(outer_ksp.getIterationNumber())
                self.ksp_inner_its.append(inner_ksp.getIterationNumber())

            un.assign(up)
            Dn.assign(Dp)

            if write:
                with timed_stage("Dump output"):
                    dumpcount = self.write(dumpcount, dumpfreq)
<|endoftext|>


<commit_before># Copyright (C) 2013 Savoir-faire Linux (<http://www.savoirfairelinux.com>).
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Document Page Approval",
    "version": "13.0.1.0.0",
    "author": "Savoir-faire Linux, Odoo Community Association (OCA)",
    "website": "http://www.savoirfairelinux.com",
    "license": "AGPL-3",
    "category": "Knowledge Management",
    "depends": ["document_page", "mail"],
    "data": [
        "data/email_template.xml",
        "views/document_page_approval.xml",
        "security/document_page_security.xml",
        "security/ir.model.access.csv",
    ],
    "images": [
        "images/category.png",
        "images/page_history_list.png",
        "images/page_history.png",
    ],
    "post_init_hook": "post_init_hook",
    "uninstall_hook": "uninstall_hook",
    "installable": True,
}
<commit_msg>document_page_approval 13.0.1.0.1<commit_after># Copyright (C) 2013 Savoir-faire Linux (<http://www.savoirfairelinux.com>).
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Document Page Approval",
    "version": "13.0.1.0.1",
    "author": "Savoir-faire Linux, Odoo Community Association (OCA)",
    "website": "http://www.savoirfairelinux.com",
    "license": "AGPL-3",
    "category": "Knowledge Management",
    "depends": ["document_page", "mail"],
    "data": [
        "data/email_template.xml",
        "views/document_page_approval.xml",
        "security/document_page_security.xml",
        "security/ir.model.access.csv",
    ],
    "images": [
        "images/category.png",
        "images/page_history_list.png",
        "images/page_history.png",
    ],
    "post_init_hook": "post_init_hook",
    "uninstall_hook": "uninstall_hook",
    "installable": True,
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from schematics.types import StringType
from schematics.exceptions import ValidationError
from zope.interface import implementer
from schematics.types.compound import ModelType
from openprocurement.api.models import ITender
from openprocurement.tender.openua.models import SifterListType, Item as BaseItem
from openprocurement.tender.openeu.models import (Tender as TenderEU, Administrator_bid_role, view_bid_role,
                                                  Organization as BaseOrganization, Bid as BidEU, ConfidentialDocument)
from openprocurement.api.models import (
    plain_role, create_role, edit_role, view_role, listing_role,
    enquiries_role, validate_cpv_group, validate_items_uniq,
    chronograph_role, chronograph_view_role,
    Identifier as BaseIdentifier, ContactPoint as BaseContactPoint,
    Administrator_role, schematics_default_role,
    schematics_embedded_role, ListType, BooleanType
)
from schematics.transforms import whitelist, blacklist
from openprocurement.tender.openeu.models import pre_qualifications_role

edit_role_ua = edit_role + blacklist('enquiryPeriod', 'status')

roles = {
    'plain': plain_role,
    'create': create_role,
    'view': view_role,
    'listing': listing_role,
    'active.pre-qualification': pre_qualifications_role,
    'active.pre-qualification.stand-still': pre_qualifications_role,
    'active.stage2.pending': enquiries_role,
    'active.stage2.waiting': pre_qualifications_role,
    'edit_active.stage2.pending': pre_qualifications_role,
    'draft': enquiries_role,
    'active.tendering': enquiries_role,
    'complete': view_role,
    'unsuccessful': view_role,
    'cancelled': view_role,
    'chronograph': chronograph_role,
    'chronograph_view': chronograph_view_role,
    'Administrator': Administrator_role,
    'default': schematics_default_role,
    'contracting': whitelist('doc_id', 'owner'),
}

class Organization(BaseOrganization):
    """Redefinition model Organization for UA dialogue"""

    name_en = StringType()  # not required for UA dialogue
    identifier = ModelType(BaseIdentifier, required=True)
    contactPoint = ModelType(BaseContactPoint, required=True)

class ProcuringEntity(Organization):
    """Redefinition model ProcuringEntity for UA dialogue"""

    class Options:
        roles = {
            'embedded': schematics_embedded_role,
            'view': schematics_default_role,
            'edit_active.tendering': schematics_default_role + blacklist("kind"),
        }

    kind = StringType(choices=['general', 'special', 'defense', 'other'])

class Document(ConfidentialDocument):
    """ Document model with new feature as Description of the decision to purchase """

    class Options:
        roles = {
            'edit': blacklist('id', 'url', 'datePublished', 'dateModified', ''),
            'embedded': schematics_embedded_role,
            'view': (blacklist('revisions') + schematics_default_role),
            'restricted_view': (blacklist('revisions', 'url') + schematics_default_role),
            'revisions': whitelist('url', 'dateModified'),
        }

    isDescriptionDecision = BooleanType(default=False)

    def validate_confidentialityRationale(self, data, val):
        if data['confidentiality'] != 'public' and not data['isDescriptionDecision']:
            if not val:
                raise ValidationError(u"confidentialityRationale is required")
            elif len(val) < 30:
                raise ValidationError(u"confidentialityRationale should contain at least 30 characters")

class Bid(BidEU):
    class Options:
        roles = {
            'Administrator': Administrator_bid_role,
            'embedded': view_bid_role,
            'view': view_bid_role,
            'create': whitelist('value', 'tenderers', 'parameters', 'lotValues',
                                'status', 'selfQualified', 'selfEligible', 'subcontractingDetails'),
            'edit': whitelist('value', 'tenderers', 'parameters', 'lotValues', 'status', 'subcontractingDetails'),
            'active.enquiries': whitelist(),
            'active.tendering': whitelist(),
            'active.pre-qualification': whitelist('id', 'status', 'documents', 'tenderers'),
            'active.pre-qualification.stand-still': whitelist('id', 'status', 'documents', 'tenderers'),
            'active.auction': whitelist('id', 'status', 'documents', 'tenderers'),
            'active.stage2.pending': whitelist('id', 'status', 'documents', 'tenderers'),
            'active.qualification': view_bid_role,
            'complete': view_bid_role,
            'unsuccessful': view_bid_role,
            'bid.unsuccessful': whitelist('id', 'status', 'tenderers', 'parameters',
                                          'selfQualified', 'selfEligible', 'subcontractingDetails'),
            'cancelled': view_bid_role,
            'invalid': whitelist('id', 'status'),
            'deleted': whitelist('id', 'status'),
        }

    documents = ListType(ModelType(Document), default=list())

@implementer(ITender)
class Tender(TenderEU):
    procurementMethodType = StringType(default="competitiveDialogue.aboveThresholdEU")
    status = StringType(choices=['draft', 'active.tendering', 'active.pre-qualification',
                                 'active.pre-qualification.stand-still', 'active.stage2.pending',
                                 'active.stage2.waiting', 'complete', 'cancelled', 'unsuccessful'],
                        default='active.tendering')
    # A list of all the companies who entered submissions for the tender.
    bids = SifterListType(ModelType(Bid), default=list(),
                          filter_by='status', filter_in_values=['invalid', 'deleted'])
    TenderID = StringType(required=False)

    class Options:
        roles = roles.copy()

CompetitiveDialogEU = Tender

@implementer(ITender)
class Tender(CompetitiveDialogEU):
    procurementMethodType = StringType(default="competitiveDialogue.aboveThresholdUA")
    title_en = StringType()
    items = ListType(ModelType(BaseItem), required=True, min_size=1,
                     validators=[validate_cpv_group, validate_items_uniq])
    procuringEntity = ModelType(ProcuringEntity, required=True)

CompetitiveDialogUA = Tender
<commit_msg>Change inheritance in UA dialogue for field "procuringEntity" (-> api.models)<commit_after># -*- coding: utf-8 -*-
from schematics.types import StringType
from schematics.exceptions import ValidationError
from zope.interface import implementer
from schematics.types.compound import ModelType
from openprocurement.api.models import ITender
from openprocurement.tender.openua.models import SifterListType, Item as BaseItem
from openprocurement.tender.openeu.models import (Tender as TenderEU, Administrator_bid_role, view_bid_role,
                                                  Organization as BaseOrganization, Bid as BidEU, ConfidentialDocument)
from openprocurement.api.models import (
    plain_role, create_role, edit_role, view_role, listing_role,
    enquiries_role, validate_cpv_group, validate_items_uniq,
    chronograph_role, chronograph_view_role, ProcuringEntity as BaseProcuringEntity,
    Administrator_role, schematics_default_role,
    schematics_embedded_role, ListType, BooleanType
)
from schematics.transforms import whitelist, blacklist
from openprocurement.tender.openeu.models import pre_qualifications_role

edit_role_ua = edit_role + blacklist('enquiryPeriod', 'status')

roles = {
    'plain': plain_role,
    'create': create_role,
    'view': view_role,
    'listing': listing_role,
    'active.pre-qualification': pre_qualifications_role,
    'active.pre-qualification.stand-still': pre_qualifications_role,
    'active.stage2.pending': enquiries_role,
    'active.stage2.waiting': pre_qualifications_role,
    'edit_active.stage2.pending': pre_qualifications_role,
    'draft': enquiries_role,
    'active.tendering': enquiries_role,
    'complete': view_role,
    'unsuccessful': view_role,
    'cancelled': view_role,
    'chronograph': chronograph_role,
    'chronograph_view': chronograph_view_role,
    'Administrator': Administrator_role,
    'default': schematics_default_role,
    'contracting': whitelist('doc_id', 'owner'),
}

class Document(ConfidentialDocument):
    """ Document model with new feature as Description of the decision to purchase """

    class Options:
        roles = {
            'edit': blacklist('id', 'url', 'datePublished', 'dateModified', ''),
            'embedded': schematics_embedded_role,
            'view': (blacklist('revisions') + schematics_default_role),
            'restricted_view': (blacklist('revisions', 'url') + schematics_default_role),
            'revisions': whitelist('url', 'dateModified'),
        }

    isDescriptionDecision = BooleanType(default=False)

    def validate_confidentialityRationale(self, data, val):
        if data['confidentiality'] != 'public' and not data['isDescriptionDecision']:
            if not val:
                raise ValidationError(u"confidentialityRationale is required")
            elif len(val) < 30:
                raise ValidationError(u"confidentialityRationale should contain at least 30 characters")

class Bid(BidEU):
    class Options:
        roles = {
            'Administrator': Administrator_bid_role,
            'embedded': view_bid_role,
            'view': view_bid_role,
            'create': whitelist('value', 'tenderers', 'parameters', 'lotValues',
                                'status', 'selfQualified', 'selfEligible', 'subcontractingDetails'),
            'edit': whitelist('value', 'tenderers', 'parameters', 'lotValues', 'status', 'subcontractingDetails'),
            'active.enquiries': whitelist(),
            'active.tendering': whitelist(),
            'active.pre-qualification': whitelist('id', 'status', 'documents', 'tenderers'),
            'active.pre-qualification.stand-still': whitelist('id', 'status', 'documents', 'tenderers'),
            'active.auction': whitelist('id', 'status', 'documents', 'tenderers'),
            'active.stage2.pending': whitelist('id', 'status', 'documents', 'tenderers'),
            'active.qualification': view_bid_role,
            'complete': view_bid_role,
            'unsuccessful': view_bid_role,
            'bid.unsuccessful': whitelist('id', 'status', 'tenderers', 'parameters',
                                          'selfQualified', 'selfEligible', 'subcontractingDetails'),
            'cancelled': view_bid_role,
            'invalid': whitelist('id', 'status'),
            'deleted': whitelist('id', 'status'),
        }

    documents = ListType(ModelType(Document), default=list())

@implementer(ITender)
class Tender(TenderEU):
    procurementMethodType = StringType(default="competitiveDialogue.aboveThresholdEU")
    status = StringType(choices=['draft', 'active.tendering', 'active.pre-qualification',
                                 'active.pre-qualification.stand-still', 'active.stage2.pending',
                                 'active.stage2.waiting', 'complete', 'cancelled', 'unsuccessful'],
                        default='active.tendering')
    # A list of all the companies who entered submissions for the tender.
    bids = SifterListType(ModelType(Bid), default=list(),
                          filter_by='status', filter_in_values=['invalid', 'deleted'])
    TenderID = StringType(required=False)

    class Options:
        roles = roles.copy()

CompetitiveDialogEU = Tender

@implementer(ITender)
class Tender(CompetitiveDialogEU):
    procurementMethodType = StringType(default="competitiveDialogue.aboveThresholdUA")
    title_en = StringType()
    items = ListType(ModelType(BaseItem), required=True, min_size=1,
                     validators=[validate_cpv_group, validate_items_uniq])
    procuringEntity = ModelType(BaseProcuringEntity, required=True)

CompetitiveDialogUA = Tender
<|endoftext|>