<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
# -*- coding: utf-8 -*-

from heat.engine import resources
from heat.engine import resource
from heat.openstack.common.gettextutils import _

from docutils import nodes
from sphinx.util.compat import Directive

class resourcepages(nodes.General, nodes.Element): pass

class ResourcePages(Directive):
    has_content = False
    required_arguments = 0
    optional_arguments = 1
    final_argument_whitespace = False
    option_spec = {}

    def run(self):
        prefix = self.arguments and self.arguments.pop() or None
        content = []
        for resource_type, resource_class in _all_resources(prefix):
            self.resource_type = resource_type
            self.resource_class = resource_class
            resourceid = 'resource-%s' % resource_type
            section = nodes.section(ids=[resourceid])
            content.append(section)

            title = nodes.title('', resource_type)
            section.append(title)

            cls_doc = resource_class.__doc__
            if cls_doc:
                para = nodes.paragraph('', cls_doc)
                section.append(para)

            self.contribute_hot_syntax(section)
            self.contribute_yaml_syntax(section)
            self.contribute_json_syntax(section)
            self.contribute_properties(section)
            self.contribute_attributes(section)

        return content

    def _prop_syntax_example(self, prop):
        if not prop or not prop.get('Type'):
            return 'Value'
        prop_type=prop.get('Type')
        if prop_type == 'List':
            sub_prop = prop.get('Schema')
            sub_type = self._prop_syntax_example(sub_prop)
            return '[%s, %s, ...]' % (sub_type, sub_type)
        elif prop_type == 'Map':
            sub_prop = prop.get('Schema', {})
            sub_props = []
            for sub_key, sub_value in sub_prop.items():
                if sub_value.get('Implemented', True):
                    sub_props.append('"%s": %s' % (
                        sub_key, self._prop_syntax_example(sub_value)))
            return '{%s}' % ', '.join(sub_props or ['...'])
        else:
            return prop_type

    def contribute_hot_syntax(self, section):
        section.append(nodes.strong('', _('HOT Syntax')))
        schema = self.resource_class.properties_schema
        props = []
        for prop_key in sorted(schema.keys()):
            prop = schema[prop_key]
            if prop.get('Implemented', True):
                props.append('%s: %s' % (prop_key, self._prop_syntax_example(prop)))

        template = '''heat_template_version: 2013-05-23
...
resources:
  ...
  the_resource:
    type: %s
    properties:
      %s''' % (self.resource_type, '\n      '.join(props))

        block = nodes.literal_block('', template)
        section.append(block)

    def contribute_yaml_syntax(self, section):
        section.append(nodes.strong('', _('YAML Syntax')))
        schema = self.resource_class.properties_schema
        props = []
        for prop_key in sorted(schema.keys()):
            prop = schema[prop_key]
            if prop.get('Implemented', True):
                props.append('%s: %s' % (prop_key, self._prop_syntax_example(prop)))

        template = '''HeatTemplateFormatVersion: '2012-12-12'
...
Resources:
  ...
  TheResource:
    Type: %s
    Properties:
      %s''' % (self.resource_type, '\n      '.join(props))

        block = nodes.literal_block('', template)
        section.append(block)

    def contribute_json_syntax(self, section):
        section.append(nodes.strong('', _('JSON Syntax')))
        schema = self.resource_class.properties_schema

        props = []
        for prop_key in sorted(schema.keys()):
            prop = schema[prop_key]
            if prop.get('Implemented', True):
                props.append('"%s": %s' % (prop_key, self._prop_syntax_example(prop)))
        template = '''{
  "AWSTemplateFormatVersion" : "2010-09-09",
  ...
  "Resources" : {
    "TheResource": {
      "Type": "%s",
      "Properties": {
        %s
      }
    }
  }
}''' % (self.resource_type, ',\n        '.join(props))
        block = nodes.literal_block('', template)
        section.append(block)

    def contribute_property(self, prop_list, prop_key, prop):
        prop_item = nodes.definition_list_item(
            '',nodes.term('', prop_key))
        prop_list.append(prop_item)

        prop_type = prop.get('Type')
        classifier = prop_type
        if prop.get('MinValue'):
            classifier += _(' from %s') % prop.get('MinValue')
        if prop.get('MaxValue'):
            classifier += _(' up to %s') % prop.get('MaxValue')
        if prop.get('MinLength'):
            classifier += _(' from length %s') % prop.get('MinLength')
        if prop.get('MaxLength'):
            classifier += _(' up to length %s') % prop.get('MaxLength')
        prop_item.append(nodes.classifier('', classifier))

        definition = nodes.definition()
        prop_item.append(definition)

        if not prop.get('Implemented', True):
            para = nodes.inline('', _('Not implemented.'))
            warning = nodes.note('', para)
            definition.append(warning)
            return

        description = prop.get('Description')
        if description:
            para = nodes.paragraph('', description)
            definition.append(para)

        if prop.get('Required'):
            para = nodes.paragraph('', _('Required property.'))
        elif prop.get('Default'):
            para = nodes.paragraph(
                '',
                _('Optional property, defaults to "%s".') % prop.get('Default'))
        else:
            para = nodes.paragraph('', _('Optional property.'))
        definition.append(para)

        if prop.get('AllowedPattern'):
            para = nodes.paragraph('', _(
                'Value must match pattern: %s') % prop.get('AllowedPattern'))
            definition.append(para)

        if prop.get('AllowedValues'):
            allowed = [str(a) for a in prop.get('AllowedValues') if a is not None]
            para = nodes.paragraph('', _(
                'Allowed values: %s') % ', '.join(allowed))
            definition.append(para)

        sub_schema = None
        if prop.get('Schema') and prop_type == 'Map':
            para = nodes.emphasis('', _('Map properties:'))
            definition.append(para)
            sub_schema = prop.get('Schema')

        elif prop_type == 'List' and prop.get('Schema', {}).get('Schema'):
            para = nodes.emphasis(
                '', _('List contains maps with the properties:'))
            definition.append(para)
            sub_schema = prop.get('Schema').get('Schema')

        if sub_schema:
            sub_prop_list = nodes.definition_list()
            definition.append(sub_prop_list)
            for sub_prop_key in sorted(sub_schema.keys()):
                sub_prop = sub_schema[sub_prop_key]
                self.contribute_property(sub_prop_list, sub_prop_key, sub_prop)

    def contribute_properties(self, section):
        schema = self.resource_class.properties_schema
        if not schema:
            return
        section.append(nodes.strong('', _('Properties')))
        prop_list = nodes.definition_list()
        section.append(prop_list)
        for prop_key in sorted(schema.keys()):
            prop = schema[prop_key]
            self.contribute_property(prop_list, prop_key, prop)

    def contribute_attributes(self, section):
        schema = self.resource_class.attributes_schema
        if not schema:
            return

        section.append(nodes.strong('', _('Attributes')))
        prop_list = nodes.definition_list()
        section.append(prop_list)
        for prop_key in sorted(schema.keys()):
            description = schema[prop_key]
            prop_item = nodes.definition_list_item(
                '',nodes.term('', prop_key))
            prop_list.append(prop_item)

            definition = nodes.definition()
            prop_item.append(definition)

            if description:
                def_para = nodes.paragraph('', description)
                definition.append(def_para)

def _all_resources(prefix=None):
    all_resources = resource._resource_classes
    for resource_type in sorted(all_resources.keys()):
        resource_class = all_resources[resource_type]
        if not prefix or resource_type.startswith(prefix):
            yield resource_type, resource_class

def setup(app):

    resources.initialise()
    app.add_node(resourcepages)

    app.add_directive('resourcepages', ResourcePages)

<commit_msg>Use subsections in resource doc generation.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
# -*- coding: utf-8 -*-

from heat.engine import resources
from heat.engine import resource
from heat.openstack.common.gettextutils import _

from docutils import nodes
from sphinx.util.compat import Directive

class resourcepages(nodes.General, nodes.Element): pass

class ResourcePages(Directive):
    has_content = False
    required_arguments = 0
    optional_arguments = 1
    final_argument_whitespace = False
    option_spec = {}

    def run(self):
        prefix = self.arguments and self.arguments.pop() or None
        content = []
        for resource_type, resource_class in _all_resources(prefix):
            self.resource_type = resource_type
            self.resource_class = resource_class
            section = self._section(content, resource_type, '%s')

            cls_doc = resource_class.__doc__
            if cls_doc:
                para = nodes.paragraph('', cls_doc)
                section.append(para)

            self.contribute_properties(section)
            self.contribute_attributes(section)

            self.contribute_hot_syntax(section)
            self.contribute_yaml_syntax(section)
            self.contribute_json_syntax(section)

        return content

    def _section(self, parent, title, id_pattern):
        id = id_pattern % self.resource_type
        section = nodes.section(ids=[id])
        parent.append(section)
        title = nodes.title('', title)
        section.append(title)
        return section

    def _prop_syntax_example(self, prop):
        if not prop or not prop.get('Type'):
            return 'Value'
        prop_type=prop.get('Type')
        if prop_type == 'List':
            sub_prop = prop.get('Schema')
            sub_type = self._prop_syntax_example(sub_prop)
            return '[%s, %s, ...]' % (sub_type, sub_type)
        elif prop_type == 'Map':
            sub_prop = prop.get('Schema', {})
            sub_props = []
            for sub_key, sub_value in sub_prop.items():
                if sub_value.get('Implemented', True):
                    sub_props.append('"%s": %s' % (
                        sub_key, self._prop_syntax_example(sub_value)))
            return '{%s}' % ', '.join(sub_props or ['...'])
        else:
            return prop_type

    def contribute_hot_syntax(self, parent):
        section = self._section(parent, _('HOT Syntax'), '%s-hot')
        schema = self.resource_class.properties_schema
        props = []
        for prop_key in sorted(schema.keys()):
            prop = schema[prop_key]
            if prop.get('Implemented', True):
                props.append('%s: %s' % (prop_key, self._prop_syntax_example(prop)))

        template = '''heat_template_version: 2013-05-23
...
resources:
  ...
  the_resource:
    type: %s
    properties:
      %s''' % (self.resource_type, '\n      '.join(props))

        block = nodes.literal_block('', template)
        section.append(block)

    def contribute_yaml_syntax(self, parent):
        section = self._section(parent, _('YAML Syntax'), '%s-yaml')
        schema = self.resource_class.properties_schema
        props = []
        for prop_key in sorted(schema.keys()):
            prop = schema[prop_key]
            if prop.get('Implemented', True):
                props.append('%s: %s' % (prop_key, self._prop_syntax_example(prop)))

        template = '''HeatTemplateFormatVersion: '2012-12-12'
...
Resources:
  ...
  TheResource:
    Type: %s
    Properties:
      %s''' % (self.resource_type, '\n      '.join(props))

        block = nodes.literal_block('', template)
        section.append(block)

    def contribute_json_syntax(self, parent):
        section = self._section(parent, _('JSON Syntax'), '%s-json')
        schema = self.resource_class.properties_schema

        props = []
        for prop_key in sorted(schema.keys()):
            prop = schema[prop_key]
            if prop.get('Implemented', True):
                props.append('"%s": %s' % (prop_key, self._prop_syntax_example(prop)))
        template = '''{
  "AWSTemplateFormatVersion" : "2010-09-09",
  ...
  "Resources" : {
    "TheResource": {
      "Type": "%s",
      "Properties": {
        %s
      }
    }
  }
}''' % (self.resource_type, ',\n        '.join(props))
        block = nodes.literal_block('', template)
        section.append(block)

    def contribute_property(self, prop_list, prop_key, prop):
        prop_item = nodes.definition_list_item(
            '',nodes.term('', prop_key))
        prop_list.append(prop_item)

        prop_type = prop.get('Type')
        classifier = prop_type
        if prop.get('MinValue'):
            classifier += _(' from %s') % prop.get('MinValue')
        if prop.get('MaxValue'):
            classifier += _(' up to %s') % prop.get('MaxValue')
        if prop.get('MinLength'):
            classifier += _(' from length %s') % prop.get('MinLength')
        if prop.get('MaxLength'):
            classifier += _(' up to length %s') % prop.get('MaxLength')
        prop_item.append(nodes.classifier('', classifier))

        definition = nodes.definition()
        prop_item.append(definition)

        if not prop.get('Implemented', True):
            para = nodes.inline('', _('Not implemented.'))
            warning = nodes.note('', para)
            definition.append(warning)
            return

        description = prop.get('Description')
        if description:
            para = nodes.paragraph('', description)
            definition.append(para)

        if prop.get('Required'):
            para = nodes.paragraph('', _('Required property.'))
        elif prop.get('Default'):
            para = nodes.paragraph(
                '',
                _('Optional property, defaults to "%s".') % prop.get('Default'))
        else:
            para = nodes.paragraph('', _('Optional property.'))
        definition.append(para)

        if prop.get('AllowedPattern'):
            para = nodes.paragraph('', _(
                'Value must match pattern: %s') % prop.get('AllowedPattern'))
            definition.append(para)

        if prop.get('AllowedValues'):
            allowed = [str(a) for a in prop.get('AllowedValues') if a is not None]
            para = nodes.paragraph('', _(
                'Allowed values: %s') % ', '.join(allowed))
            definition.append(para)

        sub_schema = None
        if prop.get('Schema') and prop_type == 'Map':
            para = nodes.emphasis('', _('Map properties:'))
            definition.append(para)
            sub_schema = prop.get('Schema')

        elif prop_type == 'List' and prop.get('Schema', {}).get('Schema'):
            para = nodes.emphasis(
                '', _('List contains maps with the properties:'))
            definition.append(para)
            sub_schema = prop.get('Schema').get('Schema')

        if sub_schema:
            sub_prop_list = nodes.definition_list()
            definition.append(sub_prop_list)
            for sub_prop_key in sorted(sub_schema.keys()):
                sub_prop = sub_schema[sub_prop_key]
                self.contribute_property(sub_prop_list, sub_prop_key, sub_prop)

    def contribute_properties(self, parent):
        schema = self.resource_class.properties_schema
        if not schema:
            return
        section = self._section(parent, _('Properties'), '%s-props')
        prop_list = nodes.definition_list()
        section.append(prop_list)
        for prop_key in sorted(schema.keys()):
            prop = schema[prop_key]
            self.contribute_property(prop_list, prop_key, prop)

    def contribute_attributes(self, parent):
        schema = self.resource_class.attributes_schema
        if not schema:
            return
        section = self._section(parent, _('Attributes'), '%s-attrs')
        prop_list = nodes.definition_list()
        section.append(prop_list)
        for prop_key in sorted(schema.keys()):
            description = schema[prop_key]
            prop_item = nodes.definition_list_item(
                '',nodes.term('', prop_key))
            prop_list.append(prop_item)

            definition = nodes.definition()
            prop_item.append(definition)

            if description:
                def_para = nodes.paragraph('', description)
                definition.append(def_para)

def _all_resources(prefix=None):
    all_resources = resource._resource_classes
    for resource_type in sorted(all_resources.keys()):
        resource_class = all_resources[resource_type]
        if not prefix or resource_type.startswith(prefix):
            yield resource_type, resource_class

def setup(app):

    resources.initialise()
    app.add_node(resourcepages)

    app.add_directive('resourcepages', ResourcePages)

<|endoftext|>


<commit_before>from django.http import HttpResponse, Http404
from django.shortcuts import render, render_to_response, redirect
from django.views.generic import TemplateView
from django.contrib.auth import authenticate, login, logout
from django.contrib import messages

from game.models import Player, Room

class RootView(TemplateView):
    def get(self, request, **kwargs):
        return HttpResponse("hello")

    def post(self, request, **kwargs):
        pass

class RoomView(TemplateView):
    def get(self, request, **kwargs):
        if not request.user.is_authenticated():
            return HttpResponse("Not authenticated!")

        player = Player.objects.filter(user=request.user).first()

        room = player.current_room
        resp = "
{0}
{1}".format(room.name, room.description)

        resp += "
Exits
"
        for exit in room.exits.all():
            resp += exit.name + "
"

        return HttpResponse(resp)

    def post(self, request, **kwargs):
        pass
<commit_msg>Added command view<commit_after>from django.http import HttpResponse, Http404
from django.shortcuts import render, render_to_response, redirect
from django.views.generic import TemplateView, View
from django.contrib.auth import authenticate, login, logout
from django.contrib import messages

from game.models import Player, Room

class RootView(TemplateView):
    def get(self, request, **kwargs):
        return HttpResponse("hello")

    def post(self, request, **kwargs):
        pass

class RoomView(TemplateView):
    def get(self, request, **kwargs):
        if not request.user.is_authenticated():
            return HttpResponse("Not authenticated!")

        player = Player.objects.filter(user=request.user).first()

        room = player.current_room
        resp = "
{0}
{1}".format(room.name, room.description)

        resp += "
Exits
"
        for exit in room.exits.all():
            resp += exit.name + "
"

        return HttpResponse(resp)

    def post(self, request, **kwargs):
        pass

class Command(View):
    def get(self, request, **kwargs):
        return Http404()

    def post(self, request, **kwargs):
        command = request.POST.get("request", None)
<|endoftext|>


<commit_before>"""
Unified expression helpers.
"""
class ExpressionFail(Exception):
    def __init__(self, pycode):
        Exception.__init__(self)
        self.pycode = pycode

    def __repr__(self):
        return "ExpressionFail: %r is not a valid expression in this context" % self.pycode

    def __str__(self):
        return self.__repr__()

def evaluate(pycode, locals):
    try:
        val = eval(pycode, {}, locals)
    except NameError, e:
        try:
            # check through the keys for anything we might want to replace
            keys = locals.keys()

            # sort the keys in reverse order so that longer matching strings take priority
            keys.sort(reverse=True)

            # replace the substrings with the string versions of the lookup value
            for key in keys:
                if key in pycode:
                    pval = locals.get(key)
                    pycode = pycode.replace(key, str(pval))

            val = eval(pycode, {}, locals)

        except NameError, e:
            raise ExpressionFail(pycode)

    return val

class ExpressionLocals(dict):
    """
    An object to act as the locals dictionary for the evaluation
    of envi expressions.  You may pass in an envi.symstore.resolver.SymbolResolver
    object to automagically use symbols in your expressions.
    """
    def __init__(self, symobj=None):
        dict.__init__(self)
        self.symobj = symobj

    def __getitem__(self, name):
        if self.symobj != None:
            ret = self.symobj.getSymByName(name)
            if ret != None: return ret
        return dict.__getitem__(self, name)

class MemoryExpressionLocals(ExpressionLocals):

    def __init__(self, memobj, symobj=None):
        ExpressionLocals.__init__(self, symobj=symobj)
        self.memobj = memobj
        self.update({
            'mapbase':self.mapbase,
            'maplen':self.maplen,
            'ispoi':self.ispoi,
            'mem':self.mem,
            'poi':self.poi,
            'sym':self.sym,
        })

    def sym(self, symstr):
        '''
        An easy to use utility for symbols which have un-pythonic names.

        Example x = sym('kernel32.??2@$$FYAPAXI@Z')
        '''
        return long(evaluate(symstr, self))

    def mapbase(self, address):
        """
        The expression mapbase(address) returns the base address of the
        memory mapped area containing "address"
        """
        map = self.memobj.getMemoryMap(address)
        if not map:
            raise Exception("ERROR - un-mapped address in mapbase()")
        return map[0]

    def maplen(self, address):
        """
        The expression maplen(address) returns the length of the
        memory mapped area containing "address".
        """
        map = self.memobj.getMemoryMap(address)
        if not map:
            raise Exception("ERROR - un-mapped address in maplen()")
        return map[1]

    def ispoi(self, addr):
        """
        The expression ispoi(value) returns True if the specified value
        is a valid pointer.  Otherwise, False.
        """
        return self.memobj.isValidPointer(addr)

    def mem(self, addr, size):
        """
        Read and return memory.

        Example: mem(ecx, 20)
        """
        return self.memobj.readMemory(addr, size)

    def poi(self, address):
        """
        When expression contains "poi(addr)" this will return
        the address pointed to by addr.
        """
        return self.memobj.readMemoryPtr(address)

<commit_msg>vw.parseExpression() now accurately uses vw names with the new evaluate()<commit_after>"""
Unified expression helpers.
"""
class ExpressionFail(Exception):
    def __init__(self, pycode):
        Exception.__init__(self)
        self.pycode = pycode

    def __repr__(self):
        return "ExpressionFail: %r is not a valid expression in this context" % self.pycode

    def __str__(self):
        return self.__repr__()

def evaluate(pycode, locals):
    try:
        # check through the keys for anything we might want to replace
        keys = locals.keys()

        # sort the keys in reverse order so that longer matching strings take priority
        keys.sort(reverse=True)

        # replace the substrings with the string versions of the lookup value
        for key in keys:
            if key in pycode:
                pval = locals[key]
                pycode = pycode.replace(key, str(pval))

        val = eval(pycode, {}, locals)

    except NameError, e:
        raise ExpressionFail(pycode)

    return val

class ExpressionLocals(dict):
    """
    An object to act as the locals dictionary for the evaluation
    of envi expressions.  You may pass in an envi.symstore.resolver.SymbolResolver
    object to automagically use symbols in your expressions.
    """
    def __init__(self, symobj=None):
        dict.__init__(self)
        self.symobj = symobj

    def __getitem__(self, name):
        if self.symobj != None:
            ret = self.symobj.getSymByName(name)
            if ret != None: return ret.value
        return dict.__getitem__(self, name)

    def __iter__(self):
        for va, name in self.symobj.getNames():
            yield name

        dict.__iter__(self)

    def keys(self):
        return [key for key in self]

class MemoryExpressionLocals(ExpressionLocals):

    def __init__(self, memobj, symobj=None):
        ExpressionLocals.__init__(self, symobj=symobj)
        self.memobj = memobj
        self.update({
            'mapbase':self.mapbase,
            'maplen':self.maplen,
            'ispoi':self.ispoi,
            'mem':self.mem,
            'poi':self.poi,
            'sym':self.sym,
        })

    def sym(self, symstr):
        '''
        An easy to use utility for symbols which have un-pythonic names.

        Example x = sym('kernel32.??2@$$FYAPAXI@Z')
        '''
        return long(evaluate(symstr, self))

    def mapbase(self, address):
        """
        The expression mapbase(address) returns the base address of the
        memory mapped area containing "address"
        """
        map = self.memobj.getMemoryMap(address)
        if not map:
            raise Exception("ERROR - un-mapped address in mapbase()")
        return map[0]

    def maplen(self, address):
        """
        The expression maplen(address) returns the length of the
        memory mapped area containing "address".
        """
        map = self.memobj.getMemoryMap(address)
        if not map:
            raise Exception("ERROR - un-mapped address in maplen()")
        return map[1]

    def ispoi(self, addr):
        """
        The expression ispoi(value) returns True if the specified value
        is a valid pointer.  Otherwise, False.
        """
        return self.memobj.isValidPointer(addr)

    def mem(self, addr, size):
        """
        Read and return memory.

        Example: mem(ecx, 20)
        """
        return self.memobj.readMemory(addr, size)

    def poi(self, address):
        """
        When expression contains "poi(addr)" this will return
        the address pointed to by addr.
        """
        return self.memobj.readMemoryPtr(address)

<|endoftext|>


<commit_before>######################
# (c) 2012 Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD 3-clause
#
# Implements structured SVM as described in Tsochantaridis et. al.
# Support Vector Machines Learning for Interdependent
# and Structures Output Spaces

import numpy as np
import cvxopt
import cvxopt.solvers

from sklearn.externals.joblib import Parallel, delayed
from sklearn.utils import gen_even_slices

from .ssvm import BaseSSVM
from ..utils import unwrap_pairwise, find_constraint

class StructuredSVM(BaseSSVM):
    """Structured SVM training with l1 slack penalty.

    Implements margin rescaled structural SVM using
    the n-slack formulation and cutting plane method, solved using CVXOPT.
    The optimization is restarted in each iteration.

    Parameters
    ----------
    problem : StructuredProblem
        Object containing problem formulation. Has to implement
        `loss`, `inference` and `loss_augmented_inference`.

    max_iter : int
        Maximum number of passes over dataset to find constraints.

    C : float
        Regularization parameter

    check_constraints : bool
        Whether to check if the new "most violated constraint" is
        more violated than previous constraints. Helpful for stopping
        and debugging, but costly.

    verbose : int
        Verbosity

    positive_constraint: list of ints
        Indices of parmeters that are constraint to be positive.

    break_on_bad: bool (default=True)
        Whether to break (start debug mode) when inference was approximate.

    n_jobs : int, default=1
        Number of parallel jobs for inference. -1 means as many as cpus.

    show_loss : string, default='augmented'
        Controlls the meaning of the loss curve and convergence messages.
        By default (show_loss='augmented') the loss of the loss-augmented
        prediction is shown, since this is computed any way.
        Setting show_loss='real' will show the true loss, i.e. the one of
        the normal prediction. Be aware that this means an additional
        call to inference in each iteration!

    batch_size : int, default=100
        Number of constraints after which we solve the QP again.
        batch_size=-1 means that an update is performed only after going once
        over the whole training set.

    Attributes
    ----------
    w : nd-array, shape=(problem.psi,)
        The learned weights of the SVM.

    old_solution : dict
        The last solution found by the qp solver.

   ``loss_curve_`` : list of float
        List of loss values after each pass thorugh the dataset.
        Either sum of slacks (loss on loss augmented predictions)
        or actual loss, depending on the value of ``show_loss``.

   ``objective_curve_`` : list of float
       Primal objective after each pass through the dataset.
    """

    def __init__(self, problem, max_iter=100, C=1.0, check_constraints=True,
                 verbose=1, positive_constraint=None, n_jobs=1,
                 break_on_bad=True, show_loss='augmented', batch_size=100,
                 tol=0.0001):

        BaseSSVM.__init__(self, problem, max_iter, C, verbose=verbose,
                          n_jobs=n_jobs, show_loss=show_loss)

        self.positive_constraint = positive_constraint
        self.check_constraints = check_constraints
        self.break_on_bad = break_on_bad
        self.batch_size = batch_size
        self.tol = tol
        if verbose < 2:
            cvxopt.solvers.options['show_progress'] = False

    def _solve_n_slack_qp(self, constraints, n_samples):
        C = self.C / float(n_samples)
        psis = [c[1] for sample in constraints for c in sample]
        losses = [c[2] for sample in constraints for c in sample]

        psi_matrix = np.vstack(psis)
        n_constraints = len(psis)
        P = cvxopt.matrix(np.dot(psi_matrix, psi_matrix.T))
        # q contains loss from margin-rescaling
        q = cvxopt.matrix(-np.array(losses, dtype=np.float))
        # constraints are a bit tricky. first, all alpha must be >zero
        idy = np.identity(n_constraints)
        tmp1 = np.zeros(n_constraints)
        # box constraint: sum of all alpha for one example must be <= C
        blocks = np.zeros((n_samples, n_constraints))
        first = 0
        for i, sample in enumerate(constraints):
            blocks[i, first: first + len(sample)] = 1
            first += len(sample)
        # positivity constraints:
        if self.positive_constraint is None:
            #empty constraints
            zero_constr = np.zeros(0)
            psis_constr = np.zeros((0, n_constraints))
        else:
            psis_constr = psi_matrix.T[self.positive_constraint]
            zero_constr = np.zeros(len(self.positive_constraint))

        # put together
        G = cvxopt.matrix(np.vstack((-idy, blocks, psis_constr)))
        tmp2 = np.ones(n_samples) * C
        h = cvxopt.matrix(np.hstack((tmp1, tmp2, zero_constr)))

        # solve QP problem
        cvxopt.solvers.options['feastol'] = 1e-5
        solution = cvxopt.solvers.qp(P, q, G, h)
        if solution['status'] != "optimal":
            print("regularizing QP!")
            P = cvxopt.matrix(np.dot(psi_matrix, psi_matrix.T)
                              + 1e-8 * np.eye(psi_matrix.shape[0]))
            solution = cvxopt.solvers.qp(P, q, G, h)
            if solution['status'] != "optimal":
                from IPython.core.debugger import Tracer
                Tracer()()

        # Lagrange multipliers
        a = np.ravel(solution['x'])
        self.alphas.append(a)
        self.old_solution = solution

        # Support vectors have non zero lagrange multipliers
        sv = a > 1e-10
        box = np.dot(blocks, a)
        if self.verbose > 1:
            print("%d support vectors out of %d points" % (np.sum(sv),
                                                           n_constraints))
            # calculate per example box constraint:
            print("Box constraints at C: %d" % np.sum(1 - box / C < 1e-3))
            print("dual objective: %f" % solution['primal objective'])
        w = np.dot(a, psi_matrix)
        return w, solution['primal objective']

    def _check_bad_constraint(self, y_hat, slack, old_constraints, w):
        if slack < 1e-5:
            return True
        y_hat_plain = unwrap_pairwise(y_hat)

        already_active = np.any([True for y__, _, _ in old_constraints
                                 if (y_hat_plain ==
                                     unwrap_pairwise(y__)).all()])
        if already_active:
            return True

        # "smart" stopping criterion
        # check if most violated constraint is more violated
        # than previous ones by more then eps.
        # If it is less violated, inference was wrong/approximate
        if self.check_constraints:
            for con in old_constraints:
                # compute slack for old constraint
                slack_tmp = max(con[2] - np.dot(w, con[1]), 0)
                if self.verbose > 5:
                    print("slack old constraint: %f" % slack_tmp)
                # if slack of new constraint is smaller or not
                # significantly larger, don't add constraint.
                # if smaller, complain about approximate inference.
                if slack - slack_tmp < -1e-5:
                    print("bad inference: %f" % (slack_tmp - slack))
                    if self.break_on_bad:
                        from IPython.core.debugger import Tracer
                        Tracer()()
                    return True

        return False

    def fit(self, X, Y, constraints=None):
        """Learn parameters using cutting plane method.

        Parameters
        ----------
        X : iterable
            Traing instances. Contains the structured input objects.
            No requirement on the particular form of entries of X is made.

        Y : iterable
            Training labels. Contains the strctured labels for inputs in X.
            Needs to have the same length as X.

        contraints : iterable
            Known constraints for warm-starts. List of same length as X.
            Each entry is itself a list of constraints for a given instance x .
            Each constraint is of the form [y_hat, delta_psi, loss], where
            y_hat is a labeling, ``delta_psi = psi(x, y) - psi(x, y_hat)``
            and loss is the loss for predicting y_hat instead of the true label
            y.
        """
        print("Training n-slack dual structural SVM")
        w = np.zeros(self.problem.size_psi)
        n_samples = len(X)
        if constraints is None:
            constraints = [[] for i in xrange(n_samples)]
            new_constraints = 0
        else:
            new_constraints = np.sum([len(c) for c in constraints])
        loss_curve = []
        objective_curve = []
        self.alphas = []  # dual solutions
        # we have to update at least once after going through the dataset
        for iteration in xrange(self.max_iter):
            # main loop
            if self.verbose > 0:
                print("iteration %d" % iteration)
            if iteration > 0:
                # don't reset in the first iteration
                # we might have been passed some
                new_constraints = 0
            current_loss = 0.
            # generate slices through dataset from batch_size
            n_batches = int(np.ceil(float(len(X)) / self.batch_size))
            slices = gen_even_slices(n_samples, n_batches)
            indices = np.arange(n_samples)
            for batch in slices:
                verbose = max(0, self.verbose - 3)
                X_b = X[batch]
                Y_b = Y[batch]
                indices_b = indices[batch]
                candidate_constraints = Parallel(n_jobs=self.n_jobs,
                                                 verbose=verbose)(
                                                     delayed(find_constraint)(
                                                         self.problem, x, y, w)
                                                     for x, y in zip(X_b, Y_b))

                # for each slice, gather new constraints
                for i, x, y, constraint in zip(indices_b, X_b, Y_b,
                                               candidate_constraints):
                    # loop over dataset
                    y_hat, delta_psi, slack, loss = constraint

                    current_loss += self._get_loss(x, y, w, loss)
                    if self.verbose > 3:
                        print("current slack: %f" % slack)

                    if not loss > 0:
                        # can have y != y_hat but loss = 0 in latent svm.
                        # we need this here as dpsi is then != 0
                        continue

                    if self._check_bad_constraint(y_hat, slack, constraints[i],
                                                  w):
                        continue

                    constraints[i].append([y_hat, delta_psi, loss])
                    new_constraints += 1
                # after processing the slice, solve the qp
                w, objective = self._solve_n_slack_qp(constraints,
                                                      n_samples)
                objective_curve.append(objective)

            if new_constraints == 0:
                print("no additional constraints")
                break

            current_loss /= len(X)
            loss_curve.append(current_loss)

            if self.verbose > 0:
                print("current loss: %f  new constraints: %d, "
                      "dual objective: %f" %
                      (current_loss, new_constraints,
                       objective))
            if (iteration > 1 and objective_curve[-2]
                    - objective_curve[-1] < self.tol):
                print("objective converged.")
                break
            if self.verbose > 5:
                print(w)

        self.w = w
        self.constraints_ = constraints
        self.loss_curve_ = loss_curve
        self.objective_curve_ = objective_curve
        print("calls to inference: %d" % self.problem.inference_calls)
<commit_msg>ENH update qp only after adding constraints, if passed some constraints, solve qp once in the beginning.<commit_after>######################
# (c) 2012 Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD 3-clause
#
# Implements structured SVM as described in Tsochantaridis et. al.
# Support Vector Machines Learning for Interdependent
# and Structures Output Spaces

import numpy as np
import cvxopt
import cvxopt.solvers

from sklearn.externals.joblib import Parallel, delayed
from sklearn.utils import gen_even_slices

from .ssvm import BaseSSVM
from ..utils import unwrap_pairwise, find_constraint

class StructuredSVM(BaseSSVM):
    """Structured SVM training with l1 slack penalty.

    Implements margin rescaled structural SVM using
    the n-slack formulation and cutting plane method, solved using CVXOPT.
    The optimization is restarted in each iteration.

    Parameters
    ----------
    problem : StructuredProblem
        Object containing problem formulation. Has to implement
        `loss`, `inference` and `loss_augmented_inference`.

    max_iter : int
        Maximum number of passes over dataset to find constraints.

    C : float
        Regularization parameter

    check_constraints : bool
        Whether to check if the new "most violated constraint" is
        more violated than previous constraints. Helpful for stopping
        and debugging, but costly.

    verbose : int
        Verbosity

    positive_constraint: list of ints
        Indices of parmeters that are constraint to be positive.

    break_on_bad: bool (default=True)
        Whether to break (start debug mode) when inference was approximate.

    n_jobs : int, default=1
        Number of parallel jobs for inference. -1 means as many as cpus.

    show_loss : string, default='augmented'
        Controlls the meaning of the loss curve and convergence messages.
        By default (show_loss='augmented') the loss of the loss-augmented
        prediction is shown, since this is computed any way.
        Setting show_loss='real' will show the true loss, i.e. the one of
        the normal prediction. Be aware that this means an additional
        call to inference in each iteration!

    batch_size : int, default=100
        Number of constraints after which we solve the QP again.
        batch_size=-1 means that an update is performed only after going once
        over the whole training set.

    Attributes
    ----------
    w : nd-array, shape=(problem.psi,)
        The learned weights of the SVM.

    old_solution : dict
        The last solution found by the qp solver.

   ``loss_curve_`` : list of float
        List of loss values after each pass thorugh the dataset.
        Either sum of slacks (loss on loss augmented predictions)
        or actual loss, depending on the value of ``show_loss``.

   ``objective_curve_`` : list of float
       Primal objective after each pass through the dataset.
    """

    def __init__(self, problem, max_iter=100, C=1.0, check_constraints=True,
                 verbose=1, positive_constraint=None, n_jobs=1,
                 break_on_bad=True, show_loss='augmented', batch_size=100,
                 tol=0.0001):

        BaseSSVM.__init__(self, problem, max_iter, C, verbose=verbose,
                          n_jobs=n_jobs, show_loss=show_loss)

        self.positive_constraint = positive_constraint
        self.check_constraints = check_constraints
        self.break_on_bad = break_on_bad
        self.batch_size = batch_size
        self.tol = tol
        if verbose < 2:
            cvxopt.solvers.options['show_progress'] = False

    def _solve_n_slack_qp(self, constraints, n_samples):
        C = self.C / float(n_samples)
        psis = [c[1] for sample in constraints for c in sample]
        losses = [c[2] for sample in constraints for c in sample]

        psi_matrix = np.vstack(psis)
        n_constraints = len(psis)
        P = cvxopt.matrix(np.dot(psi_matrix, psi_matrix.T))
        # q contains loss from margin-rescaling
        q = cvxopt.matrix(-np.array(losses, dtype=np.float))
        # constraints are a bit tricky. first, all alpha must be >zero
        idy = np.identity(n_constraints)
        tmp1 = np.zeros(n_constraints)
        # box constraint: sum of all alpha for one example must be <= C
        blocks = np.zeros((n_samples, n_constraints))
        first = 0
        for i, sample in enumerate(constraints):
            blocks[i, first: first + len(sample)] = 1
            first += len(sample)
        # positivity constraints:
        if self.positive_constraint is None:
            #empty constraints
            zero_constr = np.zeros(0)
            psis_constr = np.zeros((0, n_constraints))
        else:
            psis_constr = psi_matrix.T[self.positive_constraint]
            zero_constr = np.zeros(len(self.positive_constraint))

        # put together
        G = cvxopt.matrix(np.vstack((-idy, blocks, psis_constr)))
        tmp2 = np.ones(n_samples) * C
        h = cvxopt.matrix(np.hstack((tmp1, tmp2, zero_constr)))

        # solve QP problem
        cvxopt.solvers.options['feastol'] = 1e-5
        solution = cvxopt.solvers.qp(P, q, G, h)
        if solution['status'] != "optimal":
            print("regularizing QP!")
            P = cvxopt.matrix(np.dot(psi_matrix, psi_matrix.T)
                              + 1e-8 * np.eye(psi_matrix.shape[0]))
            solution = cvxopt.solvers.qp(P, q, G, h)
            if solution['status'] != "optimal":
                from IPython.core.debugger import Tracer
                Tracer()()

        # Lagrange multipliers
        a = np.ravel(solution['x'])
        self.alphas.append(a)
        self.old_solution = solution

        # Support vectors have non zero lagrange multipliers
        sv = a > 1e-10
        box = np.dot(blocks, a)
        if self.verbose > 1:
            print("%d support vectors out of %d points" % (np.sum(sv),
                                                           n_constraints))
            # calculate per example box constraint:
            print("Box constraints at C: %d" % np.sum(1 - box / C < 1e-3))
            print("dual objective: %f" % solution['primal objective'])
        w = np.dot(a, psi_matrix)
        return w, solution['primal objective']

    def _check_bad_constraint(self, y_hat, slack, old_constraints, w):
        if slack < 1e-5:
            return True
        y_hat_plain = unwrap_pairwise(y_hat)

        already_active = np.any([True for y__, _, _ in old_constraints
                                 if (y_hat_plain ==
                                     unwrap_pairwise(y__)).all()])
        if already_active:
            return True

        # "smart" stopping criterion
        # check if most violated constraint is more violated
        # than previous ones by more then eps.
        # If it is less violated, inference was wrong/approximate
        if self.check_constraints:
            for con in old_constraints:
                # compute slack for old constraint
                slack_tmp = max(con[2] - np.dot(w, con[1]), 0)
                if self.verbose > 5:
                    print("slack old constraint: %f" % slack_tmp)
                # if slack of new constraint is smaller or not
                # significantly larger, don't add constraint.
                # if smaller, complain about approximate inference.
                if slack - slack_tmp < -1e-5:
                    print("bad inference: %f" % (slack_tmp - slack))
                    if self.break_on_bad:
                        from IPython.core.debugger import Tracer
                        Tracer()()
                    return True

        return False

    def fit(self, X, Y, constraints=None):
        """Learn parameters using cutting plane method.

        Parameters
        ----------
        X : iterable
            Traing instances. Contains the structured input objects.
            No requirement on the particular form of entries of X is made.

        Y : iterable
            Training labels. Contains the strctured labels for inputs in X.
            Needs to have the same length as X.

        contraints : iterable
            Known constraints for warm-starts. List of same length as X.
            Each entry is itself a list of constraints for a given instance x .
            Each constraint is of the form [y_hat, delta_psi, loss], where
            y_hat is a labeling, ``delta_psi = psi(x, y) - psi(x, y_hat)``
            and loss is the loss for predicting y_hat instead of the true label
            y.
        """
        print("Training n-slack dual structural SVM")
        w = np.zeros(self.problem.size_psi)
        n_samples = len(X)
        if constraints is None:
            constraints = [[] for i in xrange(n_samples)]
        else:
            w, objective = self._solve_n_slack_qp(constraints,
                                                  n_samples)
        loss_curve = []
        objective_curve = []
        self.alphas = []  # dual solutions
        # we have to update at least once after going through the dataset
        for iteration in xrange(self.max_iter):
            # main loop
            if self.verbose > 0:
                print("iteration %d" % iteration)
            new_constraints = 0
            current_loss = 0.
            # generate slices through dataset from batch_size
            n_batches = int(np.ceil(float(len(X)) / self.batch_size))
            slices = gen_even_slices(n_samples, n_batches)
            indices = np.arange(n_samples)
            for batch in slices:
                new_constraints_batch = 0
                verbose = max(0, self.verbose - 3)
                X_b = X[batch]
                Y_b = Y[batch]
                indices_b = indices[batch]
                candidate_constraints = Parallel(n_jobs=self.n_jobs,
                                                 verbose=verbose)(
                                                     delayed(find_constraint)(
                                                         self.problem, x, y, w)
                                                     for x, y in zip(X_b, Y_b))

                # for each slice, gather new constraints
                for i, x, y, constraint in zip(indices_b, X_b, Y_b,
                                               candidate_constraints):
                    # loop over dataset
                    y_hat, delta_psi, slack, loss = constraint

                    current_loss += self._get_loss(x, y, w, loss)
                    if self.verbose > 3:
                        print("current slack: %f" % slack)

                    if not loss > 0:
                        # can have y != y_hat but loss = 0 in latent svm.
                        # we need this here as dpsi is then != 0
                        continue

                    if self._check_bad_constraint(y_hat, slack, constraints[i],
                                                  w):
                        continue

                    constraints[i].append([y_hat, delta_psi, loss])
                    new_constraints_batch += 1

                # after processing the slice, solve the qp
                if new_constraints_batch:
                    w, objective = self._solve_n_slack_qp(constraints,
                                                          n_samples)
                    objective_curve.append(objective)
                    new_constraints += new_constraints_batch

            if new_constraints == 0:
                print("no additional constraints")
                break

            current_loss /= len(X)
            loss_curve.append(current_loss)

            if self.verbose > 0:
                print("current loss: %f  new constraints: %d, "
                      "dual objective: %f" %
                      (current_loss, new_constraints,
                       objective))
            if (iteration > 1 and objective_curve[-2]
                    - objective_curve[-1] < self.tol):
                print("objective converged.")
                break
            if self.verbose > 5:
                print(w)

        self.w = w
        self.constraints_ = constraints
        self.loss_curve_ = loss_curve
        self.objective_curve_ = objective_curve
        print("calls to inference: %d" % self.problem.inference_calls)
<|endoftext|>


<commit_before>"""This module contains custom serializer classes."""
import copy

import inflection
from django.db import models
from django.utils import six
from django.utils.functional import cached_property
from rest_framework import exceptions, fields, serializers
from rest_framework.fields import SkipField
from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList

from dynamic_rest.bases import DynamicSerializerBase
from dynamic_rest.conf import settings
from dynamic_rest.fields import DynamicRelationField
from dynamic_rest.links import merge_link_object
from dynamic_rest.processors import SideloadingProcessor
from dynamic_rest.tagged import tag_dict

class DynamicListSerializer(serializers.ListSerializer):
    """Custom ListSerializer class.

    This implementation delegates DREST-specific methods to
    the child serializer and performs post-processing before
    returning the data.
    """
    def __init__(self, *args, **kwargs):
        super(DynamicListSerializer, self).__init__(*args, **kwargs)
        self.child.parent = self

    def to_representation(self, data):
        iterable = data.all() if isinstance(data, models.Manager) else data
        return [self.child.to_representation(item) for item in iterable]

    def get_model(self):
        """Get the child's model."""
        return self.child.get_model()

    def get_name(self):
        """Get the child's name."""
        return self.child.get_name()

    def get_plural_name(self):
        """Get the child's plural name."""
        return self.child.get_plural_name()

    def id_only(self):
        """Get the child's rendering mode."""
        return self.child.id_only()

    @property
    def data(self):
        """Get the data, after performing post-processing if necessary."""
        if not hasattr(self, '_sideloaded_data'):
            data = super(DynamicListSerializer, self).data
            if self.child.sideload:
                self._sideloaded_data = ReturnDict(
                    SideloadingProcessor(
                        self,
                        data
                    ).data,
                    serializer=self
                )
            else:
                self._sideloaded_data = ReturnList(data, serializer=self)
        return self._sideloaded_data

class WithDynamicSerializerMixin(DynamicSerializerBase):
    """Base class for DREST serializers.

    This class provides support for dynamic field inclusions/exclusions.
    """
    def __new__(cls, *args, **kwargs):
        """
        Custom constructor that sets the ListSerializer to
        DynamicListSerializer to avoid re-evaluating querysets.

        Addresses DRF 3.1.0 bug:
        https://github.com/tomchristie/django-rest-framework/issues/2704
        """
        meta = getattr(cls, 'Meta', None)
        if not meta:
            meta = type('Meta', (), {})
            cls.Meta = meta
        meta.list_serializer_class = DynamicListSerializer
        return super(
            WithDynamicSerializerMixin, cls
        ).__new__(
            cls, *args, **kwargs
        )

    def __init__(
            self,
            instance=None,
            data=fields.empty,
            only_fields=None,
            include_fields=None,
            exclude_fields=None,
            request_fields=None,
            sideload=False,
            dynamic=True,
            embed=False,
            **kwargs
    ):
        """
        Custom initializer that builds `request_fields`.

        Arguments:
            instance: Instance to be managed by the serializer.
            only_fields: List of field names to render.
            include_fields: List of field names to include.
            exclude_fields: List of field names to exclude.
            request_fields: map of field names that supports
                inclusions, exclusions, and nested sideloads.
            sideload: If False, do not perform any sideloading at this level.
            embed: If True, force the current representation to be embedded.
            dynamic: If False, ignore deferred rules and
                revert to standard DRF `.fields` behavior.
        """
        name = self.get_name()
        if data is not fields.empty and name in data and len(data) == 1:
            # support POST/PUT key'd by resource name
            data = data[name]

        if data is not fields.empty:
            # if a field is nullable but not required and the implementation
            # passes null as a value, remove the field from the data
            # this addresses the frontends that send
            # undefined resource fields as null on POST/PUT
            for field_name, field in six.iteritems(self.get_all_fields()):
                if (
                    field.allow_null is False and field.required is False and
                    field_name in data and data[field_name] is None
                ):
                    data.pop(field_name)

        kwargs['instance'] = instance
        kwargs['data'] = data
        super(WithDynamicSerializerMixin, self).__init__(**kwargs)

        self.sideload = sideload
        self.dynamic = dynamic
        self.request_fields = request_fields or {}
        self.embed = embed

        self._dynamic_init(only_fields, include_fields, exclude_fields)
        self.enable_optimization = settings.ENABLE_SERIALIZER_OPTIMIZATIONS

    def _dynamic_init(self, only_fields, include_fields, exclude_fields):
        """
        Modifies `request_fields` via higher-level dynamic field interfaces.

        Arguments:
            only_fields: List of field names to render.
                All other fields will be deferred (respects sideloads).
            include_fields: List of field names to include.
                Adds to default field set, (respects sideloads).
                `*` means include all fields.
            exclude_fields: List of field names to exclude.
                Removes from default field set. If set to '*', all fields are
                removed, except for ones that are explicitly included.
        """

        if not self.dynamic:
            return

        if (
            isinstance(self.request_fields, dict)
            and self.request_fields.pop('*', None) is False
        ):
            exclude_fields = '*'

        only_fields = set(only_fields or [])
        include_fields = include_fields or []
        exclude_fields = exclude_fields or []
        all_fields = set(self.get_all_fields().keys())

        if only_fields:
            exclude_fields = '*'
            include_fields = only_fields

        if exclude_fields == '*':
            # First exclude all, then add back in explicitly included fields.
            include_fields = set(
                list(include_fields) + [
                    field for field, val in six.iteritems(self.request_fields)
                    if val or val == {}
                ]
            )
            exclude_fields = all_fields - include_fields
        elif include_fields == '*':
            include_fields = all_fields

        for name in exclude_fields:
            self.request_fields[name] = False

        for name in include_fields:
            if not isinstance(self.request_fields.get(name), dict):
                # not sideloading this field
                self.request_fields[name] = True

    @classmethod
    def get_model(cls):
        """Get the model, if the serializer has one.

        Model serializers should implement this method.
        """
        return None

    @classmethod
    def get_name(cls):
        """Get the serializer name.

        The name can be defined on the Meta class or will be generated
        automatically from the model name.
        """
        class_name = getattr(cls.get_model(), '__name__', None)
        return getattr(
            cls.Meta,
            'name',
            inflection.underscore(class_name) if class_name else None
        )

    @classmethod
    def get_plural_name(cls):
        """Get the serializer's plural name.

        The plural name may be defined on the Meta class.
        If the plural name is not defined,
        the pluralized form of the name will be returned.
        """
        return getattr(
            cls.Meta,
            'plural_name',
            inflection.pluralize(cls.get_name())
        )

    def get_all_fields(self):
        """Returns the entire serializer field set.

        Does not respect dynamic field inclusions/exclusions.
        """
        if not hasattr(self, '_all_fields'):
            self._all_fields = super(
                WithDynamicSerializerMixin,
                self
            ).get_fields()
            for k, field in six.iteritems(self._all_fields):
                field.field_name = k
                field.parent = self
        return self._all_fields

    def _get_deferred_field_names(self, serializer_fields):
        """Return set of deferred field names."""
        meta_deferred = set(getattr(self.Meta, 'deferred_fields', []))
        return {
            name for name, field in six.iteritems(serializer_fields)
            if getattr(field, 'deferred', None) is True or name in
            meta_deferred
        }

    def get_fields(self):
        """Returns the serializer's field set.

        If `dynamic` is True, respects field inclusions/exlcusions.
        Otherwise, reverts back to standard DRF behavior.
        """
        all_fields = self.get_all_fields()
        if self.dynamic is False:
            return all_fields

        if self.id_only():
            return {}

        serializer_fields = copy.deepcopy(all_fields)
        request_fields = self.request_fields
        deferred = self._get_deferred_field_names(serializer_fields)

        # apply request overrides
        if request_fields:
            for name, include in six.iteritems(request_fields):
                if name not in serializer_fields:
                    raise exceptions.ParseError(
                        "'%s' is not a valid field name for '%s'" %
                        (name, self.get_name()))
                if include is not False and name in deferred:
                    deferred.remove(name)
                elif include is False:
                    deferred.add(name)

        for name in deferred:
            serializer_fields.pop(name)

        return serializer_fields

    def get_link_fields(self):
        """Construct dict of name:field for linkable fields."""
        if not hasattr(self, '_link_fields'):
            all_fields = self.get_all_fields()
            self._link_fields = {
                name: field for name, field in six.iteritems(all_fields)
                if isinstance(field, DynamicRelationField)
                and getattr(field, 'link', True)
                and not (
                    # Skip sideloaded fields
                    name in self.fields
                    and not field.serializer.id_only()
                ) and not (
                    # Skip included single relations
                    # TODO: Use links, when we can generate canonical URLs
                    name in self.fields
                    and not getattr(field.serializer, 'many', False)
                )
            }

        return self._link_fields

    @cached_property
    def _readable_fields(self):
        # NOTE: Copied from DRF, exists in 3.2.x but not 3.1
        return [
            field for field in self.fields.values()
            if not field.write_only
        ]

    def _faster_to_representation(self, instance):
        """Modified to_representation with optimizations.

        1) Returns a plain old dict as opposed to OrderedDict.
            (Constructing ordered dict is ~100x slower than `{}`.)
        2) Ensure we use a cached list of fields
            (this optimization exists in DRF 3.2 but not 3.1)

        Arguments:
            instance: a model instance or data object
        Returns:
            Dict of primitive datatypes.
        """

        ret = {}
        fields = self._readable_fields

        for field in fields:
            try:
                attribute = field.get_attribute(instance)
            except SkipField:
                continue

            if attribute is None:
                # We skip `to_representation` for `None` values so that
                # fields do not have to explicitly deal with that case.
                ret[field.field_name] = None
            else:
                ret[field.field_name] = field.to_representation(attribute)

        return ret

    def to_representation(self, instance):
        """Modified to_representation method.

        Arguments:
            instance: A model instance or data object.
        Returns:
            Instance ID if the serializer is meant to represent its ID.
            Otherwise, a tagged data dict representation.
        """
        if self.id_only():
            return instance.pk
        else:
            if self.enable_optimization:
                representation = self._faster_to_representation(instance)
            else:
                representation = super(
                    WithDynamicSerializerMixin,
                    self
                ).to_representation(instance)

            if settings.ENABLE_LINKS:
                # TODO: Make this function configurable to support other
                #       formats like JSON API link objects.
                representation = merge_link_object(
                    self, representation, instance
                )

        # tag the representation with the serializer and instance
        return tag_dict(
            representation,
            serializer=self,
            instance=instance,
            embed=self.embed
        )

    def save(self, *args, **kwargs):
        """Serializer save that address prefetch issues."""
        update = getattr(self, 'instance', None) is not None
        instance = super(
            WithDynamicSerializerMixin,
            self
        ).save(
            *args,
            **kwargs
        )
        view = self._context.get('view')
        if update and view:
            # Reload the object on update
            # to get around prefetch cache issues
            instance = self.instance = view.get_object()
        return instance

    def id_only(self):
        """Check whether the serializer should return an ID instead of an object.

        Returns:
            True if and only if `request_fields` is True.
        """
        return self.dynamic and self.request_fields is True

    @property
    def data(self):
        if not hasattr(self, '_sideloaded_data'):
            data = super(WithDynamicSerializerMixin, self).data
            self._sideloaded_data = ReturnDict(
                SideloadingProcessor(
                    self,
                    data
                ).data if self.sideload else data,
                serializer=self
            )
        return self._sideloaded_data

class WithDynamicModelSerializerMixin(WithDynamicSerializerMixin):

    """Adds DREST serializer methods specific to model-based serializers."""

    @classmethod
    def get_model(cls):
        return cls.Meta.model

    def get_id_fields(self):
        """
        Called to return a list of fields consisting of, at minimum,
        the PK field name. The output of this method is used to
        construct a Prefetch object with a .only() queryset
        when this field is not being sideloaded but we need to
        return a list of IDs.
        """
        model = self.get_model()

        out = [model._meta.pk.name]  # get PK field name

        # If this is being called, it means it
        # is a many-relation  to its parent.
        # Django wants the FK to the parent,
        # but since accurately inferring the FK
        # pointing back to the parent is less than trivial,
        # we will just pull all ID fields.
        # TODO: We also might need to return all non-nullable fields,
        #    or else it is possible Django will issue another request.
        for field in model._meta.fields:
            if isinstance(field, models.ForeignKey):
                out.append(field.name + '_id')

        return out

class DynamicModelSerializer(
    WithDynamicModelSerializerMixin,
    serializers.ModelSerializer
):

    """DREST-compatible model-based serializer."""
    pass

class EphemeralObject(object):

    """Object that initializes attributes from a dict."""

    def __init__(self, values_dict):
        if 'pk' not in values_dict:
            raise Exception("'pk' key is required")
        self.__dict__.update(values_dict)

class DynamicEphemeralSerializer(
    WithDynamicSerializerMixin,
    serializers.Serializer
):

    """DREST-compatible baseclass for non-model serializers."""

    def to_representation(self, instance):
        """
        Provides post processing. Sub-classes should implement their own
        to_representation method, but pass the resulting dict through
        this function to get tagging and field selection.

        Arguments:
            instance: Serialized dict, or object. If object,
                it will be serialized by the super class's
                to_representation() method.
        """

        if not isinstance(instance, dict):
            data = super(
                DynamicEphemeralSerializer,
                self
            ).to_representation(instance)
        else:
            data = instance
            instance = EphemeralObject(data)

        if self.id_only():
            return data
        else:
            return tag_dict(data, serializer=self, instance=instance)
<commit_msg>cache serializer names<commit_after>"""This module contains custom serializer classes."""
import copy

import inflection
from django.db import models
from django.utils import six
from django.utils.functional import cached_property
from rest_framework import exceptions, fields, serializers
from rest_framework.fields import SkipField
from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList

from dynamic_rest.bases import DynamicSerializerBase
from dynamic_rest.conf import settings
from dynamic_rest.fields import DynamicRelationField
from dynamic_rest.links import merge_link_object
from dynamic_rest.processors import SideloadingProcessor
from dynamic_rest.tagged import tag_dict

class DynamicListSerializer(serializers.ListSerializer):
    """Custom ListSerializer class.

    This implementation delegates DREST-specific methods to
    the child serializer and performs post-processing before
    returning the data.
    """
    def __init__(self, *args, **kwargs):
        super(DynamicListSerializer, self).__init__(*args, **kwargs)
        self.child.parent = self

    def to_representation(self, data):
        iterable = data.all() if isinstance(data, models.Manager) else data
        return [self.child.to_representation(item) for item in iterable]

    def get_model(self):
        """Get the child's model."""
        return self.child.get_model()

    def get_name(self):
        """Get the child's name."""
        return self.child.get_name()

    def get_plural_name(self):
        """Get the child's plural name."""
        return self.child.get_plural_name()

    def id_only(self):
        """Get the child's rendering mode."""
        return self.child.id_only()

    @property
    def data(self):
        """Get the data, after performing post-processing if necessary."""
        if not hasattr(self, '_sideloaded_data'):
            data = super(DynamicListSerializer, self).data
            if self.child.sideload:
                self._sideloaded_data = ReturnDict(
                    SideloadingProcessor(
                        self,
                        data
                    ).data,
                    serializer=self
                )
            else:
                self._sideloaded_data = ReturnList(data, serializer=self)
        return self._sideloaded_data

class WithDynamicSerializerMixin(DynamicSerializerBase):
    """Base class for DREST serializers.

    This class provides support for dynamic field inclusions/exclusions.
    """
    def __new__(cls, *args, **kwargs):
        """
        Custom constructor that sets the ListSerializer to
        DynamicListSerializer to avoid re-evaluating querysets.

        Addresses DRF 3.1.0 bug:
        https://github.com/tomchristie/django-rest-framework/issues/2704
        """
        meta = getattr(cls, 'Meta', None)
        if not meta:
            meta = type('Meta', (), {})
            cls.Meta = meta
        meta.list_serializer_class = DynamicListSerializer
        return super(
            WithDynamicSerializerMixin, cls
        ).__new__(
            cls, *args, **kwargs
        )

    def __init__(
            self,
            instance=None,
            data=fields.empty,
            only_fields=None,
            include_fields=None,
            exclude_fields=None,
            request_fields=None,
            sideload=False,
            dynamic=True,
            embed=False,
            **kwargs
    ):
        """
        Custom initializer that builds `request_fields`.

        Arguments:
            instance: Instance to be managed by the serializer.
            only_fields: List of field names to render.
            include_fields: List of field names to include.
            exclude_fields: List of field names to exclude.
            request_fields: map of field names that supports
                inclusions, exclusions, and nested sideloads.
            sideload: If False, do not perform any sideloading at this level.
            embed: If True, force the current representation to be embedded.
            dynamic: If False, ignore deferred rules and
                revert to standard DRF `.fields` behavior.
        """
        name = self.get_name()
        if data is not fields.empty and name in data and len(data) == 1:
            # support POST/PUT key'd by resource name
            data = data[name]

        if data is not fields.empty:
            # if a field is nullable but not required and the implementation
            # passes null as a value, remove the field from the data
            # this addresses the frontends that send
            # undefined resource fields as null on POST/PUT
            for field_name, field in six.iteritems(self.get_all_fields()):
                if (
                    field.allow_null is False and field.required is False and
                    field_name in data and data[field_name] is None
                ):
                    data.pop(field_name)

        kwargs['instance'] = instance
        kwargs['data'] = data
        super(WithDynamicSerializerMixin, self).__init__(**kwargs)

        self.sideload = sideload
        self.dynamic = dynamic
        self.request_fields = request_fields or {}
        self.embed = embed

        self._dynamic_init(only_fields, include_fields, exclude_fields)
        self.enable_optimization = settings.ENABLE_SERIALIZER_OPTIMIZATIONS

    def _dynamic_init(self, only_fields, include_fields, exclude_fields):
        """
        Modifies `request_fields` via higher-level dynamic field interfaces.

        Arguments:
            only_fields: List of field names to render.
                All other fields will be deferred (respects sideloads).
            include_fields: List of field names to include.
                Adds to default field set, (respects sideloads).
                `*` means include all fields.
            exclude_fields: List of field names to exclude.
                Removes from default field set. If set to '*', all fields are
                removed, except for ones that are explicitly included.
        """

        if not self.dynamic:
            return

        if (
            isinstance(self.request_fields, dict)
            and self.request_fields.pop('*', None) is False
        ):
            exclude_fields = '*'

        only_fields = set(only_fields or [])
        include_fields = include_fields or []
        exclude_fields = exclude_fields or []
        all_fields = set(self.get_all_fields().keys())

        if only_fields:
            exclude_fields = '*'
            include_fields = only_fields

        if exclude_fields == '*':
            # First exclude all, then add back in explicitly included fields.
            include_fields = set(
                list(include_fields) + [
                    field for field, val in six.iteritems(self.request_fields)
                    if val or val == {}
                ]
            )
            exclude_fields = all_fields - include_fields
        elif include_fields == '*':
            include_fields = all_fields

        for name in exclude_fields:
            self.request_fields[name] = False

        for name in include_fields:
            if not isinstance(self.request_fields.get(name), dict):
                # not sideloading this field
                self.request_fields[name] = True

    @classmethod
    def get_model(cls):
        """Get the model, if the serializer has one.

        Model serializers should implement this method.
        """
        return None

    @classmethod
    def get_name(cls):
        """Get the serializer name.

        The name can be defined on the Meta class or will be generated
        automatically from the model name.
        """
        if not hasattr(cls.Meta, 'name'):
            class_name = getattr(cls.get_model(), '__name__', None)
            setattr(
                cls.Meta,
                'name',
                inflection.underscore(class_name) if class_name else None
            )

        return cls.Meta.name

    @classmethod
    def get_plural_name(cls):
        """Get the serializer's plural name.

        The plural name may be defined on the Meta class.
        If the plural name is not defined,
        the pluralized form of the name will be returned.
        """
        if not hasattr(cls.Meta, 'plural_name'):
            setattr(
                cls.Meta,
                'plural_name',
                inflection.pluralize(cls.get_name())
            )
        return cls.Meta.plural_name

    def get_all_fields(self):
        """Returns the entire serializer field set.

        Does not respect dynamic field inclusions/exclusions.
        """
        if not hasattr(self, '_all_fields'):
            self._all_fields = super(
                WithDynamicSerializerMixin,
                self
            ).get_fields()
            for k, field in six.iteritems(self._all_fields):
                field.field_name = k
                field.parent = self
        return self._all_fields

    def _get_deferred_field_names(self, serializer_fields):
        """Return set of deferred field names."""
        meta_deferred = set(getattr(self.Meta, 'deferred_fields', []))
        return {
            name for name, field in six.iteritems(serializer_fields)
            if getattr(field, 'deferred', None) is True or name in
            meta_deferred
        }

    def get_fields(self):
        """Returns the serializer's field set.

        If `dynamic` is True, respects field inclusions/exlcusions.
        Otherwise, reverts back to standard DRF behavior.
        """
        all_fields = self.get_all_fields()
        if self.dynamic is False:
            return all_fields

        if self.id_only():
            return {}

        serializer_fields = copy.deepcopy(all_fields)
        request_fields = self.request_fields
        deferred = self._get_deferred_field_names(serializer_fields)

        # apply request overrides
        if request_fields:
            for name, include in six.iteritems(request_fields):
                if name not in serializer_fields:
                    raise exceptions.ParseError(
                        "'%s' is not a valid field name for '%s'" %
                        (name, self.get_name()))
                if include is not False and name in deferred:
                    deferred.remove(name)
                elif include is False:
                    deferred.add(name)

        for name in deferred:
            serializer_fields.pop(name)

        return serializer_fields

    def get_link_fields(self):
        """Construct dict of name:field for linkable fields."""
        if not hasattr(self, '_link_fields'):
            all_fields = self.get_all_fields()
            self._link_fields = {
                name: field for name, field in six.iteritems(all_fields)
                if isinstance(field, DynamicRelationField)
                and getattr(field, 'link', True)
                and not (
                    # Skip sideloaded fields
                    name in self.fields
                    and not field.serializer.id_only()
                ) and not (
                    # Skip included single relations
                    # TODO: Use links, when we can generate canonical URLs
                    name in self.fields
                    and not getattr(field.serializer, 'many', False)
                )
            }

        return self._link_fields

    @cached_property
    def _readable_fields(self):
        # NOTE: Copied from DRF, exists in 3.2.x but not 3.1
        return [
            field for field in self.fields.values()
            if not field.write_only
        ]

    def _faster_to_representation(self, instance):
        """Modified to_representation with optimizations.

        1) Returns a plain old dict as opposed to OrderedDict.
            (Constructing ordered dict is ~100x slower than `{}`.)
        2) Ensure we use a cached list of fields
            (this optimization exists in DRF 3.2 but not 3.1)

        Arguments:
            instance: a model instance or data object
        Returns:
            Dict of primitive datatypes.
        """

        ret = {}
        fields = self._readable_fields

        for field in fields:
            try:
                attribute = field.get_attribute(instance)
            except SkipField:
                continue

            if attribute is None:
                # We skip `to_representation` for `None` values so that
                # fields do not have to explicitly deal with that case.
                ret[field.field_name] = None
            else:
                ret[field.field_name] = field.to_representation(attribute)

        return ret

    def to_representation(self, instance):
        """Modified to_representation method.

        Arguments:
            instance: A model instance or data object.
        Returns:
            Instance ID if the serializer is meant to represent its ID.
            Otherwise, a tagged data dict representation.
        """
        if self.id_only():
            return instance.pk
        else:
            if self.enable_optimization:
                representation = self._faster_to_representation(instance)
            else:
                representation = super(
                    WithDynamicSerializerMixin,
                    self
                ).to_representation(instance)

            if settings.ENABLE_LINKS:
                # TODO: Make this function configurable to support other
                #       formats like JSON API link objects.
                representation = merge_link_object(
                    self, representation, instance
                )

        # tag the representation with the serializer and instance
        return tag_dict(
            representation,
            serializer=self,
            instance=instance,
            embed=self.embed
        )

    def save(self, *args, **kwargs):
        """Serializer save that address prefetch issues."""
        update = getattr(self, 'instance', None) is not None
        instance = super(
            WithDynamicSerializerMixin,
            self
        ).save(
            *args,
            **kwargs
        )
        view = self._context.get('view')
        if update and view:
            # Reload the object on update
            # to get around prefetch cache issues
            instance = self.instance = view.get_object()
        return instance

    def id_only(self):
        """Check whether the serializer should return an ID instead of an object.

        Returns:
            True if and only if `request_fields` is True.
        """
        return self.dynamic and self.request_fields is True

    @property
    def data(self):
        if not hasattr(self, '_sideloaded_data'):
            data = super(WithDynamicSerializerMixin, self).data
            self._sideloaded_data = ReturnDict(
                SideloadingProcessor(
                    self,
                    data
                ).data if self.sideload else data,
                serializer=self
            )
        return self._sideloaded_data

class WithDynamicModelSerializerMixin(WithDynamicSerializerMixin):

    """Adds DREST serializer methods specific to model-based serializers."""

    @classmethod
    def get_model(cls):
        return cls.Meta.model

    def get_id_fields(self):
        """
        Called to return a list of fields consisting of, at minimum,
        the PK field name. The output of this method is used to
        construct a Prefetch object with a .only() queryset
        when this field is not being sideloaded but we need to
        return a list of IDs.
        """
        model = self.get_model()

        out = [model._meta.pk.name]  # get PK field name

        # If this is being called, it means it
        # is a many-relation  to its parent.
        # Django wants the FK to the parent,
        # but since accurately inferring the FK
        # pointing back to the parent is less than trivial,
        # we will just pull all ID fields.
        # TODO: We also might need to return all non-nullable fields,
        #    or else it is possible Django will issue another request.
        for field in model._meta.fields:
            if isinstance(field, models.ForeignKey):
                out.append(field.name + '_id')

        return out

class DynamicModelSerializer(
    WithDynamicModelSerializerMixin,
    serializers.ModelSerializer
):

    """DREST-compatible model-based serializer."""
    pass

class EphemeralObject(object):

    """Object that initializes attributes from a dict."""

    def __init__(self, values_dict):
        if 'pk' not in values_dict:
            raise Exception("'pk' key is required")
        self.__dict__.update(values_dict)

class DynamicEphemeralSerializer(
    WithDynamicSerializerMixin,
    serializers.Serializer
):

    """DREST-compatible baseclass for non-model serializers."""

    def to_representation(self, instance):
        """
        Provides post processing. Sub-classes should implement their own
        to_representation method, but pass the resulting dict through
        this function to get tagging and field selection.

        Arguments:
            instance: Serialized dict, or object. If object,
                it will be serialized by the super class's
                to_representation() method.
        """

        if not isinstance(instance, dict):
            data = super(
                DynamicEphemeralSerializer,
                self
            ).to_representation(instance)
        else:
            data = instance
            instance = EphemeralObject(data)

        if self.id_only():
            return data
        else:
            return tag_dict(data, serializer=self, instance=instance)
<|endoftext|>


<commit_before># Copyright ClusterHQ Limited. See LICENSE file for details.

"""
Tests that verify that Docker behaviour is unchanged when running through
Powerstrip.

Run these tests as so:

$ docker run -d --name powerstrip \
    -v /var/run/docker.sock:/var/run/docker.sock \
    -v ./blank_adapters.yml:/etc/powerstrip/adapters.yml \
    -p 2375:2375 \
    clusterhq/powerstrip:docker-compat-tests
$ docker run --name powerstrip-test
    -v /var/run/docker.sock:/var/run/docker.sock \
    -e TEST_PASSTHRU=1
    --link powerstrip:powerstrip
    clusterhq/powerstrip:docker-compat-tests tox powerstrip.test.test_passthru
"""

import os

from twisted.internet.utils import getProcessOuput
from twisted.trial.unittest import TestCase

def CompareDockerAndPowerstrip(test_case, cmd):
    """
    Compare the output of a real Docker server against a Powerstrip passthu.

    There will be environment variables set which determine whether any
    ``docker`` execs will talk to Docker or Powerstrip.

    :param test_case: The current ``TestCase`` context.

    :param cmd: A shell command. E.g. ``echo ls | docker run -i ubuntu bash``.
        This command should include at least one call to Docker.

    :return: A ``Deferred`` which fires when the test has been completed.
    """
    DOCKER = b"unix:///var/run/docker.sock"
    POWERSTRIP = b"tcp://powerstrip:2375"

    d = getProcessOuput(b"/bin/bash", ["-c", cmd], { b"DOCKER_HOST": DOCKER },
        errortoo=True)

    def got_result(docker_result):
        if not docker_result:
            raise ValueError("Command did not produce any output went sent to "
                    "Docker daemon.")
        d = getProcessOuput(b"/bin/bash", ["-c", cmd], { b"DOCKER_HOST": POWERSTRIP },
            errortoo=True)

        def compare_result(powerstrip_result, docker_result):
            test_case.assertEquals(docker_result, powerstrip_result)

        d.addCallback(compare_result, docker_result)
        return d

    d.addCallback(got_result)
    return d

class BasicTests(TestCase):
    """
    Tests for basic Docker functionality.
    """

    if "TEST_PASSTHRU" not in os.environ:
        skip = "Skipping passthru tests."

    def test_run(self):
        """
        Test basic ``docker run`` functionality.
        """

        # XXX this will need to prime the Docker instance in most cases, e.g.
        # docker pull
        return CompareDockerAndPowerstrip(self,
            "docker run -ti ubuntu echo hello")
<commit_msg>We can just use trial because all the deps are already installed<commit_after># Copyright ClusterHQ Limited. See LICENSE file for details.

"""
Tests that verify that Docker behaviour is unchanged when running through
Powerstrip.

Run these tests as so:

$ # For the blank_adapters.yml
$ cd $POWERSTRIP_HOME/powerstrip/test
$ sudo docker run -d --name powerstrip \
    -v /var/run/docker.sock:/var/run/docker.sock \
    -v $PWD/blank_adapters.yml:/etc/powerstrip/adapters.yml \
    -p 2375:2375 \
    clusterhq/powerstrip:docker-compat-tests
$ sudo docker run --name powerstrip-test \
    -v /var/run/docker.sock:/var/run/docker.sock \
    -e TEST_PASSTHRU=1 \
    --link powerstrip:powerstrip \
    clusterhq/powerstrip:docker-compat-tests trial powerstrip.test.test_passthru
"""

import os

from twisted.internet.utils import getProcessOuput
from twisted.trial.unittest import TestCase

def CompareDockerAndPowerstrip(test_case, cmd):
    """
    Compare the output of a real Docker server against a Powerstrip passthu.

    There will be environment variables set which determine whether any
    ``docker`` execs will talk to Docker or Powerstrip.

    :param test_case: The current ``TestCase`` context.

    :param cmd: A shell command. E.g. ``echo ls | docker run -i ubuntu bash``.
        This command should include at least one call to Docker.

    :return: A ``Deferred`` which fires when the test has been completed.
    """
    DOCKER = b"unix:///var/run/docker.sock"
    POWERSTRIP = b"tcp://powerstrip:2375"

    d = getProcessOuput(b"/bin/bash", ["-c", cmd], { b"DOCKER_HOST": DOCKER },
        errortoo=True)

    def got_result(docker_result):
        if not docker_result:
            raise ValueError("Command did not produce any output went sent to "
                    "Docker daemon.")
        d = getProcessOuput(b"/bin/bash", ["-c", cmd], { b"DOCKER_HOST": POWERSTRIP },
            errortoo=True)

        def compare_result(powerstrip_result, docker_result):
            test_case.assertEquals(docker_result, powerstrip_result)

        d.addCallback(compare_result, docker_result)
        return d

    d.addCallback(got_result)
    return d

class BasicTests(TestCase):
    """
    Tests for basic Docker functionality.
    """

    if "TEST_PASSTHRU" not in os.environ:
        skip = "Skipping passthru tests."

    def test_run(self):
        """
        Test basic ``docker run`` functionality.
        """

        # XXX this will need to prime the Docker instance in most cases, e.g.
        # docker pull
        return CompareDockerAndPowerstrip(self,
            "docker run -ti ubuntu echo hello")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.contrib import admin
from helios.store.models import *
from helios.store.forms import MyCategoryAdminForm
from helios.conf import settings
if settings.IS_MULTILINGUAL:
    import multilingual
    admin_info = {
        'class': multilingual.ModelAdmin,
        'suffix': '_en'
    }
else:
    admin_info = {
        'class': admin.ModelAdmin,
        'suffix': ''
    }

class ProductImageInline(admin.TabularInline):
    model = ProductImage

class ProductImageAdmin(admin.ModelAdmin):
    list_display = ['picture', 'product']

class CategoryAdmin(admin_info['class']):
    form = MyCategoryAdminForm
    list_display = ('name', 'desc', 'parent')
    prepopulated_fields = {'slug': (''.join(['name', admin_info['suffix']]),)}

def set_category(modeladmin, request, queryset):
    categories = Category.objects.all()
    queryset.update(category)

class ProductAdmin(admin_info['class']):
    inlines = [ProductImageInline]
    list_display = ('name', 'base_price', 'price', 'stock', 'last_modified', 'category', 'is_active', 'is_featured')
    list_filter = ('category', 'is_active', 'is_featured',)
    prepopulated_fields = {'slug': (''.join(['name', admin_info['suffix']]),)}
    search_fields = ['name']

class TaxAdmin(admin_info['class']):
    list_display = ('name', 'rate', 'factor',)
    prepopulated_fields = {'slug': (''.join(['name', admin_info['suffix']]),)}

class CollectionAdmin(admin_info['class']):
    filter_horizontal = ['products']
    prepopulated_fields = {'slug': (''.join(['name', admin_info['suffix']]),)}

admin.site.register(Category, CategoryAdmin)
admin.site.register(Product, ProductAdmin)
admin.site.register(ProductImage, ProductImageAdmin)
admin.site.register(Tax, TaxAdmin)
admin.site.register(Collection, CollectionAdmin)
<commit_msg>trying to locate the ordeline error.<commit_after># -*- coding: utf-8 -*-
from django.contrib import admin
from helios.store.models import *
from helios.store.forms import MyCategoryAdminForm
from helios.conf import settings
if settings.IS_MULTILINGUAL:
    import multilingual
    admin_info = {
        'class': multilingual.ModelAdmin,
        'suffix': '_en'
    }
else:
    admin_info = {
        'class': admin.ModelAdmin,
        'suffix': ''
    }

class ProductImageInline(admin.TabularInline):
    model = ProductImage

class ProductImageAdmin(admin.ModelAdmin):
    list_display = ['picture', 'product']

class CategoryAdmin(admin_info['class']):
    form = MyCategoryAdminForm
    list_display = ('name', 'desc', 'parent')
    prepopulated_fields = {'slug': (''.join(['name', admin_info['suffix']]),)}

def set_category(modeladmin, request, queryset):
    categories = Category.objects.all()
    queryset.update(category)

class ProductAdmin(admin.ModelAdmin):
    inlines = [ProductImageInline]
    #list_display = ('name', 'base_price', 'price', 'stock', 'last_modified', 'category', 'is_active', 'is_featured')
    #list_filter = ('category', 'is_active', 'is_featured',)
    #prepopulated_fields = {'slug': (''.join(['name', admin_info['suffix']]),)}
    #search_fields = ['name']

class TaxAdmin(admin_info['class']):
    list_display = ('name', 'rate', 'factor',)
    prepopulated_fields = {'slug': (''.join(['name', admin_info['suffix']]),)}

class CollectionAdmin(admin_info['class']):
    filter_horizontal = ['products']
    prepopulated_fields = {'slug': (''.join(['name', admin_info['suffix']]),)}

admin.site.register(Category, CategoryAdmin)
admin.site.register(Product, ProductAdmin)
admin.site.register(ProductImage, ProductImageAdmin)
admin.site.register(Tax, TaxAdmin)
admin.site.register(Collection, CollectionAdmin)
<|endoftext|>