<commit_before>"""OPP servo implementation."""

from mpf.platforms.interfaces.servo_platform_interface import ServoPlatformInterface
from mpf.platforms.opp.opp_rs232_intf import OppRs232Intf
import logging

MYPY = False
if MYPY:  # pragma: no cover
    from mpf.platforms.opp.opp import OppHardwarePlatform  # pylint: disable-msg=cyclic-import,unused-import

class OPPServo(ServoPlatformInterface):
    """A servo in the OPP platform."""

    __slots__ = ["number", "chain_serial", "platform", "speed", "current_position"]

    def __init__(self, chain_serial, servo_num, platform: "OppHardwarePlatform"):
        """Initialise servo."""
        self.number = servo_num
        self.platform = platform
        self.chain_serial = chain_serial
        self.speed = 0
        self.current_position = 0

    def stop(self):
        """Disable servo.
        Set position to 0 to disable servo.
        """
        self.go_to_position(0)

    def go_to_position(self, position):
        """Set a servo position.

        position [0 to 1.0] is converted to position_numeric [0 to 255].
        position_numeric is measured in 10us intervals, so a position_numeric of 100 is a 1ms pulse
        and 150 is 1.5ms. A position_numeric of 0 disables the servo. Use caution with extreme
        position values as it could cause a servo to drive to a position it cannot reach.
        """

        # convert from [0,1] to [0, 255]
        position_numeric = int(position * 255)
        servo_offset = 0x3000 + self.number

        if position_numeric == 0 or self.current_position == 0 or self.speed <= 0:
            fade_ms = 0
        else:
            fade_ms = 600 * abs(position_numeric - self.current_position) / self.speed
            if fade_ms > 65535:
                fade_ms = 65535

        msg = bytearray()
        msg.append(0x20 + int(self.chain_serial))
        msg.append(OppRs232Intf.SERIAL_LED_CMD_FADE)
        msg.append(int(servo_offset / 256))
        msg.append(int(servo_offset % 256))
        msg.append(int(0))  #number of servos (high)
        msg.append(int(1))  #number of servos (low)...only commanding one at a time.
        msg.append(int(fade_ms / 256))
        msg.append(int(fade_ms % 256))
        msg.append(position_numeric)
        msg.extend(OppRs232Intf.calc_crc8_whole_msg(msg))
        cmd = bytes(msg)

        self.platform.debug_log("Set servo position on %s: %s", self.chain_serial, "".join(" 0x%02x" % b for b in cmd))

        self.platform.send_to_processor(self.chain_serial, cmd)

        self.current_position = position_numeric

    def set_speed_limit(self, speed_limit):
        """Set the speed of this servo

        For the standard 1ms pulse width change to move a servo between
        extremes, a speed of 1 will take 1 minute, and a speed of 60 would take
        1 second.

        A speed <= 0 is unrestricted by firmware
        """
        self.platform.debug_log("Change servo speed limit on %s: %s", self.chain_serial, int(speed_limit))

        self.speed = speed_limit

    def set_acceleration_limit(self, acceleration_limit):
        """Not implemented."""
<commit_msg>Fix board address in servo command<commit_after>"""OPP servo implementation."""

from mpf.platforms.interfaces.servo_platform_interface import ServoPlatformInterface
from mpf.platforms.opp.opp_rs232_intf import OppRs232Intf
import logging

MYPY = False
if MYPY:  # pragma: no cover
    from mpf.platforms.opp.opp import OppHardwarePlatform  # pylint: disable-msg=cyclic-import,unused-import

class OPPServo(ServoPlatformInterface):
    """A servo in the OPP platform."""

    __slots__ = ["number", "chain_serial", "platform", "speed", "current_position"]

    def __init__(self, chain_serial, servo_num, platform: "OppHardwarePlatform"):
        """Initialise servo."""
        self.number = servo_num
        self.platform = platform
        self.chain_serial = chain_serial
        self.speed = 0
        self.current_position = 0

    def stop(self):
        """Disable servo.
        Set position to 0 to disable servo.
        """
        self.go_to_position(0)

    def go_to_position(self, position):
        """Set a servo position.

        position [0 to 1.0] is converted to position_numeric [0 to 255].
        position_numeric is measured in 10us intervals, so a position_numeric of 100 is a 1ms pulse
        and 150 is 1.5ms. A position_numeric of 0 disables the servo. Use caution with extreme
        position values as it could cause a servo to drive to a position it cannot reach.
        """

        # convert from [0,1] to [0, 255]
        position_numeric = int(position * 255)
        servo_offset = 0x3000 + self.number

        if position_numeric == 0 or self.current_position == 0 or self.speed <= 0:
            fade_ms = 0
        else:
            fade_ms = 600 * abs(position_numeric - self.current_position) / self.speed
            if fade_ms > 65535:
                fade_ms = 65535

        msg = bytearray()
        msg.append(0x20)
        msg.append(OppRs232Intf.SERIAL_LED_CMD_FADE)
        msg.append(int(servo_offset / 256))
        msg.append(int(servo_offset % 256))
        msg.append(int(0))  #number of servos (high)
        msg.append(int(1))  #number of servos (low)...only commanding one at a time.
        msg.append(int(fade_ms / 256))
        msg.append(int(fade_ms % 256))
        msg.append(position_numeric)
        msg.extend(OppRs232Intf.calc_crc8_whole_msg(msg))
        cmd = bytes(msg)

        self.platform.debug_log("Set servo position on %s: %s", self.chain_serial, "".join(" 0x%02x" % b for b in cmd))

        self.platform.send_to_processor(self.chain_serial, cmd)

        self.current_position = position_numeric

    def set_speed_limit(self, speed_limit):
        """Set the speed of this servo

        For the standard 1ms pulse width change to move a servo between
        extremes, a speed of 1 will take 1 minute, and a speed of 60 would take
        1 second.

        A speed <= 0 is unrestricted by firmware
        """
        self.platform.debug_log("Change servo speed limit on %s: %s", self.chain_serial, int(speed_limit))

        self.speed = speed_limit

    def set_acceleration_limit(self, acceleration_limit):
        """Not implemented."""
<|endoftext|>


<commit_before>#! /usr/bin/python

from decimal import Decimal
from argparse import ArgumentParser

supported_formats="Supported formats are: \
'af': OpenVSP airfoil format.\n           \
'lednicer': Lednicer format as found at http://www.ae.illinois.edu/m-selig/ads.html.\n \
'selig': Selig format as found at http://www.ae.illinois.edu/m-selig/ads.html."""

def _closest_to_origin( points ):
    """Take list of points (2-tuples) and retun the index of the one closest
    to the origin."""
    norm = lambda xx: ( xx[0]**2 + xx[1]**2 )**Decimal(0.5) #two-norm function for sorting

    sorted_points = sorted( points, key=norm ) #ascending sort by distance from origin (leading edge)

    return points.index( sorted_points[0] )

class Airfoil(object):
    comment = ""
    name = ""
    symmetric_flag = False
    num_upper_surface_points = 0
    num_lower_surface_points = 0

    upper_surface_points = []
    lower_surface_points = []

    def __str__(self):
        string = "{comment}\n{name}\n{symmetric}\n{upper}\n".format(
                    comment=self.comment,
                    name=self.name,
                    symmetric=int(self.symmetric_flag),
                    upper=self.num_upper_surface_points )

        if self.symmetric_flag == False:
            string = string+"{0}\n".format( num_lower_surface_points )

        for point in self.upper_surface_points:
            string = string + "{0} {1}\n".format( point[0], point[1] )

        #pad space between upper and lower points
        string = string+"\n"

        for point in self.lower_surface_points:
            string = string + "{0} {1}\n".format( point[0], point[1] )

        return( string )

    def _load_lednicer_like( self, lines ):
        """Common subroutines between load_from_af_format and load_from_lednicer_format

        Loads from a list of lines from a file in Lednicer format. Only the point locations are loaded. """

        #read upper surface points
        for xx in range( self.num_upper_surface_points):
            chord_position = Decimal( lines[xx].split()[0] )
            height = Decimal( lines[xx].split()[1] )
            self.upper_surface_points.append( (chord_position, height) )

        #read lower surface points starting from the line after the blank line
        #that separates upper and lower surface points
        if self.symmetric_flag == False:
            for xx in range( 1+self.num_upper_surface_points,
                         1+self.num_upper_surface_points+self.num_lower_surface_points ):
                chord_position = Decimal( lines[xx].split()[0] )
                height = Decimal( lines[xx].split()[1] )
                self.lower_surface_points.append( (chord_position, height) )

    def load_from_af_format( self, file_id ):
        """Load airfoil from file object in .af format"""
        lines = file_id.readlines()
        self.comment = lines.pop(0).strip('\n')
        self.name = lines.pop(0).strip('\n')
        self.symmetric_flag = bool( int( lines.pop(0).split()[0] ) )
        self.num_upper_surface_points = int( lines.pop(0).split()[0] )
        if self.symmetric_flag == False:
            self.num_lower_surface_points = int( lines.pop(0).split()[0] )
        self._load_lednicer_like( lines )

    def load_from_lednicer_format(self, file_id):
        """Load files from Lednicer format. Seems identical to .af format except
           there is no comment field, no symmetrical field and number of points in different format (upper and lower on the same line)"""
        lines = file_id.readlines()
        self.name = lines.pop(0).strip('\n')
        num_points_line = lines.pop(0).strip().split()
        self.num_upper_surface_points = int( Decimal(num_points_line[0]) )
        self.num_lower_surface_points = int( Decimal(num_points_line[1]) )
        lines.pop(0) #remove blank line
        self._load_lednicer_like( lines )

    def load_from_selig_format(self, file_id ):
        """Load from the Selig format. http://www.ae.illinois.edu/m-selig/ads.html"""
        lines = file_id.readlines()
        self.name = lines.pop(0).strip('\n')
        pointify = lambda xx: (Decimal(line.split()[0]),Decimal(line.split()[1]))

        points = [ pointify(line) for line in lines ]

        #find point closest to leading edge. put preceding points in upper surface points. put points after in lower surface points. if it's positive in height, put it in upper surface points, if it's negative, put it in lower surface points.
        closest_to_le = _closest_to_origin( points )

        upper_points = points[0:closest_to_le]
        upper_points.reverse()
        lower_points = points[closest_to_le+1:]

        if points[closest_to_le][1] >= Decimal(0.0):
            upper_points.insert( 0, points[closest_to_le] )
        else:
            lower_points.insert( 0, points[closest_to_le] )

        #insert leading edge
        upper_points.insert(0, (Decimal(0,0), Decimal(0,0) ) )
        lower_points.insert(0, (Decimal(0,0), Decimal(0,0) ) )

        self.symmetric_flag = False #just set this to false for now. could implement symmetry checking later
        self.num_upper_surface_points = len(upper_points)
        self.num_lower_surface_points = len(lower_points)

        self.upper_surface_points = upper_points
        self.lower_surface_points = lower_points

    def load( self, file_id, format = 'af' ):
        """Load airfoil from file object in file format 'format'

            {}""".format(supported_formats)

        if format == 'af':
            self.load_from_af_format( file_id )

        if format == 'lednicer':
            self.load_from_lednicer_format( file_id )

        if format == 'selig':
            self.load_from_selig_format( file_id )

    def load_file( self, file_name, format = 'selig' ):
        """Load airfoil from file in file format 'format'

            {}""".format(supported_formats)
        file_id = open(file_name)
        self.load( file_id, format )
        file_id.close()

def convert_file( in_file_name, out_file_name, format='selig'  ):
    """Convert in_file_name to out_file_name in OpenVSP .af format from format 'format'. Default input format is 'selig'

       {}""".format(supported_formats)

    out_file = open(out_file_name,'w')

    af = Airfoil()
    af.load_file(in_file_name, format)
    out_file.write( str(af) )
    out_file.close()

def convert_multiple_files( file_list, suffix='.af', format='selig',change_suffix='True' ):
    """Convert list of files in file_list. Appends/changes the suffix. Defaults to changing suffix to .af"""
    for file_name in file_list:
        if change_suffix:
            stem = file_name.rpartition('.')[0]
        else:
            stem = file_name

        new_name = stem+suffix
        convert_file( file_name, new_name, format )

if __name__=="__main__":
    parser = ArgumentParser( description="""Convert airfoil files into OpenVSP airfoil (.af) files.

{}""".format(supported_formats),
    epilog="Author: Nathan Alday, n.c.alday@gmail.com")

    parser.add_argument( 'files', metavar='FILE',
        help='Name of file to be converted',nargs='+')
    parser.add_argument( '--format', metavar='FORMAT',
        help='Format of input files, defaults to selig',
        action='store', default='selig', type=str  )

    args=parser.parse_args()

    convert_multiple_files( args.files, format = args.format )
<commit_msg>Fixed bugs I introduced on last fix. Corrected behavior on write so that only fixed point values get written rather than floating point/engineering notation<commit_after>#! /usr/bin/python

from decimal import Decimal
from argparse import ArgumentParser

supported_formats="Supported formats are: \
'af': OpenVSP airfoil format.\n           \
'lednicer': Lednicer format as found at http://www.ae.illinois.edu/m-selig/ads.html.\n \
'selig': Selig format as found at http://www.ae.illinois.edu/m-selig/ads.html."""

def _closest_to_origin( points ):
    """Take list of points (2-tuples) and retun the index of the one closest
    to the origin."""
    norm = lambda xx: ( xx[0]**2 + xx[1]**2 )**Decimal(0.5) #two-norm function for sorting

    sorted_points = sorted( points, key=norm ) #ascending sort by distance from origin (leading edge)

    return points.index( sorted_points[0] )

class Airfoil(object):
    comment = ""
    name = ""
    symmetric_flag = False
    num_upper_surface_points = 0
    num_lower_surface_points = 0

    upper_surface_points = []
    lower_surface_points = []

    def __str__(self):
        string = "{comment}\n{name}\n{symmetric}\n{upper}\n".format(
                    comment=self.comment,
                    name=self.name,
                    symmetric=int(self.symmetric_flag),
                    upper=self.num_upper_surface_points )

        if self.symmetric_flag == False:
            string = string+"{0}\n".format( self.num_lower_surface_points )

        for point in self.upper_surface_points:
            string = string + "{0:f} {1:f}\n".format( point[0], point[1] )

        #pad space between upper and lower points
        string = string+"\n"

        for point in self.lower_surface_points:
            string = string + "{0:f} {1:f}\n".format( point[0], point[1] )

        return( string )

    def _load_lednicer_like( self, lines ):
        """Common subroutines between load_from_af_format and load_from_lednicer_format

        Loads from a list of lines from a file in Lednicer format. Only the point locations are loaded. """

        #read upper surface points
        for xx in range( self.num_upper_surface_points):
            chord_position = Decimal( lines[xx].split()[0] )
            height = Decimal( lines[xx].split()[1] )
            self.upper_surface_points.append( (chord_position, height) )

        #read lower surface points starting from the line after the blank line
        #that separates upper and lower surface points
        if self.symmetric_flag == False:
            for xx in range( 1+self.num_upper_surface_points,
                         1+self.num_upper_surface_points+self.num_lower_surface_points ):
                chord_position = Decimal( lines[xx].split()[0] )
                height = Decimal( lines[xx].split()[1] )
                self.lower_surface_points.append( (chord_position, height) )

    def load_from_af_format( self, file_id ):
        """Load airfoil from file object in .af format"""
        lines = file_id.readlines()
        self.comment = lines.pop(0).strip('\n')
        self.name = lines.pop(0).strip('\n')
        self.symmetric_flag = bool( int( lines.pop(0).split()[0] ) )
        self.num_upper_surface_points = int( lines.pop(0).split()[0] )
        if self.symmetric_flag == False:
            self.num_lower_surface_points = int( lines.pop(0).split()[0] )
        self._load_lednicer_like( lines )

    def load_from_lednicer_format(self, file_id):
        """Load files from Lednicer format. Seems identical to .af format except
           there is no comment field, no symmetrical field and number of points in different format (upper and lower on the same line)"""
        lines = file_id.readlines()
        self.name = lines.pop(0).strip('\n')
        num_points_line = lines.pop(0).strip().split()
        self.num_upper_surface_points = int( Decimal(num_points_line[0]) )
        self.num_lower_surface_points = int( Decimal(num_points_line[1]) )
        lines.pop(0) #remove blank line
        self._load_lednicer_like( lines )

    def load_from_selig_format(self, file_id ):
        """Load from the Selig format. http://www.ae.illinois.edu/m-selig/ads.html"""
        lines = file_id.readlines()
        self.name = lines.pop(0).strip('\n')
        pointify = lambda xx: (Decimal(line.split()[0]),Decimal(line.split()[1]))

        points = [ pointify(line) for line in lines ]

        #find point closest to leading edge. put preceding points in upper surface points. put points after in lower surface points. if it's positive in height, put it in upper surface points, if it's negative, put it in lower surface points.
        closest_to_le = _closest_to_origin( points )

        upper_points = points[0:closest_to_le]
        upper_points.reverse()
        lower_points = points[closest_to_le+1:]

        if points[closest_to_le][1] >= Decimal(0.0):
            upper_points.insert( 0, points[closest_to_le] )
        else:
            lower_points.insert( 0, points[closest_to_le] )

        #insert leading edge
        upper_points.insert(0, (Decimal(0,0), Decimal(0,0) ) )
        lower_points.insert(0, (Decimal(0,0), Decimal(0,0) ) )

        self.symmetric_flag = False #just set this to false for now. could implement symmetry checking later
        self.num_upper_surface_points = len(upper_points)
        self.num_lower_surface_points = len(lower_points)

        self.upper_surface_points = upper_points
        self.lower_surface_points = lower_points

    def load( self, file_id, format = 'af' ):
        """Load airfoil from file object in file format 'format'

            {}""".format(supported_formats)

        if format == 'af':
            self.load_from_af_format( file_id )

        if format == 'lednicer':
            self.load_from_lednicer_format( file_id )

        if format == 'selig':
            self.load_from_selig_format( file_id )

    def load_file( self, file_name, format = 'selig' ):
        """Load airfoil from file in file format 'format'

            {}""".format(supported_formats)
        file_id = open(file_name)
        self.load( file_id, format )
        file_id.close()

def convert_file( in_file_name, out_file_name, format='selig'  ):
    """Convert in_file_name to out_file_name in OpenVSP .af format from format 'format'. Default input format is 'selig'

       {}""".format(supported_formats)

    out_file = open(out_file_name,'w')

    af = Airfoil()
    af.load_file(in_file_name, format)
    out_file.write( str(af) )
    out_file.close()

def convert_multiple_files( file_list, suffix='.af', format='selig',change_suffix='True' ):
    """Convert list of files in file_list. Appends/changes the suffix. Defaults to changing suffix to .af"""
    for file_name in file_list:
        if change_suffix:
            stem = file_name.rpartition('.')[0]
        else:
            stem = file_name

        new_name = stem+suffix
        convert_file( file_name, new_name, format )

if __name__=="__main__":
    parser = ArgumentParser( description="""Convert airfoil files into OpenVSP airfoil (.af) files.

{}""".format(supported_formats),
    epilog="Author: Nathan Alday, n.c.alday@gmail.com")

    parser.add_argument( 'files', metavar='FILE',
        help='Name of file to be converted',nargs='+')
    parser.add_argument( '--format', metavar='FORMAT',
        help='Format of input files, defaults to selig',
        action='store', default='selig', type=str  )

    args=parser.parse_args()

    convert_multiple_files( args.files, format = args.format )
<|endoftext|>


<commit_before># Third-Party
import rest_framework_filters as filters
from django_filters import Filter
from django_filters.fields import Lookup
from dry_rest_permissions.generics import DRYPermissionFiltersBase
from rest_framework.filters import BaseFilterBackend

# Local
from .models import (
    Catalog,
    Judge,
    Contestant,
    Convention,
    Group,
    Performer,
    Person,
    Round,
    Session,
    Submission,
    Venue,
)

class CoalesceFilterBackend(BaseFilterBackend):
    """Support Ember Data coalesceFindRequests."""

    def filter_queryset(self, request, queryset, view):
        raw = request.query_params.get('filter[id]')
        if raw:
            ids = raw.split(',')
            # Disable pagination, so all records can load.
            view.pagination_class = None
            queryset = queryset.filter(id__in=ids)
        return queryset

class RoundFilterBackend(DRYPermissionFiltersBase):
    def filter_list_queryset(self, request, queryset, view):
        """Limit all list requests to at least validated if not superuser."""
        if request.user.is_staff:
            return queryset.all()
        else:
            return queryset.filter(status__gte=Round.STATUS.validated)

class ConventionFilterBackend(DRYPermissionFiltersBase):
    def filter_list_queryset(self, request, queryset, view):
        """Limit all list requests to at least validated if not superuser."""
        if request.user.is_staff:
            return queryset.all()
        else:
            return queryset.filter(status__gte=Round.STATUS.listed)

class SessionFilterBackend(DRYPermissionFiltersBase):
    def filter_list_queryset(self, request, queryset, view):
        """Limit all list requests to at least validated if not superuser."""
        if request.user.is_staff:
            return queryset.all()
        else:
            return queryset.filter(status__gte=Session.STATUS.listed)

class ListFilter(Filter):
    def filter(self, qs, value):
        value_list = value.split(u',')
        return super(ListFilter, self).filter(qs, Lookup(value_list, 'in'))

class CatalogFilter(filters.FilterSet):

    class Meta:
        model = Catalog
        fields = {
            'title': '__all__',
        }

class ConventionFilter(filters.FilterSet):
    season = ListFilter(name='season')

    class Meta:
        model = Convention
        fields = {
            'status': '__all__',
            'year': '__all__',
            # 'season': '__all__',
        }

class GroupFilter(filters.FilterSet):
    class Meta:
        model = Group
        fields = {
            'name': '__all__',
            'nomen': '__all__',
            'kind': '__all__',
            'status': '__all__',
        }

class JudgeFilter(filters.FilterSet):
    class Meta:
        model = Judge
        fields = {
            'name': '__all__',
            'category': '__all__',
        }

class ContestantFilter(filters.FilterSet):
    class Meta:
        model = Contestant
        fields = {
            'name': '__all__',
        }

class PerformerFilter(filters.FilterSet):
    class Meta:
        model = Performer
        fields = {
            'name': '__all__',
        }

class PersonFilter(filters.FilterSet):
    judges__category = ListFilter(name='judges__category')

    class Meta:
        model = Person
        fields = [
            'status',
        ]
        fields = {
            'name': '__all__',
            'nomen': '__all__',
            # 'judges__category': '__all__',
        }

class SubmissionFilter(filters.FilterSet):
    class Meta:
        model = Submission
        fields = {
            'performer__id': '__all__',
        }

class VenueFilter(filters.FilterSet):
    class Meta:
        model = Venue
        fields = {
            'name': '__all__',
        }
<commit_msg>First pass related filter<commit_after># Third-Party
import rest_framework_filters as filters
from dry_rest_permissions.generics import DRYPermissionFiltersBase
from rest_framework.filters import BaseFilterBackend

# Local
from .models import (
    Catalog,
    Judge,
    Contestant,
    Convention,
    Group,
    Performer,
    Person,
    Round,
    Session,
    Submission,
    Venue,
)

class CoalesceFilterBackend(BaseFilterBackend):
    """Support Ember Data coalesceFindRequests."""

    def filter_queryset(self, request, queryset, view):
        raw = request.query_params.get('filter[id]')
        if raw:
            ids = raw.split(',')
            # Disable pagination, so all records can load.
            view.pagination_class = None
            queryset = queryset.filter(id__in=ids)
        return queryset

class RoundFilterBackend(DRYPermissionFiltersBase):
    def filter_list_queryset(self, request, queryset, view):
        """Limit all list requests to at least validated if not superuser."""
        if request.user.is_staff:
            return queryset.all()
        else:
            return queryset.filter(status__gte=Round.STATUS.validated)

class ConventionFilterBackend(DRYPermissionFiltersBase):
    def filter_list_queryset(self, request, queryset, view):
        """Limit all list requests to at least validated if not superuser."""
        if request.user.is_staff:
            return queryset.all()
        else:
            return queryset.filter(status__gte=Round.STATUS.listed)

class SessionFilterBackend(DRYPermissionFiltersBase):
    def filter_list_queryset(self, request, queryset, view):
        """Limit all list requests to at least validated if not superuser."""
        if request.user.is_staff:
            return queryset.all()
        else:
            return queryset.filter(status__gte=Session.STATUS.listed)

class CatalogFilter(filters.FilterSet):

    class Meta:
        model = Catalog
        fields = {
            'title': '__all__',
        }

class ConventionFilter(filters.FilterSet):

    class Meta:
        model = Convention
        fields = {
            'status': '__all__',
            'year': '__all__',
        }

class GroupFilter(filters.FilterSet):
    name = filters.AllLookupsFilter(name='name')

    class Meta:
        model = Group
        fields = {
            'nomen': '__all__',
            'kind': '__all__',
            'status': '__all__',
        }

class JudgeFilter(filters.FilterSet):
    class Meta:
        model = Judge
        fields = {
            'name': '__all__',
            'category': '__all__',
        }

class ContestantFilter(filters.FilterSet):
    class Meta:
        model = Contestant
        fields = {
            'name': '__all__',
        }

class PerformerFilter(filters.FilterSet):
    group = filters.RelatedFilter(GroupFilter, name='group')

    class Meta:
        model = Performer
        fields = {
            'name': '__all__',
        }

class PersonFilter(filters.FilterSet):

    class Meta:
        model = Person
        fields = [
            'status',
        ]
        fields = {
            'name': '__all__',
            'nomen': '__all__',
        }

class SubmissionFilter(filters.FilterSet):
    class Meta:
        model = Submission
        fields = {
            'performer__id': '__all__',
        }

class VenueFilter(filters.FilterSet):
    class Meta:
        model = Venue
        fields = {
            'name': '__all__',
        }
<|endoftext|>


<commit_before>"""
Use this module to start a new Pabiana Area.
Each Area is initialized with a Receiver Interface.
The Receiver Interface accepts Requests for remote Triggers.
Subscription Interfaces are created as defined.
Subscription Interfaces call defined Reactions.
The Area name must not contain "_" characters.
A Publishing Interface can be added optionally.
"""

import json
import logging
import os
import sys
from threading import Thread
import time

import zmq

config = {}
_cbks = {}
_trgs = {}
goon = True

def rslv(interface):
	"""
	Returns a dictionary containing the ip and the port of the interface.
	"""
	with open(config['ifs-path'], encoding='utf-8') as f:
		ifs = json.load(f)
	return ifs[interface]

def register(func):
	"""
	Registers this function as remote Trigger.
	"""
	_trgs[func.__name__] = func
	return func

def reg_name(name):
	"""
	Registers this function as remote Trigger with the specified name.
	"""
	def decorator(func):
		_trgs[name] = func
		return func
	return decorator

def split_func_name(name):
	pos = name.find('_')
	if pos == -1:
		pos = len(name)
	area_name = name[:pos]
	topic = name[pos + 1:]
	topic = topic.replace('_', '.')
	topic = topic.replace('𒐀', '-')
	return area_name, topic

def subscribe(func):
	"""
	Registers this function as a Reaction to a Subscription.

	The area name must be separated from the topic through a "_" character.
	Within the topic, "_" characters are replaced with topic separators ("." characters).
	Within the topic, "𒐀" characters are replaced with "-" characters.
	"""
	area_name, topic = split_func_name(func.__name__)
	_cbks[area_name] = {}
	_cbks[area_name][topic] = func
	return func

def sub_name(name, topic):
	"""
	Registers this function as a Reaction to a Subscription with the specified Area name and topic.
	"""
	def decorator(func):
		_cbks[name] = {}
		_cbks[name][topic] = func
		return func
	return decorator

# TODO: Add functions to subscribe/unsubscribe

def _decoder(rcvd):
	if len(rcvd) == 1:
		rcvd = [b''] + rcvd
	[topic, message] = [x.decode('utf-8') for x in rcvd]
	message = json.loads(message)
	return [topic, message]

def run(own_name, host=None, delay=0, timeout=1000):
	"""
	Starts a new Pabiana Area.
	Use Control-C to stop.
	"""
	config['name'] = own_name
	context = zmq.Context.instance()
	poller = zmq.Poller()

	# Receiver Interface
	ip = rslv(own_name + '-rcv')
	host = host or ip['ip']
	receiver = context.socket(zmq.REP)
	receiver.bind('tcp://{}:{}'.format(host, ip['port']))
	poller.register(receiver, zmq.POLLIN)
	logging.info('Waiting for Connections on %s:%s', host, ip['port'])

	# Subscription Interfaces
	subs = {}
	for area_nm in _cbks:
		ip = rslv(area_nm + '-pub')
		subscriber = context.socket(zmq.SUB)
		subscriber.connect('tcp://{}:{}'.format(ip['ip'], ip['port']))
		for topic in _cbks[area_nm]:
			subscriber.setsockopt(zmq.SUBSCRIBE, topic.encode('utf-8'))
		poller.register(subscriber, zmq.POLLIN)
		subs[subscriber] = area_nm
	logging.info('Listening to %s', list(_cbks))

	try:
		global goon
		goon = True
		update = False
		if own_name in _cbks and 'update' in _cbks[own_name]:
			update = True
		while goon:  # TODO: What happens if > 1 messages
			socks = dict(poller.poll(timeout))
			for sock in socks:
				if sock == receiver:
					message = receiver.recv_json()
					logging.debug('Receiver Message: %s', message)
					receiver.send(b'')
					func_name = message['function']
					del message['function']
					try:
						_trgs[func_name](**message)
					except KeyError:
						logging.warning('Unavailable Trigger called')
				else:
					area_nm = subs[sock]
					[topic, message] = _decoder(sock.recv_multipart())
					logging.debug('Subscriber Message from %s: %s/%s', area_nm, topic, message)
					key = next(key for key in _cbks[area_nm] if topic.startswith(key))
					_cbks[area_nm][key](**message)
			if update:
				_cbks['internal']['update']()
			if delay:
				time.sleep(delay)
	except KeyboardInterrupt:
		pass
	finally:
		context.destroy()
		logging.debug('Context destroyed')

def create_publisher(own_name, host=None):
	"""
	Creates and returns an optional Publishing Interface.
	"""
	ip = rslv(own_name + '-pub')
	host = host or ip['ip']
	context = zmq.Context.instance()
	publisher = context.socket(zmq.PUB)
	publisher.bind('tcp://{}:{}'.format(host, ip['port']))
	return publisher

def trigger(area_name, trigger_name, params={}, context=None):
	"""
	Sends a Request for a remote Trigger to a Receiver Interface.
	"""
	ip = rslv(area_name + '-rcv')
	context = context or zmq.Context.instance()
	requester = context.socket(zmq.REQ)
	requester.connect('tcp://{}:{}'.format(ip['ip'], ip['port']))
	requester.setsockopt(zmq.LINGER, 0)
	params['function'] = trigger_name
	requester.send_json(params)
	requester.recv()
	requester.close()
	logging.debug('Trigger %s of %s called', trigger_name, area_name)

def _timer(seconds):
	time.sleep(seconds)
	trigger(config['name'], 'timer', context=zmq.Context())

def set_timer(seconds):
	"""
	Sends a Request to the 'timer'-Trigger of this Area, after the specified number of seconds.
	"""
	Thread(target=_timer, args=(seconds,), daemon=True).start()

def __init():
	config['name'] = None
	if sys.argv[0]:
		config['main-path'] = os.path.dirname(os.path.realpath(sys.argv[0]))
		config['global-path'] = os.path.dirname(config['main-path'])
	else:
		config['main-path'] = os.getcwd()
		config['global-path'] = config['main-path']
	config['ifs-path'] = os.path.join(config['main-path'], 'interfaces.json')
	if not os.path.isfile(config['ifs-path']):
		config['ifs-path'] = os.path.join(config['global-path'], 'interfaces.json')

__init()
<commit_msg>Self-subscription problem<commit_after>"""
Use this module to start a new Pabiana Area.
Each Area is initialized with a Receiver Interface.
The Receiver Interface accepts Requests for remote Triggers.
Subscription Interfaces are created as defined.
Subscription Interfaces call defined Reactions.
The Area name must not contain "_" characters.
A Publishing Interface can be added optionally.
"""

import json
import logging
import os
import sys
from threading import Thread
import time

import zmq

config = {}
_cbks = {}
_trgs = {}
goon = True

def rslv(interface):
	"""
	Returns a dictionary containing the ip and the port of the interface.
	"""
	with open(config['ifs-path'], encoding='utf-8') as f:
		ifs = json.load(f)
	return ifs[interface]

def register(func):
	"""
	Registers this function as remote Trigger.
	"""
	_trgs[func.__name__] = func
	return func

def reg_name(name):
	"""
	Registers this function as remote Trigger with the specified name.
	"""
	def decorator(func):
		_trgs[name] = func
		return func
	return decorator

def split_func_name(name):
	pos = name.find('_')
	if pos == -1:
		pos = len(name)
	area_name = name[:pos]
	topic = name[pos + 1:]
	topic = topic.replace('_', '.')
	topic = topic.replace('𒐀', '-')
	return area_name, topic

def subscribe(func):
	"""
	Registers this function as a Reaction to a Subscription.

	The area name must be separated from the topic through a "_" character.
	Within the topic, "_" characters are replaced with topic separators ("." characters).
	Within the topic, "𒐀" characters are replaced with "-" characters.
	"""
	area_name, topic = split_func_name(func.__name__)
	_cbks[area_name] = {}
	_cbks[area_name][topic] = func
	return func

def sub_name(name, topic):
	"""
	Registers this function as a Reaction to a Subscription with the specified Area name and topic.
	"""
	def decorator(func):
		_cbks[name] = {}
		_cbks[name][topic] = func
		return func
	return decorator

# TODO: Add functions to subscribe/unsubscribe

def _decoder(rcvd):
	if len(rcvd) == 1:
		rcvd = [b''] + rcvd
	[topic, message] = [x.decode('utf-8') for x in rcvd]
	message = json.loads(message)
	return [topic, message]

def run(own_name, host=None, delay=0, timeout=1000):
	"""
	Starts a new Pabiana Area.
	Use Control-C to stop.
	"""
	config['name'] = own_name
	context = zmq.Context.instance()
	poller = zmq.Poller()

	# Receiver Interface
	ip = rslv(own_name + '-rcv')
	host = host or ip['ip']
	receiver = context.socket(zmq.REP)
	receiver.bind('tcp://{}:{}'.format(host, ip['port']))
	poller.register(receiver, zmq.POLLIN)
	logging.info('Waiting for Connections on %s:%s', host, ip['port'])

	# Subscription Interfaces
	subs = {}
	for area_nm in _cbks:
		if area_nm == 'internal':
			continue
		ip = rslv(area_nm + '-pub')
		subscriber = context.socket(zmq.SUB)
		subscriber.connect('tcp://{}:{}'.format(ip['ip'], ip['port']))
		for topic in _cbks[area_nm]:
			subscriber.setsockopt(zmq.SUBSCRIBE, topic.encode('utf-8'))
		poller.register(subscriber, zmq.POLLIN)
		subs[subscriber] = area_nm
	logging.info('Listening to %s', list(_cbks))

	try:
		global goon
		goon = True
		update = False
		if own_name in _cbks and 'update' in _cbks[own_name]:
			update = True
		while goon:  # TODO: What happens if > 1 messages
			socks = dict(poller.poll(timeout))
			for sock in socks:
				if sock == receiver:
					message = receiver.recv_json()
					logging.debug('Receiver Message: %s', message)
					receiver.send(b'')
					func_name = message['function']
					del message['function']
					try:
						_trgs[func_name](**message)
					except KeyError:
						logging.warning('Unavailable Trigger called')
				else:
					area_nm = subs[sock]
					[topic, message] = _decoder(sock.recv_multipart())
					logging.debug('Subscriber Message from %s: %s/%s', area_nm, topic, message)
					key = next(key for key in _cbks[area_nm] if topic.startswith(key))
					_cbks[area_nm][key](**message)
			if update:
				_cbks['internal']['update']()
			if delay:
				time.sleep(delay)
	except KeyboardInterrupt:
		pass
	finally:
		context.destroy()
		logging.debug('Context destroyed')

def create_publisher(own_name, host=None):
	"""
	Creates and returns an optional Publishing Interface.
	"""
	ip = rslv(own_name + '-pub')
	host = host or ip['ip']
	context = zmq.Context.instance()
	publisher = context.socket(zmq.PUB)
	publisher.bind('tcp://{}:{}'.format(host, ip['port']))
	return publisher

def trigger(area_name, trigger_name, params={}, context=None):
	"""
	Sends a Request for a remote Trigger to a Receiver Interface.
	"""
	ip = rslv(area_name + '-rcv')
	context = context or zmq.Context.instance()
	requester = context.socket(zmq.REQ)
	requester.connect('tcp://{}:{}'.format(ip['ip'], ip['port']))
	requester.setsockopt(zmq.LINGER, 0)
	params['function'] = trigger_name
	requester.send_json(params)
	requester.recv()
	requester.close()
	logging.debug('Trigger %s of %s called', trigger_name, area_name)

def _timer(seconds):
	time.sleep(seconds)
	trigger(config['name'], 'timer', context=zmq.Context())

def set_timer(seconds):
	"""
	Sends a Request to the 'timer'-Trigger of this Area, after the specified number of seconds.
	"""
	Thread(target=_timer, args=(seconds,), daemon=True).start()

def __init():
	config['name'] = None
	if sys.argv[0]:
		config['main-path'] = os.path.dirname(os.path.realpath(sys.argv[0]))
		config['global-path'] = os.path.dirname(config['main-path'])
	else:
		config['main-path'] = os.getcwd()
		config['global-path'] = config['main-path']
	config['ifs-path'] = os.path.join(config['main-path'], 'interfaces.json')
	if not os.path.isfile(config['ifs-path']):
		config['ifs-path'] = os.path.join(config['global-path'], 'interfaces.json')

__init()
<|endoftext|>


<commit_before>import getpass
import glob
import grp
import logging
import os
import os.path
import pickle
import sys
import time

from . import envs, dput, run, util

log = logging.getLogger(__name__)

DEFAULT_PKGS = [
	'build-essential',
	'ccache',
	'eatmydata',
	'lintian',
	'debhelper',
]

CCACHE_DIR = '/var/cache/ccache-sbuild'

SUFFIX = '-debs'

def _preconfigure():
	# Share apt archive cache
	_check_file_has(
		'/etc/schroot/sbuild/fstab',
		'/var/cache/apt/archives',
		rest=' /var/cache/apt/archives	none rw,bind	0	0',
		ignore_missing=True)

	_check_ccache()
	_check_user_in_group()

def _check_ccache():
	if not os.path.exists(CCACHE_DIR):
		run.check(
			'sudo',
			'install',
			'--compare',
			'--group=sbuild',
			'--mode=2775', # Inherit sbuild group
			'-d', CCACHE_DIR)

	# Share ccache stuffs
	_check_file_has(
		'/etc/schroot/sbuild/fstab',
		'{}'.format(CCACHE_DIR),
		rest='	{}	none rw,bind	0	0'.format(CCACHE_DIR),
		ignore_missing=True)

	_check_file_is(
		os.path.join(CCACHE_DIR, 'ccache.conf'),
		util.strip_lines('''
		cache_dir = {DIR}
		umask = 002
		hard_link = false
		compression = true
		'''.format(
			DIR=CCACHE_DIR)))

	_check_file_is(
		os.path.join(CCACHE_DIR, 'env'),
		util.strip_lines('''#! /bin/bash
		export CCACHE_DIR={}
		export PATH="/usr/lib/ccache:$PATH"
		exec "$@"
		'''.format(CCACHE_DIR)),
		executable=True)

def _check_user_in_group():
	u = getpass.getuser()
	if u != 'root' and u not in grp.getgrnam('sbuild').gr_mem:
		log.error('Current user (%s) not in sbuild group. Adding...', u)
		run.check('sudo', 'adduser', u, 'sbuild')
		log.error('You need to log out and log back in before you can use sbuild.')

		if not os.getenv('CI'):
			sys.exit(1)

def _dir(release, arch):
	return '/var/lib/debs/{}-{}'.format(release, arch)

def _cfg_file(release, arch):
	f = '/etc/schroot/chroot.d/{}-{}-debs-*'.format(release, arch)
	gs = glob.glob(f)
	if gs:
		return gs[0]
	return f

def _schroot_name(release, arch, source=False):
	name = '{}-{}{}'.format(release, arch, SUFFIX)
	if source:
		name = 'source:{}'.format(name)

	return name

def _run_in_chroot(release, arch, *args):
	run.check(
		'sudo',
		'schroot',
		'-c', _schroot_name(release, arch, source=True),
		'--directory', '/root',
		'--', *args)

def _create(cfg, release, arch):
	dir = _dir(release, arch)
	if os.path.exists(dir):
		raise SbuildException(
			'Cowardly refusing to create '
			'{}-{} in existing dir: {}'.format(release, arch, dir))

	try:
		run.check('sudo', 'mkdir', '-p', dir)

		run.check(
			'sudo',
			'sbuild-createchroot',
			'--chroot-suffix={}'.format(SUFFIX),
			'--arch={}'.format(arch),
			'--components={}'.format(envs.components(release, sep=',')),
			release,
			dir,
			envs.main_mirror(release, cfg=cfg))

		_reconfig(cfg, release, arch)
	except:
		_delete(release, arch)
		raise

def _reconfig(cfg, release, arch):
	dir = _dir(release, arch)
	p = os.path.join(dir, '.debscfg')

	c = {}
	cs = None
	lastmod = -1
	if os.path.exists(p):
		with open(p, 'rb') as f:
			cs = f.read()
		c = pickle.loads(cs)
		lastmod = os.path.getmtime(p)

	_check_cfg_has(release, arch, 'union-type=', 'overlay')
	_reconfig_mirrors(release, arch, cfg, dir)
	_reconfig_pkgs(release, arch, cfg, c)
	_check_cfg_has(release, arch,
		'command-prefix=',
		'{}/env,eatmydata'.format(CCACHE_DIR))

	upgraded = False
	if lastmod != -1:
		upgraded = _check_upgrade(cfg, lastmod, release, arch)

	ncs = pickle.dumps(c)
	if ncs != cs or upgraded:
		run.write(p, ncs)

def _check_cfg_has(release, arch, pfx, rest):
	_check_file_has(_cfg_file(release, arch), pfx, rest)

def _reconfig_mirrors(release, arch, cfg, dir):
	srcs = os.path.join(dir, 'etc', 'apt', 'sources.list')

	if not _check_file_is(srcs, envs.sources(release, arch, cfg=cfg)):
		_upgrade(release, arch)

def _check_file_has(path, pfx, rest, ignore_missing=False):
	if ignore_missing and not os.path.exists(path):
		return

	with open(path) as f:
		contents = f.read()

	if not contents.endswith('\n'):
		contents += '\n'

	complete = pfx + rest
	if not complete.endswith('\n'):
		complete += '\n'

	if complete in contents:
		return

	lines = contents.splitlines()
	for i, l in enumerate(lines):
		if l.startswith(pfx):
			lines[i] = complete.strip()
			break
	else:
		lines.append(complete.strip())

	out = '\n'.join(lines) + '\n'
	run.write(path, out)

def _check_file_is(path, contents, executable=False):
	have = ''
	if os.path.exists(path):
		with open(path) as f:
			have = f.read()

	had = contents == have
	if not had:
		run.write(path, contents)

	if executable and not had:
		run.check('sudo', 'chmod', '+x', path)

	return had

def _reconfig_pkgs(release, arch, cfg, c):
	have = set(c.get('pkgs', []))
	want = envs.packages(release, arch, cfg=cfg) | set(DEFAULT_PKGS)

	if have != want:
		rm = have - want
		if rm:
			_run_in_chroot(release, arch,
				'apt-get',
				'remove', *rm)

		inst = want - have
		if inst:
			_run_in_chroot(release, arch,
				'apt-get',
				'-y', '--force-yes',
				'install', *inst)

	c['pkgs'] = sorted(want)

def _check_upgrade(cfg, lastmod, release, arch):
	now = time.time()
	age = now - lastmod

	if age < cfg.refresh_after:
		return False

	log.info('%s-%s out of date; upgrading...', release, arch)
	_upgrade(release, arch)

	return True

def _upgrade(release, arch):
	run.check(
		'sudo',
		'sbuild-update',
		'-udcar',
		_schroot_name(release, arch))

def _make_sbuildrc(cfg, path):
	require = None
	curr = os.getenv('SBUILD_CONFIG')
	if curr:
		curr = os.path.abspath(curr)
		if os.path.exists(curr):
			require = curr

	with open(path, 'w') as f:
		if require:
			f.write('require "{}";\n'.format(require))

		if cfg.key:
			f.write('$key_id = "{}";\n'.format(cfg.key))

		f.write(util.strip_lines('''
			$run_lintian = {LINTIAN};
			$lintian_opts = {LINTIAN_OPTS};

			$apt_update = {APT_UPGRADE};
			$apt_distupgrade = {APT_UPGRADE};

			1;
			'''.format(
				LINTIAN=int(cfg.lintian),
				LINTIAN_OPTS=cfg.lintian_args,
				APT_UPGRADE=int(cfg.apt_upgrade),
			)))

def _delete(release, arch):
	name = _schroot_name(release, arch)

	run.ignore('sudo', 'schroot',
		'--end-session', '--all-sessions',
		'chroot:{}'.format(name))
	run.ignore('sudo', 'schroot',
		'--end-session', '--all-sessions',
		'source:{}'.format(name))
	run.ignore('sudo', 'rm', '-rf', _dir(release, arch))
	run.ignore('sudo', 'rm', '-rf', _cfg_file(release, arch))

def delete(env):
	release, arch = envs.split(env)
	_delete(release, arch)

def installed():
	envs = set()

	scs = run.get('schroot', '-l', '--all-source-chroots').splitlines()
	for sc in filter(lambda s: SUFFIX in s, scs):
		envs.add(sc
			.replace('source:', '')
			.replace(SUFFIX, ''))

	return sorted(envs)

def match(*specs, only_installed=False):
	"""
	Find all sbuild environments matching the given specs.
	"""

	inst = set(installed())
	ds = envs.match(*specs, installed=inst)

	if only_installed:
		ds = ds.intersection(inst)

	return sorted(ds)

def ensure(cfg, *envss):
	"""
	Ensure that an sbuild environment for the given release-arch environments
	exist.
	"""

	inst = installed()
	for env in envss:
		release, arch = envs.split(env)

		if env in inst:
			_reconfig(cfg, release, arch)
		else:
			_create(cfg, release, arch)

def build(pkg, env, remotes=[]):
	"""
	Build the given package for in the given environment. Return if the
	package was uploaded anywhere.
	"""

	cfg = pkg.cfg

	if not cfg.dry_run:
		ensure(cfg, env)

	release, arch = envs.split(env)

	with util.tmpdir() as tmpdir:
		sbuildrc = os.path.join(tmpdir, '.sbuildrc')
		dsc = pkg.gen_src(tmpdir)
		_make_sbuildrc(cfg, sbuildrc)

		if cfg.dry_run:
			return

		args = []
		if cfg.user:
			args += ['sudo']

			if cfg.user_keep_env:
				args += ['-E']

			args += ['-u', cfg.user]

		args += [
			'sbuild',
			'--arch-all',
			'-j', str(cfg.jobs),
			'--dist', release,
			'--chroot', _schroot_name(release, arch),
		]

		if cfg.include_src:
			args.append('--source')

		args.append(dsc)

		run.check(
			*args,
			cwd=tmpdir,
			env={
				'SBUILD_CONFIG': sbuildrc,
			})

		chgs = glob.glob(os.path.join(tmpdir, '*.changes'))[0]
		for r in remotes:
			dput.put(r, chgs)

class SbuildException(Exception):
	pass

_preconfigure()
<commit_msg>sbuild: Always upload original source<commit_after>import getpass
import glob
import grp
import logging
import os
import os.path
import pickle
import sys
import time

from . import envs, dput, run, util

log = logging.getLogger(__name__)

DEFAULT_PKGS = [
	'build-essential',
	'ccache',
	'eatmydata',
	'lintian',
	'debhelper',
]

CCACHE_DIR = '/var/cache/ccache-sbuild'

SUFFIX = '-debs'

def _preconfigure():
	# Share apt archive cache
	_check_file_has(
		'/etc/schroot/sbuild/fstab',
		'/var/cache/apt/archives',
		rest=' /var/cache/apt/archives	none rw,bind	0	0',
		ignore_missing=True)

	_check_ccache()
	_check_user_in_group()

def _check_ccache():
	if not os.path.exists(CCACHE_DIR):
		run.check(
			'sudo',
			'install',
			'--compare',
			'--group=sbuild',
			'--mode=2775', # Inherit sbuild group
			'-d', CCACHE_DIR)

	# Share ccache stuffs
	_check_file_has(
		'/etc/schroot/sbuild/fstab',
		'{}'.format(CCACHE_DIR),
		rest='	{}	none rw,bind	0	0'.format(CCACHE_DIR),
		ignore_missing=True)

	_check_file_is(
		os.path.join(CCACHE_DIR, 'ccache.conf'),
		util.strip_lines('''
		cache_dir = {DIR}
		umask = 002
		hard_link = false
		compression = true
		'''.format(
			DIR=CCACHE_DIR)))

	_check_file_is(
		os.path.join(CCACHE_DIR, 'env'),
		util.strip_lines('''#! /bin/bash
		export CCACHE_DIR={}
		export PATH="/usr/lib/ccache:$PATH"
		exec "$@"
		'''.format(CCACHE_DIR)),
		executable=True)

def _check_user_in_group():
	u = getpass.getuser()
	if u != 'root' and u not in grp.getgrnam('sbuild').gr_mem:
		log.error('Current user (%s) not in sbuild group. Adding...', u)
		run.check('sudo', 'adduser', u, 'sbuild')
		log.error('You need to log out and log back in before you can use sbuild.')

		if not os.getenv('CI'):
			sys.exit(1)

def _dir(release, arch):
	return '/var/lib/debs/{}-{}'.format(release, arch)

def _cfg_file(release, arch):
	f = '/etc/schroot/chroot.d/{}-{}-debs-*'.format(release, arch)
	gs = glob.glob(f)
	if gs:
		return gs[0]
	return f

def _schroot_name(release, arch, source=False):
	name = '{}-{}{}'.format(release, arch, SUFFIX)
	if source:
		name = 'source:{}'.format(name)

	return name

def _run_in_chroot(release, arch, *args):
	run.check(
		'sudo',
		'schroot',
		'-c', _schroot_name(release, arch, source=True),
		'--directory', '/root',
		'--', *args)

def _create(cfg, release, arch):
	dir = _dir(release, arch)
	if os.path.exists(dir):
		raise SbuildException(
			'Cowardly refusing to create '
			'{}-{} in existing dir: {}'.format(release, arch, dir))

	try:
		run.check('sudo', 'mkdir', '-p', dir)

		run.check(
			'sudo',
			'sbuild-createchroot',
			'--chroot-suffix={}'.format(SUFFIX),
			'--arch={}'.format(arch),
			'--components={}'.format(envs.components(release, sep=',')),
			release,
			dir,
			envs.main_mirror(release, cfg=cfg))

		_reconfig(cfg, release, arch)
	except:
		_delete(release, arch)
		raise

def _reconfig(cfg, release, arch):
	dir = _dir(release, arch)
	p = os.path.join(dir, '.debscfg')

	c = {}
	cs = None
	lastmod = -1
	if os.path.exists(p):
		with open(p, 'rb') as f:
			cs = f.read()
		c = pickle.loads(cs)
		lastmod = os.path.getmtime(p)

	_check_cfg_has(release, arch, 'union-type=', 'overlay')
	_reconfig_mirrors(release, arch, cfg, dir)
	_reconfig_pkgs(release, arch, cfg, c)
	_check_cfg_has(release, arch,
		'command-prefix=',
		'{}/env,eatmydata'.format(CCACHE_DIR))

	upgraded = False
	if lastmod != -1:
		upgraded = _check_upgrade(cfg, lastmod, release, arch)

	ncs = pickle.dumps(c)
	if ncs != cs or upgraded:
		run.write(p, ncs)

def _check_cfg_has(release, arch, pfx, rest):
	_check_file_has(_cfg_file(release, arch), pfx, rest)

def _reconfig_mirrors(release, arch, cfg, dir):
	srcs = os.path.join(dir, 'etc', 'apt', 'sources.list')

	if not _check_file_is(srcs, envs.sources(release, arch, cfg=cfg)):
		_upgrade(release, arch)

def _check_file_has(path, pfx, rest, ignore_missing=False):
	if ignore_missing and not os.path.exists(path):
		return

	with open(path) as f:
		contents = f.read()

	if not contents.endswith('\n'):
		contents += '\n'

	complete = pfx + rest
	if not complete.endswith('\n'):
		complete += '\n'

	if complete in contents:
		return

	lines = contents.splitlines()
	for i, l in enumerate(lines):
		if l.startswith(pfx):
			lines[i] = complete.strip()
			break
	else:
		lines.append(complete.strip())

	out = '\n'.join(lines) + '\n'
	run.write(path, out)

def _check_file_is(path, contents, executable=False):
	have = ''
	if os.path.exists(path):
		with open(path) as f:
			have = f.read()

	had = contents == have
	if not had:
		run.write(path, contents)

	if executable and not had:
		run.check('sudo', 'chmod', '+x', path)

	return had

def _reconfig_pkgs(release, arch, cfg, c):
	have = set(c.get('pkgs', []))
	want = envs.packages(release, arch, cfg=cfg) | set(DEFAULT_PKGS)

	if have != want:
		rm = have - want
		if rm:
			_run_in_chroot(release, arch,
				'apt-get',
				'remove', *rm)

		inst = want - have
		if inst:
			_run_in_chroot(release, arch,
				'apt-get',
				'-y', '--force-yes',
				'install', *inst)

	c['pkgs'] = sorted(want)

def _check_upgrade(cfg, lastmod, release, arch):
	now = time.time()
	age = now - lastmod

	if age < cfg.refresh_after:
		return False

	log.info('%s-%s out of date; upgrading...', release, arch)
	_upgrade(release, arch)

	return True

def _upgrade(release, arch):
	run.check(
		'sudo',
		'sbuild-update',
		'-udcar',
		_schroot_name(release, arch))

def _make_sbuildrc(cfg, path):
	require = None
	curr = os.getenv('SBUILD_CONFIG')
	if curr:
		curr = os.path.abspath(curr)
		if os.path.exists(curr):
			require = curr

	with open(path, 'w') as f:
		if require:
			f.write('require "{}";\n'.format(require))

		if cfg.key:
			f.write('$key_id = "{}";\n'.format(cfg.key))

		f.write(util.strip_lines('''
			$run_lintian = {LINTIAN};
			$lintian_opts = {LINTIAN_OPTS};

			$apt_update = {APT_UPGRADE};
			$apt_distupgrade = {APT_UPGRADE};

			1;
			'''.format(
				LINTIAN=int(cfg.lintian),
				LINTIAN_OPTS=cfg.lintian_args,
				APT_UPGRADE=int(cfg.apt_upgrade),
			)))

def _delete(release, arch):
	name = _schroot_name(release, arch)

	run.ignore('sudo', 'schroot',
		'--end-session', '--all-sessions',
		'chroot:{}'.format(name))
	run.ignore('sudo', 'schroot',
		'--end-session', '--all-sessions',
		'source:{}'.format(name))
	run.ignore('sudo', 'rm', '-rf', _dir(release, arch))
	run.ignore('sudo', 'rm', '-rf', _cfg_file(release, arch))

def delete(env):
	release, arch = envs.split(env)
	_delete(release, arch)

def installed():
	envs = set()

	scs = run.get('schroot', '-l', '--all-source-chroots').splitlines()
	for sc in filter(lambda s: SUFFIX in s, scs):
		envs.add(sc
			.replace('source:', '')
			.replace(SUFFIX, ''))

	return sorted(envs)

def match(*specs, only_installed=False):
	"""
	Find all sbuild environments matching the given specs.
	"""

	inst = set(installed())
	ds = envs.match(*specs, installed=inst)

	if only_installed:
		ds = ds.intersection(inst)

	return sorted(ds)

def ensure(cfg, *envss):
	"""
	Ensure that an sbuild environment for the given release-arch environments
	exist.
	"""

	inst = installed()
	for env in envss:
		release, arch = envs.split(env)

		if env in inst:
			_reconfig(cfg, release, arch)
		else:
			_create(cfg, release, arch)

def build(pkg, env, remotes=[]):
	"""
	Build the given package for in the given environment. Return if the
	package was uploaded anywhere.
	"""

	cfg = pkg.cfg

	if not cfg.dry_run:
		ensure(cfg, env)

	release, arch = envs.split(env)

	with util.tmpdir() as tmpdir:
		sbuildrc = os.path.join(tmpdir, '.sbuildrc')
		dsc = pkg.gen_src(tmpdir)
		_make_sbuildrc(cfg, sbuildrc)

		if cfg.dry_run:
			return

		args = []
		if cfg.user:
			args += ['sudo']

			if cfg.user_keep_env:
				args += ['-E']

			args += ['-u', cfg.user]

		args += [
			'sbuild',
			'--arch-all',
			'-j', str(cfg.jobs),
			'--dist', release,
			'--chroot', _schroot_name(release, arch),
		]

		if cfg.include_src:
			args += ['--source', '--force-orig-source']

		args.append(dsc)

		run.check(
			*args,
			cwd=tmpdir,
			env={
				'SBUILD_CONFIG': sbuildrc,
			})

		chgs = glob.glob(os.path.join(tmpdir, '*.changes'))[0]
		for r in remotes:
			dput.put(r, chgs)

class SbuildException(Exception):
	pass

_preconfigure()
<|endoftext|>


<commit_before>class GameHangman:
    def __init__(self, word, total_tries = 10):
        self.word = word.lower ()
        self.right_letters = []
        self.wrong_letters = []
        self.total_tries = total_tries
        self.tries = 0

    @property
    def tries_left (self):
        return self.total_tries - self.tries

    @property
    def won (self):
        return len (self.right_letters) == len (self.word)

    @property
    def lost (self):
        return self.tries_left < 1

    def guess (self, letter):
        letter = letter.lower ()

        if letter in self.word:
            self.right_letters.append (letter)
            return True
        else:
            self.wrong_letters.append (letter)
            return False

        self.tries += 1

    def get_beautified_string (self):
        return " ".join [letter if letter in self.right_letters else "_" for letter in self.word]
<commit_msg>missing parenthesis<commit_after>class GameHangman:
    def __init__(self, word, total_tries = 10):
        self.word = word.lower ()
        self.right_letters = []
        self.wrong_letters = []
        self.total_tries = total_tries
        self.tries = 0

    @property
    def tries_left (self):
        return self.total_tries - self.tries

    @property
    def won (self):
        return len (self.right_letters) == len (self.word)

    @property
    def lost (self):
        return self.tries_left < 1

    def guess (self, letter):
        letter = letter.lower ()

        if letter in self.word:
            self.right_letters.append (letter)
            return True
        else:
            self.wrong_letters.append (letter)
            return False

        self.tries += 1

    def get_beautified_string (self):
        return " ".join ([letter if letter in self.right_letters else "_" for letter in self.word])
<|endoftext|>


<commit_before># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Tests for the control service REST API.
"""

from os import environ
import socket

from uuid import uuid4
from json import dumps, loads

from twisted.internet.defer import succeed
from twisted.trial.unittest import TestCase
from unittest import SkipTest
from treq import get, post, content, delete, json_content
from pyrsistent import PRecord, field, CheckedPVector

from ..testtools import loop_until
from ..node.agents.test.test_blockdevice import REALISTIC_BLOCKDEVICE_SIZE
from ..control.httpapi import REST_API_PORT

class Node(PRecord):
    """
    A record of a cluster node.

    :ivar bytes address: The IPv4 address of the node.
    """
    address = field(type=bytes)

class _NodeList(CheckedPVector):
    """
    A list of nodes.

    See https://github.com/tobgu/pyrsistent/issues/26 for more succinct
    idiom combining this with ``field()``.
    """
    __type__ = Node

class Cluster(PRecord):
    """
    A record of the control service and the nodes in a cluster for acceptance
    testing.

    :param Node control_node: The node running the ``flocker-control``
        service.
    :param list nodes: The ``Node`` s in this cluster.
    """
    control_node = field(type=Node)
    nodes = field(type=_NodeList)

    @property
    def base_url(self):
        """
        :returns: The base url for API requests to this cluster's control
            service.
        """
        return b"http://{}:{}/v1".format(
            self.control_node.address, REST_API_PORT
        )

    def datasets_state(self):
        """
        Return the actual dataset state of the cluster.

        :return: ``Deferred`` firing with a list of dataset dictionaries,
            the state of the cluster.
        """
        request = get(self.base_url + b"/state/datasets", persistent=False)
        request.addCallback(json_content)
        return request

    def wait_for_dataset(self, dataset_properties):
        """
        Poll the dataset state API until the supplied dataset exists.

        :param dict dataset_properties: The attributes of the dataset that
            we're waiting for.
        :returns: A ``Deferred`` which fires with a 2-tuple of ``Cluster`` and
            API response when a dataset with the supplied properties appears in
            the cluster.
        """
        def created():
            """
            Check the dataset state list for the expected dataset.
            """
            request = self.datasets_state()

            def got_body(body):
                # State listing doesn't have metadata or deleted, but does
                # have unpredictable path.
                expected_dataset = dataset_properties.copy()
                del expected_dataset[u"metadata"]
                del expected_dataset[u"deleted"]
                for dataset in body:
                    dataset.pop("path")
                return expected_dataset in body
            request.addCallback(got_body)
            return request

        waiting = loop_until(created)
        waiting.addCallback(lambda ignored: (self, dataset_properties))
        return waiting

    def create_dataset(self, dataset_properties):
        """
        Create a dataset with the supplied ``dataset_properties``.

        :param dict dataset_properties: The properties of the dataset to
            create.
        :returns: A ``Deferred`` which fires with a 2-tuple of ``Cluster`` and
            API response when a dataset with the supplied properties has been
            persisted to the cluster configuration.
        """
        request = post(
            self.base_url + b"/configuration/datasets",
            data=dumps(dataset_properties),
            headers={b"content-type": b"application/json"},
            persistent=False
        )

        request.addCallback(content)
        request.addCallback(loads)
        # Return cluster and API response
        request.addCallback(lambda response: (self, response))
        return request

    def update_dataset(self, dataset_id, dataset_properties):
        """
        Update a dataset with the supplied ``dataset_properties``.

        :param unicode dataset_id: The uuid of the dataset to be modified.
        :param dict dataset_properties: The properties of the dataset to
            create.
        :returns: A 2-tuple of (cluster, api_response)
        """
        request = post(
            self.base_url + b"/configuration/datasets/%s" % (
                dataset_id.encode('ascii'),
            ),
            data=dumps(dataset_properties),
            headers={b"content-type": b"application/json"},
            persistent=False
        )

        request.addCallback(content)
        request.addCallback(loads)
        # Return cluster and API response
        request.addCallback(lambda response: (self, response))
        return request

    def delete_dataset(self, dataset_id):
        """
        Delete a dataset.

        :param unicode dataset_id: The uuid of the dataset to be modified.

        :returns: A 2-tuple of (cluster, api_response)
        """
        request = delete(
            self.base_url + b"/configuration/datasets/%s" % (
                dataset_id.encode('ascii'),
            ),
            headers={b"content-type": b"application/json"},
            persistent=False
        )

        request.addCallback(json_content)
        # Return cluster and API response
        request.addCallback(lambda response: (self, response))
        return request

    def create_container(self, properties):
        """
        Create a container with the specified properties.

        :param dict properties: A ``dict`` mapping to the API request fields
            to create a container.

        :returns: A tuple of (cluster, api_response)
        """
        request = post(
            self.base_url + b"/configuration/containers",
            data=dumps(properties),
            headers={b"content-type": b"application/json"},
            persistent=False
        )

        request.addCallback(json_content)
        request.addCallback(lambda response: (self, response))
        return request

def get_test_cluster(test_case, node_count):
    """
    Build a ``Cluster`` instance with at least ``node_count`` nodes.

    :param TestCase test_case: The test case instance on which to register
        cleanup operations.
    :param int node_count: The number of nodes to request in the cluster.
    :returns: A ``Deferred`` which fires with a ``Cluster`` instance.
    """
    control_node = environ.get('FLOCKER_ACCEPTANCE_CONTROL_NODE')

    if control_node is None:
        raise SkipTest(
            "Set acceptance testing control node IP address using the " +
            "FLOCKER_ACCEPTANCE_CONTROL_NODE environment variable.")

    agent_nodes_env_var = environ.get('FLOCKER_ACCEPTANCE_AGENT_NODES')

    if agent_nodes_env_var is None:
        raise SkipTest(
            "Set acceptance testing node IP addresses using the " +
            "FLOCKER_ACCEPTANCE_AGENT_NODES environment variable and a " +
            "colon separated list.")

    agent_nodes = filter(None, agent_nodes_env_var.split(':'))

    if len(agent_nodes) < node_count:
        raise SkipTest("This test requires a minimum of {necessary} nodes, "
                       "{existing} node(s) are set.".format(
                           necessary=node_count, existing=len(agent_nodes)))

    return succeed(Cluster(
        control_node=Node(address=control_node),
        nodes=map(lambda address: Node(address=address), agent_nodes),
    ))

class ContainerAPITests(TestCase):
    """
    Tests for the container API.
    """
    def test_create_container_with_ports(self):
        """
        Create a container including port mappings on a single-node cluster.
        """
        data = {
            u"name": "my_container",
            u"host": None,
            u"image": "clusterhq/flask:latest",
            u"ports": [{u"internal": 80, u"external": 8080}],
            u'restart_policy': {u'name': u'never'}
        }
        waiting_for_cluster = get_test_cluster(test_case=self, node_count=1)

        def create_container(cluster, data):
            data[u"host"] = cluster.nodes[0].address
            return cluster.create_container(data)

        d = waiting_for_cluster.addCallback(create_container, data)

        def check_result(result):
            response = result[1]

            def can_connect():
                s = socket.socket()
                conn = s.connect_ex((data[u"host"], 8080))
                return False if conn else True

            dl = loop_until(can_connect)
            self.assertEqual(response, data)
            return dl

        d.addCallback(check_result)
        return d

    def test_create_container_with_environment(self):
        """
        Create a container including environment variables on a single-node
        cluster.
        """
        data = {
            u"name": "my_env_container",
            u"host": None,
            u"image": "clusterhq/flaskenv:latest",
            u"ports": [{u"internal": 8080, u"external": 8081}],
            u"environment": {u"ACCEPTANCE_ENV_LABEL": 'acceptance test ok'},
            u'restart_policy': {u'name': u'never'},
        }
        waiting_for_cluster = get_test_cluster(test_case=self, node_count=1)

        def create_container(cluster, data):
            data[u"host"] = cluster.nodes[0].address
            return cluster.create_container(data)

        d = waiting_for_cluster.addCallback(create_container, data)

        def check_result((cluster, response)):
            self.assertEqual(response, data)

        def verify_socket(host, port):
            def can_connect():
                s = socket.socket()
                conn = s.connect_ex((host, port))
                return False if conn else True

            dl = loop_until(can_connect)
            return dl

        def query_environment(host, port):
            """
            The running container, clusterhq/flaskenv, is a simple Flask app
            that returns a JSON dump of the container's environment, so we
            make an HTTP request and parse the response.
            """
            req = get(
                "http://{host}:{port}".format(host=host, port=port),
                persistent=False
            ).addCallback(json_content)
            return req

        d.addCallback(check_result)
        d.addCallback(lambda _: verify_socket(data[u"host"], 8081))
        d.addCallback(lambda _: query_environment(data[u"host"], 8081))
        d.addCallback(
            lambda response:
                self.assertDictContainsSubset(data[u"environment"], response)
        )
        return d

class DatasetAPITests(TestCase):
    """
    Tests for the dataset API.
    """
    def _create_test(self):
        """
        Create a dataset on a single-node cluster.

        :return: ``Deferred`` firing with a tuple of (``Cluster``
            instance, dataset dictionary) once the dataset is present in
            actual cluster state.
        """
        # Create a 1 node cluster
        waiting_for_cluster = get_test_cluster(test_case=self, node_count=1)

        # Configure a dataset on node1
        def configure_dataset(cluster):
            """
            Send a dataset creation request on node1.
            """
            requested_dataset = {
                u"primary": cluster.nodes[0].address,
                u"dataset_id": unicode(uuid4()),
                u"maximum_size": REALISTIC_BLOCKDEVICE_SIZE,
                u"metadata": {u"name": u"my_volume"},
            }

            return cluster.create_dataset(requested_dataset)
        configuring_dataset = waiting_for_cluster.addCallback(
            configure_dataset
        )

        # Wait for the dataset to be created
        waiting_for_create = configuring_dataset.addCallback(
            lambda (cluster, dataset): cluster.wait_for_dataset(dataset)
        )

        return waiting_for_create

    def test_dataset_creation(self):
        """
        A dataset can be created on a specific node.
        """
        return self._create_test()

    def test_dataset_move(self):
        """
        A dataset can be moved from one node to another.
        """
        # Create a 2 node cluster
        waiting_for_cluster = get_test_cluster(test_case=self, node_count=2)

        # Configure a dataset on node1
        def configure_dataset(cluster):
            """
            Send a dataset creation request on node1.
            """
            requested_dataset = {
                u"primary": cluster.nodes[0].address,
                u"dataset_id": unicode(uuid4()),
                u"metadata": {u"name": u"my_volume"}
            }

            return cluster.create_dataset(requested_dataset)
        configuring_dataset = waiting_for_cluster.addCallback(
            configure_dataset
        )

        # Wait for the dataset to be created
        waiting_for_create = configuring_dataset.addCallback(
            lambda (cluster, dataset): cluster.wait_for_dataset(dataset)
        )

        # Once created, request to move the dataset to node2
        def move_dataset((cluster, dataset)):
            moved_dataset = {
                u'primary': cluster.nodes[1].address
            }
            return cluster.update_dataset(dataset['dataset_id'], moved_dataset)
        dataset_moving = waiting_for_create.addCallback(move_dataset)

        # Wait for the dataset to be moved
        waiting_for_move = dataset_moving.addCallback(
            lambda (cluster, dataset): cluster.wait_for_dataset(dataset)
        )

        return waiting_for_move

    def test_dataset_deletion(self):
        """
        A dataset can be deleted, resulting in its removal from the node.
        """
        created = self._create_test()

        def delete_dataset(result):
            cluster, dataset = result
            deleted = cluster.delete_dataset(dataset["dataset_id"])

            def not_exists():
                request = cluster.datasets_state()
                request.addCallback(
                    lambda actual_datasets: dataset["dataset_id"] not in
                    (d["dataset_id"] for d in actual_datasets))
                return request
            deleted.addCallback(lambda _: loop_until(not_exists))
            return deleted
        created.addCallback(delete_dataset)
        return created
<commit_msg>An idea for an acceptance test that bypasses the not-yet-implemented resize API<commit_after># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Tests for the control service REST API.
"""

from os import environ
import socket

from uuid import uuid4
from json import dumps, loads

from twisted.internet.defer import succeed
from twisted.trial.unittest import TestCase
from unittest import SkipTest
from treq import get, post, content, delete, json_content
from pyrsistent import PRecord, field, CheckedPVector

from ..testtools import loop_until
from ..node.agents.test.test_blockdevice import REALISTIC_BLOCKDEVICE_SIZE
from ..control.httpapi import REST_API_PORT

class Node(PRecord):
    """
    A record of a cluster node.

    :ivar bytes address: The IPv4 address of the node.
    """
    address = field(type=bytes)

class _NodeList(CheckedPVector):
    """
    A list of nodes.

    See https://github.com/tobgu/pyrsistent/issues/26 for more succinct
    idiom combining this with ``field()``.
    """
    __type__ = Node

class Cluster(PRecord):
    """
    A record of the control service and the nodes in a cluster for acceptance
    testing.

    :param Node control_node: The node running the ``flocker-control``
        service.
    :param list nodes: The ``Node`` s in this cluster.
    """
    control_node = field(type=Node)
    nodes = field(type=_NodeList)

    @property
    def base_url(self):
        """
        :returns: The base url for API requests to this cluster's control
            service.
        """
        return b"http://{}:{}/v1".format(
            self.control_node.address, REST_API_PORT
        )

    def datasets_state(self):
        """
        Return the actual dataset state of the cluster.

        :return: ``Deferred`` firing with a list of dataset dictionaries,
            the state of the cluster.
        """
        request = get(self.base_url + b"/state/datasets", persistent=False)
        request.addCallback(json_content)
        return request

    def wait_for_dataset(self, dataset_properties):
        """
        Poll the dataset state API until the supplied dataset exists.

        :param dict dataset_properties: The attributes of the dataset that
            we're waiting for.
        :returns: A ``Deferred`` which fires with a 2-tuple of ``Cluster`` and
            API response when a dataset with the supplied properties appears in
            the cluster.
        """
        def created():
            """
            Check the dataset state list for the expected dataset.
            """
            request = self.datasets_state()

            def got_body(body):
                # State listing doesn't have metadata or deleted, but does
                # have unpredictable path.
                expected_dataset = dataset_properties.copy()
                del expected_dataset[u"metadata"]
                del expected_dataset[u"deleted"]
                for dataset in body:
                    dataset.pop("path")
                return expected_dataset in body
            request.addCallback(got_body)
            return request

        waiting = loop_until(created)
        waiting.addCallback(lambda ignored: (self, dataset_properties))
        return waiting

    def create_dataset(self, dataset_properties):
        """
        Create a dataset with the supplied ``dataset_properties``.

        :param dict dataset_properties: The properties of the dataset to
            create.
        :returns: A ``Deferred`` which fires with a 2-tuple of ``Cluster`` and
            API response when a dataset with the supplied properties has been
            persisted to the cluster configuration.
        """
        request = post(
            self.base_url + b"/configuration/datasets",
            data=dumps(dataset_properties),
            headers={b"content-type": b"application/json"},
            persistent=False
        )

        request.addCallback(content)
        request.addCallback(loads)
        # Return cluster and API response
        request.addCallback(lambda response: (self, response))
        return request

    def update_dataset(self, dataset_id, dataset_properties):
        """
        Update a dataset with the supplied ``dataset_properties``.

        :param unicode dataset_id: The uuid of the dataset to be modified.
        :param dict dataset_properties: The properties of the dataset to
            create.
        :returns: A 2-tuple of (cluster, api_response)
        """
        request = post(
            self.base_url + b"/configuration/datasets/%s" % (
                dataset_id.encode('ascii'),
            ),
            data=dumps(dataset_properties),
            headers={b"content-type": b"application/json"},
            persistent=False
        )

        request.addCallback(content)
        request.addCallback(loads)
        # Return cluster and API response
        request.addCallback(lambda response: (self, response))
        return request

    def delete_dataset(self, dataset_id):
        """
        Delete a dataset.

        :param unicode dataset_id: The uuid of the dataset to be modified.

        :returns: A 2-tuple of (cluster, api_response)
        """
        request = delete(
            self.base_url + b"/configuration/datasets/%s" % (
                dataset_id.encode('ascii'),
            ),
            headers={b"content-type": b"application/json"},
            persistent=False
        )

        request.addCallback(json_content)
        # Return cluster and API response
        request.addCallback(lambda response: (self, response))
        return request

    def create_container(self, properties):
        """
        Create a container with the specified properties.

        :param dict properties: A ``dict`` mapping to the API request fields
            to create a container.

        :returns: A tuple of (cluster, api_response)
        """
        request = post(
            self.base_url + b"/configuration/containers",
            data=dumps(properties),
            headers={b"content-type": b"application/json"},
            persistent=False
        )

        request.addCallback(json_content)
        request.addCallback(lambda response: (self, response))
        return request

def get_test_cluster(test_case, node_count):
    """
    Build a ``Cluster`` instance with at least ``node_count`` nodes.

    :param TestCase test_case: The test case instance on which to register
        cleanup operations.
    :param int node_count: The number of nodes to request in the cluster.
    :returns: A ``Deferred`` which fires with a ``Cluster`` instance.
    """
    control_node = environ.get('FLOCKER_ACCEPTANCE_CONTROL_NODE')

    if control_node is None:
        raise SkipTest(
            "Set acceptance testing control node IP address using the " +
            "FLOCKER_ACCEPTANCE_CONTROL_NODE environment variable.")

    agent_nodes_env_var = environ.get('FLOCKER_ACCEPTANCE_AGENT_NODES')

    if agent_nodes_env_var is None:
        raise SkipTest(
            "Set acceptance testing node IP addresses using the " +
            "FLOCKER_ACCEPTANCE_AGENT_NODES environment variable and a " +
            "colon separated list.")

    agent_nodes = filter(None, agent_nodes_env_var.split(':'))

    if len(agent_nodes) < node_count:
        raise SkipTest("This test requires a minimum of {necessary} nodes, "
                       "{existing} node(s) are set.".format(
                           necessary=node_count, existing=len(agent_nodes)))

    return succeed(Cluster(
        control_node=Node(address=control_node),
        nodes=map(lambda address: Node(address=address), agent_nodes),
    ))

class ContainerAPITests(TestCase):
    """
    Tests for the container API.
    """
    def test_create_container_with_ports(self):
        """
        Create a container including port mappings on a single-node cluster.
        """
        data = {
            u"name": "my_container",
            u"host": None,
            u"image": "clusterhq/flask:latest",
            u"ports": [{u"internal": 80, u"external": 8080}],
            u'restart_policy': {u'name': u'never'}
        }
        waiting_for_cluster = get_test_cluster(test_case=self, node_count=1)

        def create_container(cluster, data):
            data[u"host"] = cluster.nodes[0].address
            return cluster.create_container(data)

        d = waiting_for_cluster.addCallback(create_container, data)

        def check_result(result):
            response = result[1]

            def can_connect():
                s = socket.socket()
                conn = s.connect_ex((data[u"host"], 8080))
                return False if conn else True

            dl = loop_until(can_connect)
            self.assertEqual(response, data)
            return dl

        d.addCallback(check_result)
        return d

    def test_create_container_with_environment(self):
        """
        Create a container including environment variables on a single-node
        cluster.
        """
        data = {
            u"name": "my_env_container",
            u"host": None,
            u"image": "clusterhq/flaskenv:latest",
            u"ports": [{u"internal": 8080, u"external": 8081}],
            u"environment": {u"ACCEPTANCE_ENV_LABEL": 'acceptance test ok'},
            u'restart_policy': {u'name': u'never'},
        }
        waiting_for_cluster = get_test_cluster(test_case=self, node_count=1)

        def create_container(cluster, data):
            data[u"host"] = cluster.nodes[0].address
            return cluster.create_container(data)

        d = waiting_for_cluster.addCallback(create_container, data)

        def check_result((cluster, response)):
            self.assertEqual(response, data)

        def verify_socket(host, port):
            def can_connect():
                s = socket.socket()
                conn = s.connect_ex((host, port))
                return False if conn else True

            dl = loop_until(can_connect)
            return dl

        def query_environment(host, port):
            """
            The running container, clusterhq/flaskenv, is a simple Flask app
            that returns a JSON dump of the container's environment, so we
            make an HTTP request and parse the response.
            """
            req = get(
                "http://{host}:{port}".format(host=host, port=port),
                persistent=False
            ).addCallback(json_content)
            return req

        d.addCallback(check_result)
        d.addCallback(lambda _: verify_socket(data[u"host"], 8081))
        d.addCallback(lambda _: query_environment(data[u"host"], 8081))
        d.addCallback(
            lambda response:
                self.assertDictContainsSubset(data[u"environment"], response)
        )
        return d

class DatasetAPITests(TestCase):
    """
    Tests for the dataset API.
    """
    def _create_test(self):
        """
        Create a dataset on a single-node cluster.

        :return: ``Deferred`` firing with a tuple of (``Cluster``
            instance, dataset dictionary) once the dataset is present in
            actual cluster state.
        """
        # Create a 1 node cluster
        waiting_for_cluster = get_test_cluster(test_case=self, node_count=1)

        # Configure a dataset on node1
        def configure_dataset(cluster):
            """
            Send a dataset creation request on node1.
            """
            requested_dataset = {
                u"primary": cluster.nodes[0].address,
                u"dataset_id": unicode(uuid4()),
                u"maximum_size": REALISTIC_BLOCKDEVICE_SIZE,
                u"metadata": {u"name": u"my_volume"},
            }

            return cluster.create_dataset(requested_dataset)
        configuring_dataset = waiting_for_cluster.addCallback(
            configure_dataset
        )

        # Wait for the dataset to be created
        waiting_for_create = configuring_dataset.addCallback(
            lambda (cluster, dataset): cluster.wait_for_dataset(dataset)
        )

        return waiting_for_create

    def test_dataset_creation(self):
        """
        A dataset can be created on a specific node.
        """
        return self._create_test()

    def test_dataset_move(self):
        """
        A dataset can be moved from one node to another.
        """
        # Create a 2 node cluster
        waiting_for_cluster = get_test_cluster(test_case=self, node_count=2)

        # Configure a dataset on node1
        def configure_dataset(cluster):
            """
            Send a dataset creation request on node1.
            """
            requested_dataset = {
                u"primary": cluster.nodes[0].address,
                u"dataset_id": unicode(uuid4()),
                u"metadata": {u"name": u"my_volume"}
            }

            return cluster.create_dataset(requested_dataset)
        configuring_dataset = waiting_for_cluster.addCallback(
            configure_dataset
        )

        # Wait for the dataset to be created
        waiting_for_create = configuring_dataset.addCallback(
            lambda (cluster, dataset): cluster.wait_for_dataset(dataset)
        )

        # Once created, request to move the dataset to node2
        def move_dataset((cluster, dataset)):
            moved_dataset = {
                u'primary': cluster.nodes[1].address
            }
            return cluster.update_dataset(dataset['dataset_id'], moved_dataset)
        dataset_moving = waiting_for_create.addCallback(move_dataset)

        # Wait for the dataset to be moved
        waiting_for_move = dataset_moving.addCallback(
            lambda (cluster, dataset): cluster.wait_for_dataset(dataset)
        )

        return waiting_for_move

    def test_dataset_deletion(self):
        """
        A dataset can be deleted, resulting in its removal from the node.
        """
        created = self._create_test()

        def delete_dataset(result):
            cluster, dataset = result
            deleted = cluster.delete_dataset(dataset["dataset_id"])

            def not_exists():
                request = cluster.datasets_state()
                request.addCallback(
                    lambda actual_datasets: dataset["dataset_id"] not in
                    (d["dataset_id"] for d in actual_datasets))
                return request
            deleted.addCallback(lambda _: loop_until(not_exists))
            return deleted
        created.addCallback(delete_dataset)
        return created

    def test_dataset_grow_(self):
        """
        The maximum size of a dataset can be increased.

        A not quite end-to-end test that the size of a dataset can be increased.

        We can't issue the resize request from the API until
        https://clusterhq.atlassian.net/browse/FLOC-1599 is resolved.

        So manipulate the control service configuration file directly and
        restart it....or something.
        """
        # Create a dataset of REALISTIC_BLOCKDEVICE_SIZE
        # Wait for it to be created.
        # Add some data to it somehow.
        # Stop the dataset agent
        # Stop the control service
        # Wipe the configuration file
        # Start the control service
        # Create a dataset with the same dataset_id but with maximum_size=REALISTIC_BLOCKDEVICE_SIZE * 2
        # Wait for the dataset to be resized.
        # Check that the data (written earlier) is still intact and that the dataset state reports the correct new size.
<|endoftext|>