<commit_before>import unittest
import json
from labsuite.engine.protocol import Protocol

class ProtocolTest(unittest.TestCase):

    def setUp(self):
        self.protocol = Protocol()
        self.protocol.add_instrument('p10')

    @property
    def instructions(self):
        return self.protocol._commands

    def test_normalize_address(self):
        self.protocol.add_container('A1', 'microplate.96', label="Output")
        label = self.protocol._normalize_address('Output:A1')
        self.assertEqual(label, ('output', (0, 0)))
        slot = self.protocol._normalize_address('A1:A1')
        self.assertEqual(slot, ((0, 0), (0, 0)))

    def test_transfer(self):
        """ Basic transfer. """
        self.protocol.transfer('A1:A1', 'B1:B1', ul=100)
        expected = [{
            'transfer': {
                'tool': 'p10',
                'volume': 100,
                'start': ((0, 0), (0, 0)),
                'end': ((1, 0), (1, 0)),
                'blowout': True,
                'touchtip': True
            }
        }]
        self.assertEqual(self.instructions, expected)

    def test_transfer_group(self):
        """ Transfer group. """
        expected = [{
            'transfer_group': {
                'tool': 'p10',
                'transfers': [
                    {
                        'volume': 15,
                        'start': ((0, 0), (0, 0)),  # A1:A1
                        'end': ((1, 0), (1, 0)),  # B1:B1
                        'blowout': True,
                        'touchtip': True
                    },
                    {
                        'volume': 1000,
                        'start': ((0, 1), (0, 1)),  # A2:A2
                        'end': ((1, 1), (1, 1)),  # B2:B2
                        'blowout': True,
                        'touchtip': True
                    },
                    {
                        'volume': 12,
                        'start': ((0, 2), (0, 2)),  # A3:A3
                        'end': ((1, 2), (1, 2)),  # B3:B3
                        'blowout': False,
                        'touchtip': True
                    },
                    {
                        'volume': 12,
                        'start': ((0, 3), (0, 3)),  # A4:A4
                        'end': ((1, 3), (1, 3)),  # B4:B4
                        'blowout': True,
                        'touchtip': True
                    },
                    {
                        'volume': 12,
                        'start': ('label', (0, 4)),  # label:A5
                        'end': ((1, 4), (2, 0)),  # B5:C1
                        'blowout': True,
                        'touchtip': True
                    }
                ]
            }
        }]
        self.protocol.add_container('A1', 'microplate.96', label="Label")
        self.protocol.transfer_group(
            ('A1:A1', 'B1:B1', {'ul': 15}),
            ('A2:A2', 'B2:B2', {'ml': 1}),
            ('A3:A3', 'B3:B3', {'blowout': False}),
            ('A4:A4', 'B4:B4'),
            ('Label:A5', 'B5:C1'),
            ul=12
        )
        self.assertEqual(self.instructions, expected)

    def test_distribute(self):
        self.protocol.distribute(
            'A1:A1',
            ('B1:B1', 50),
            ('C1:C1', 5),
            ('D1:D1', 10)
        )
        expected = [{
            'distribute': {
                'tool': 'p10',
                'blowout': True,
                'start': ((0, 0), (0, 0)),
                'transfers': [
                    {
                        'volume': 50,
                        'end': ((1, 0), (1, 0)),  # B1:B1
                    },
                    {
                        'volume': 5,
                        'end': ((2, 0), (2, 0)),  # C1:C1
                    },
                    {
                        'volume': 10,
                        'end': ((3, 0), (3, 0))  # D1:D1
                    }
                ]
            }
        }]
        self.assertEqual(self.instructions, expected)

    def test_consolidate(self):
        self.protocol.consolidate(
            'A1:A1',
            ('B1:B1', 50),
            ('C1:C1', 5),
            ('D1:D1', 10)
        )
        expected = [{
            'consolidate': {
                'tool': 'p10',
                'blowout': True,
                'end': ((0, 0), (0, 0)),
                'transfers': [
                    {
                        'volume': 50,
                        'start': ((1, 0), (1, 0)),  # B1:B1
                    },
                    {
                        'volume': 5,
                        'start': ((2, 0), (2, 0)),  # C1:C1
                    },
                    {
                        'volume': 10,
                        'start': ((3, 0), (3, 0))  # D1:D1
                    }
                ]
            }
        }]
        self.assertEqual(self.instructions, expected)

    def test_mix(self):
        self.protocol.mix(
            'A1:A1',
            volume=50,
            repetitions=10
        )
        expected = [{'mix': {
            'tool': 'p10',
            'start': ((0, 0), (0, 0)),  # A1:A1
            'blowout': True,
            'volume': 50,
            'reps': 10
        }}]
        self.assertEqual(self.instructions, expected)

    def test_context(self):
        self.protocol.add_container('A1', 'microplate.96')
        self.protocol.calibrate('A1', x=1, y=2, z=3)
        self.protocol.transfer('A1:A1', 'A1:A2', ul=100)
        self.protocol.run_next()
        vol = self.protocol._context.get_volume('A1:A2')
        self.assertEqual(vol, 100)
<commit_msg>Protocol: Test multiple transfer commands in sequence.<commit_after>import unittest
import json
from labsuite.engine.protocol import Protocol

class ProtocolTest(unittest.TestCase):

    def setUp(self):
        self.protocol = Protocol()
        self.protocol.add_instrument('p10')

    @property
    def instructions(self):
        return self.protocol._commands

    def test_normalize_address(self):
        self.protocol.add_container('A1', 'microplate.96', label="Output")
        label = self.protocol._normalize_address('Output:A1')
        self.assertEqual(label, ('output', (0, 0)))
        slot = self.protocol._normalize_address('A1:A1')
        self.assertEqual(slot, ((0, 0), (0, 0)))

    def test_transfer(self):
        """ Basic transfer. """
        self.protocol.transfer('A1:A1', 'B1:B1', ul=100)
        expected = [{
            'transfer': {
                'tool': 'p10',
                'volume': 100,
                'start': ((0, 0), (0, 0)),
                'end': ((1, 0), (1, 0)),
                'blowout': True,
                'touchtip': True
            }
        }]
        self.assertEqual(self.instructions, expected)

    def test_transfer_group(self):
        """ Transfer group. """
        expected = [{
            'transfer_group': {
                'tool': 'p10',
                'transfers': [
                    {
                        'volume': 15,
                        'start': ((0, 0), (0, 0)),  # A1:A1
                        'end': ((1, 0), (1, 0)),  # B1:B1
                        'blowout': True,
                        'touchtip': True
                    },
                    {
                        'volume': 1000,
                        'start': ((0, 1), (0, 1)),  # A2:A2
                        'end': ((1, 1), (1, 1)),  # B2:B2
                        'blowout': True,
                        'touchtip': True
                    },
                    {
                        'volume': 12,
                        'start': ((0, 2), (0, 2)),  # A3:A3
                        'end': ((1, 2), (1, 2)),  # B3:B3
                        'blowout': False,
                        'touchtip': True
                    },
                    {
                        'volume': 12,
                        'start': ((0, 3), (0, 3)),  # A4:A4
                        'end': ((1, 3), (1, 3)),  # B4:B4
                        'blowout': True,
                        'touchtip': True
                    },
                    {
                        'volume': 12,
                        'start': ('label', (0, 4)),  # label:A5
                        'end': ((1, 4), (2, 0)),  # B5:C1
                        'blowout': True,
                        'touchtip': True
                    }
                ]
            }
        }]
        self.protocol.add_container('A1', 'microplate.96', label="Label")
        self.protocol.transfer_group(
            ('A1:A1', 'B1:B1', {'ul': 15}),
            ('A2:A2', 'B2:B2', {'ml': 1}),
            ('A3:A3', 'B3:B3', {'blowout': False}),
            ('A4:A4', 'B4:B4'),
            ('Label:A5', 'B5:C1'),
            ul=12
        )
        self.assertEqual(self.instructions, expected)

    def test_distribute(self):
        self.protocol.distribute(
            'A1:A1',
            ('B1:B1', 50),
            ('C1:C1', 5),
            ('D1:D1', 10)
        )
        expected = [{
            'distribute': {
                'tool': 'p10',
                'blowout': True,
                'start': ((0, 0), (0, 0)),
                'transfers': [
                    {
                        'volume': 50,
                        'end': ((1, 0), (1, 0)),  # B1:B1
                    },
                    {
                        'volume': 5,
                        'end': ((2, 0), (2, 0)),  # C1:C1
                    },
                    {
                        'volume': 10,
                        'end': ((3, 0), (3, 0))  # D1:D1
                    }
                ]
            }
        }]
        self.assertEqual(self.instructions, expected)

    def test_consolidate(self):
        self.protocol.consolidate(
            'A1:A1',
            ('B1:B1', 50),
            ('C1:C1', 5),
            ('D1:D1', 10)
        )
        expected = [{
            'consolidate': {
                'tool': 'p10',
                'blowout': True,
                'end': ((0, 0), (0, 0)),
                'transfers': [
                    {
                        'volume': 50,
                        'start': ((1, 0), (1, 0)),  # B1:B1
                    },
                    {
                        'volume': 5,
                        'start': ((2, 0), (2, 0)),  # C1:C1
                    },
                    {
                        'volume': 10,
                        'start': ((3, 0), (3, 0))  # D1:D1
                    }
                ]
            }
        }]
        self.assertEqual(self.instructions, expected)

    def test_mix(self):
        self.protocol.mix(
            'A1:A1',
            volume=50,
            repetitions=10
        )
        expected = [{'mix': {
            'tool': 'p10',
            'start': ((0, 0), (0, 0)),  # A1:A1
            'blowout': True,
            'volume': 50,
            'reps': 10
        }}]
        self.assertEqual(self.instructions, expected)

    def test_context(self):
        self.protocol.add_container('A1', 'microplate.96')
        self.protocol.calibrate('A1', x=1, y=2, z=3)
        self.protocol.transfer('A1:A1', 'A1:A2', ul=100)
        self.protocol.transfer('A1:A2', 'A1:A3', ul=80)
        vol1 = self.protocol._context.get_volume('A1:A2')
        self.assertEqual(vol1, 0)
        self.protocol.run_next()
        vol2 = self.protocol._context.get_volume('A1:A2')
        self.assertEqual(vol2, 100)
        self.protocol.run_next()
        vol2 = self.protocol._context.get_volume('A1:A3')
        self.assertEqual(vol2, 80)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import datetime
import plistlib
from decimal import Decimal

from biplist import is_stream_binary_plist, readPlist
from django.template.defaultfilters import escape
from django.utils import dateparse, six
from django.utils.encoding import force_bytes, force_text
from django.utils.six.moves.html_parser import HTMLParser

DATETIME_TYPES = (datetime.datetime, datetime.date, datetime.time)

if hasattr(plistlib, 'dumps') and hasattr(plistlib, 'loads'):

    def _clean(obj):
        if isinstance(obj, datetime.date):
            return datetime.datetime.combine(obj, datetime.time.min)
        elif isinstance(obj, dict):
            return {k: _clean(v) for k, v in obj.items()}
        elif isinstance(obj, (list, tuple)):
            return map(_clean, obj)
        elif obj is None:
            return b''
        elif isinstance(obj, six.string_types):
            return force_bytes(obj)
        else:
            return obj

    def read(stream):
        return plistlib.loads(force_bytes(stream))

    def write(data):
        return plistlib.dumps(_clean(data))

else:

    class RFPlistParser(plistlib.PlistParser):
        '''
        A Plist parser that correctly handles timezones on dates
        '''

        def end_date(self):
            value = self.getData()
            if dateparse.parse_datetime(value) is not None:
                self.addObject(dateparse.parse_datetime(value))
            elif dateparse.parse_date(value) is not None:
                self.addObject(dateparse.parse_date(value))
            elif dateparse.parse_time(value) is not None:
                self.addObject(dateparse.parse_time(value))
            else:
                raise ValueError('"{0}" is not a valid datetime'.format(value))

        def end_real(self):
            self.addObject(Decimal(self.getData()))

        def end_data(self):
            h = HTMLParser()
            value = h.unescape(force_text(
                plistlib.Data.fromBase64(self.getData()).data
            ))
            self.addObject(value)

        def end_string(self):
            h = HTMLParser()
            data = h.unescape(force_text(self.getData()))
            if data == '__PyNone__':
                self.addObject(None)
            else:
                self.addObject(data)

    class RFPlistWriter(plistlib.PlistWriter):
        '''
        A Plist writer that supports ``None`` type objects and has better
        datetime support
        '''

        def writeValue(self, value):
            if isinstance(value, DATETIME_TYPES):
                self.simpleElement('date', value.isoformat())
            elif isinstance(value, Decimal):
                self.simpleElement(
                    'real',
                    force_text(value) if six.PY3 else force_bytes(value)
                )
            elif isinstance(value, (six.text_type, six.binary_type)):
                try:
                    data = escape(value)
                    if six.PY3:
                        data = force_text(data)
                    else:
                        data = force_bytes(data)
                    self.simpleElement('string', data)
                except ValueError as e:
                    if 'contains control characters' in e.args[0]:
                        data = plistlib.Data(force_bytes(
                            escape(value)
                        )).asBase64()
                        if six.PY3:
                            data = force_text(data)
                        self.simpleElement('data', data)
                    else:
                        raise
            elif value is None:
                self.simpleElement('string', '__PyNone__')
            elif value:
                plistlib.PlistWriter.writeValue(self, value)

    def read(stream):
        if not isinstance(stream, six.BytesIO):
            stream = six.BytesIO(stream)
        if is_stream_binary_plist(stream):
            stream.seek(0)
            return readPlist(stream)
        stream.seek(0)
        return RFPlistParser().parse(stream)

    def write(data):
        writer = RFPlistWriter(six.BytesIO())
        writer.writeValue(data)
        return writer.file.getvalue()
<commit_msg>Fix decimal and iterable handling<commit_after># -*- coding: utf-8 -*-

import datetime
import plistlib
from decimal import Decimal

from biplist import is_stream_binary_plist, readPlist
from django.template.defaultfilters import escape
from django.utils import dateparse, six
from django.utils.encoding import force_bytes, force_text
from django.utils.six.moves.html_parser import HTMLParser

DATETIME_TYPES = (datetime.datetime, datetime.date, datetime.time)

if hasattr(plistlib, 'dumps') and hasattr(plistlib, 'loads'):

    def _clean(obj):
        if isinstance(obj, datetime.date):
            return datetime.datetime.combine(obj, datetime.time.min)
        elif isinstance(obj, dict):
            return {k: _clean(v) for k, v in obj.items()}
        elif isinstance(obj, (list, tuple)):
            return [_clean(o) for o in obj]
        elif isinstance(obj, Decimal):
            return float(obj)
        elif obj is None:
            return b''
        elif isinstance(obj, six.string_types):
            return force_bytes(obj)
        else:
            return obj

    def read(stream):
        return plistlib.loads(force_bytes(stream))

    def write(data):
        return plistlib.dumps(_clean(data))

else:

    class RFPlistParser(plistlib.PlistParser):
        '''
        A Plist parser that correctly handles timezones on dates
        '''

        def end_date(self):
            value = self.getData()
            if dateparse.parse_datetime(value) is not None:
                self.addObject(dateparse.parse_datetime(value))
            elif dateparse.parse_date(value) is not None:
                self.addObject(dateparse.parse_date(value))
            elif dateparse.parse_time(value) is not None:
                self.addObject(dateparse.parse_time(value))
            else:
                raise ValueError('"{0}" is not a valid datetime'.format(value))

        def end_real(self):
            self.addObject(Decimal(self.getData()))

        def end_data(self):
            h = HTMLParser()
            value = h.unescape(force_text(
                plistlib.Data.fromBase64(self.getData()).data
            ))
            self.addObject(value)

        def end_string(self):
            h = HTMLParser()
            data = h.unescape(force_text(self.getData()))
            if data == '__PyNone__':
                self.addObject(None)
            else:
                self.addObject(data)

    class RFPlistWriter(plistlib.PlistWriter):
        '''
        A Plist writer that supports ``None`` type objects and has better
        datetime support
        '''

        def writeValue(self, value):
            if isinstance(value, DATETIME_TYPES):
                self.simpleElement('date', value.isoformat())
            elif isinstance(value, Decimal):
                self.simpleElement(
                    'real',
                    force_text(value) if six.PY3 else force_bytes(value)
                )
            elif isinstance(value, (six.text_type, six.binary_type)):
                try:
                    data = escape(value)
                    if six.PY3:
                        data = force_text(data)
                    else:
                        data = force_bytes(data)
                    self.simpleElement('string', data)
                except ValueError as e:
                    if 'contains control characters' in e.args[0]:
                        data = plistlib.Data(force_bytes(
                            escape(value)
                        )).asBase64()
                        if six.PY3:
                            data = force_text(data)
                        self.simpleElement('data', data)
                    else:
                        raise
            elif value is None:
                self.simpleElement('string', '__PyNone__')
            elif value:
                plistlib.PlistWriter.writeValue(self, value)

    def read(stream):
        if not isinstance(stream, six.BytesIO):
            stream = six.BytesIO(stream)
        if is_stream_binary_plist(stream):
            stream.seek(0)
            return readPlist(stream)
        stream.seek(0)
        return RFPlistParser().parse(stream)

    def write(data):
        writer = RFPlistWriter(six.BytesIO())
        writer.writeValue(data)
        return writer.file.getvalue()
<|endoftext|>


<commit_before>##############################################################################
#
# xpdView.Tif_File_Finder      SULI and HSRP
#                              (c) 2016 Brookhaven Science Associates,
#                              Brookhaven National Laboratory.
#                              All rights reserved.
#
# File coded by:               Caleb Duff
#                              Joseph Kaming-Thanassi
#
# See AUTHORS.txt for a list of people who contributed.
# See LICENSE.txt for license information.
#
##############################################################################

"""
This class is what gets the .tif files from the directory when entered and returns the numpy arrays used
in XPD_view. This class is designed to ensure that files are ordered according to time signature, and that
all dark tifs and raw tifs are ignored as they are being read in.
"""

from tifffile import imread
import os

class TifFileFinder(object):
    """
    This class finds tif and tiff files for the user and reads them in to be used later in the GUI

    Attributes
    ----------
    _directory_name : str
        The name of directory that contains tif files that user wants to see
    dir_fil : list of strings
        list of all files in directory
    file_list : list of strings
        list of all tif files in directory
    pic_list : list of 2D numpy arrays
        list of all data read in from the tif files
    """

    def __init__(self):
        """
        This initializes the TifFileFinder class

        Returns
        -------
        None
        """
        self._directory_name = None
        self.dir_fil = []
        self.file_list = []
        self.pic_list = []

    def get_file_list(self):
        """
        This class gets the initial list of tif files

        Parameters
        ----------
        self

        Returns
        -------
        None

        """
        if self._directory_name is None:
            raise NotADirectoryError
        if self._directory_name[-1] != '/' or '\\':
            self._directory_name += '/'
        self.dir_fil = os.listdir(self._directory_name)
        self.dir_fil.sort(key=lambda x: os.path.getmtime(self._directory_name + x))
        self.file_list = [file for file in self.dir_fil if file.endswith('.tif') and not (file.endswith('.dark.tif') or
                                                                                file.endswith('.raw.tif'))]
        self.get_image_arrays()

    def get_image_arrays(self):
        """
        This method reads in the tif files into 2D numpy arrays and appends them to the class's pic_list

        Parameters
        ----------
        self

        Returns
        -------
        None

        """
        for i in self.file_list:
            self.pic_list.append(imread(self._directory_name + i))

    def get_new_files(self):
        """
        This method finds new tif files in the directory for the user

        Parameters
        ----------
        self

        Returns
        -------
        all returns from the get_new_images method

        """
        self.dir_fil = os.listdir(self._directory_name)
        self.dir_fil.sort(key=lambda x: os.path.getmtime(self._directory_name + x))
        new_file_list = [file for file in self.dir_fil if file.endswith('.tif') and not (file.endswith('.dark.tif') or
                                                                                file.endswith('.raw.tif'))]
        need_read_files = []
        for i in new_file_list:
            add = True
            for j in self.file_list:
                if i == j:
                    add = False
                    break
            if add:
                self.file_list.append(i)
                need_read_files.append(i)
        return self.get_new_images(need_read_files)

    def get_new_images(self, temp_file_list):
        """
        This method reads in the newly found files if there are any and returns them

        Parameters
        ----------
        self
        temp_file_list : list of strings
            temporary list containing all of the new names

        Returns
        -------
        temp_file_list: list of strings
            see above
        new_pic: list of 2D numpy arrays
            temporary list of all new read in data

        """
        new_pics = []
        if temp_file_list is not None:
            for i in temp_file_list:
                self.pic_list.append(imread(self._directory_name + i))
                new_pics.append(imread(self._directory_name + i))
        return temp_file_list, new_pics
<commit_msg>Fixing issue #19<commit_after>##############################################################################
#
# xpdView.Tif_File_Finder      SULI and HSRP
#                              (c) 2016 Brookhaven Science Associates,
#                              Brookhaven National Laboratory.
#                              All rights reserved.
#
# File coded by:               Caleb Duff
#                              Joseph Kaming-Thanassi
#
# See AUTHORS.txt for a list of people who contributed.
# See LICENSE.txt for license information.
#
##############################################################################

"""
This class is what gets the .tif files from the directory when entered and returns the numpy arrays used
in XPD_view. This class is designed to ensure that files are ordered according to time signature, and that
all dark tifs and raw tifs are ignored as they are being read in.
"""

from tifffile import imread
import os

class TifFileFinder(object):
    """
    This class finds tif and tiff files for the user and reads them in to be used later in the GUI

    Attributes
    ----------
    _directory_name : str
        The name of directory that contains tif files that user wants to see
    dir_fil : list of strings
        list of all files in directory
    file_list : list of strings
        list of all tif files in directory
    pic_list : list of 2D numpy arrays
        list of all data read in from the tif files
    """

    def __init__(self):
        """
        This initializes the TifFileFinder class

        Returns
        -------
        None
        """
        self._directory_name = None
        self.dir_fil = []
        self.file_list = []
        self.pic_list = []

    def get_file_list(self):
        """
        This class gets the initial list of tif files

        Parameters
        ----------
        self

        Returns
        -------
        None

        """
        try:
            if self._directory_name is None:
                raise NotADirectoryError
            if self._directory_name[-1] != '/' or '\\':
                self._directory_name += '/'
            self.dir_fil = os.listdir(self._directory_name)
            self.dir_fil.sort(key=lambda x: os.path.getmtime(self._directory_name + x))
            self.file_list = [file for file in self.dir_fil if file.endswith('.tif') and not
                              (file.endswith('.dark.tif') or file.endswith('.raw.tif'))]
            self.get_image_arrays()
        except IndexError:
            pass

    def get_image_arrays(self):
        """
        This method reads in the tif files into 2D numpy arrays and appends them to the class's pic_list

        Parameters
        ----------
        self

        Returns
        -------
        None

        """
        for i in self.file_list:
            self.pic_list.append(imread(self._directory_name + i))

    def get_new_files(self):
        """
        This method finds new tif files in the directory for the user

        Parameters
        ----------
        self

        Returns
        -------
        all returns from the get_new_images method

        """
        self.dir_fil = os.listdir(self._directory_name)
        self.dir_fil.sort(key=lambda x: os.path.getmtime(self._directory_name + x))
        new_file_list = [file for file in self.dir_fil if file.endswith('.tif') and not (file.endswith('.dark.tif') or
                                                                                file.endswith('.raw.tif'))]
        need_read_files = []
        for i in new_file_list:
            add = True
            for j in self.file_list:
                if i == j:
                    add = False
                    break
            if add:
                self.file_list.append(i)
                need_read_files.append(i)
        return self.get_new_images(need_read_files)

    def get_new_images(self, temp_file_list):
        """
        This method reads in the newly found files if there are any and returns them

        Parameters
        ----------
        self
        temp_file_list : list of strings
            temporary list containing all of the new names

        Returns
        -------
        temp_file_list: list of strings
            see above
        new_pic: list of 2D numpy arrays
            temporary list of all new read in data

        """
        new_pics = []
        if temp_file_list is not None:
            for i in temp_file_list:
                self.pic_list.append(imread(self._directory_name + i))
                new_pics.append(imread(self._directory_name + i))
        return temp_file_list, new_pics
<|endoftext|>


<commit_before>import pwd
import grp
import signal
import os
import log

def make_pid_file_path(name, pid_file_path):
    return os.path.join(pid_file_path, name + ".pid")

def pid_store(name, pid_file_path="/var/run"):
    os.umask(077) # set umask for pid
    pid_path = make_pid_file_path(name, pid_file_path)

    with open(pid_path, "w") as f:
        f.write(str(os.getpid()))

def pid_read(name, pid_file_path="/var/run"):
    pid_path = make_pid_file_path(name, pid_file_path)
    log.debug("Checking pid path: %s" % pid_path)

    try:
        with open(pid_path, "r") as f:
            pid = int(f.read())
            log.warn("Old process running at %d" % pid)
            return pid
    except IOError:
        return -1

def still_running(name, pid_file_path="/var/run"):
    pid = pid_read(name, pid_file_path=pid_file_path)

    if pid == -1:
        log.debug("Returned pid not running at %s" % pid)
        return False
    else:
        # check if the process is still running with kill
        try:
            os.kill(pid, 0)
            log.warn("Process really running at %d" % pid)
            return True
        except OSError:
            # this means the process is gone
            log.warn("Stale pid file %r has %d pid." % (
                make_pid_file_path(name, pid_file_path), pid))
            return False

def pid_remove_dead(name, pid_file_path="/var/run"):
    if not still_running(name, pid_file_path=pid_file_path):
        os.remove(make_pid_file_path(name, pid_file_path))

def daemonize(prog_name, pid_path="/var/run"):
    if os.fork() == 0:
        os.setsid()
        signal.signal(signal.SIGHUP, signal.SIG_IGN)
        pid = os.fork()

        if pid != 0:
            os._exit(0)
        else:
            pid_remove_dead(prog_name, pid_path)
            pid_store(prog_name, pid_path)
    else:
        os._exit(0)

def chroot_jail(root="/tmp"):
    os.chroot(root)

def drop_privileges(uid_name='nobody', gid_name='nogroup'):
    if os.getuid() != 0:
        return

    # Get the uid/gid from the name
    running_uid = pwd.getpwnam(uid_name).pw_uid
    running_gid = grp.getgrnam(gid_name).gr_gid

    # Remove group privileges
    os.setgroups([])

    # Try setting the new uid/gid
    os.setgid(running_gid)
    os.setuid(running_uid)

    # Ensure a very conservative umask
    os.umask(077)

def register_shutdown(handler):
    signal.signal(signal.SIGINT, handler)
    signal.signal(signal.SIGTERM, handler)

<commit_msg>Added some functions for killing and reloading servers with signals from pids.<commit_after>import pwd
import grp
import signal
import os
import log

def make_pid_file_path(name, pid_file_path="/var/run"):
    return os.path.join(pid_file_path, name + ".pid")

def pid_store(name, pid_file_path="/var/run"):
    os.umask(077) # set umask for pid
    pid_path = make_pid_file_path(name, pid_file_path)

    with open(pid_path, "w") as f:
        f.write(str(os.getpid()))

def pid_read(name, pid_file_path="/var/run"):
    pid_path = make_pid_file_path(name, pid_file_path)
    log.debug("Checking pid path: %s" % pid_path)

    try:
        with open(pid_path, "r") as f:
            return int(f.read())
    except IOError:
        return -1

def still_running(name, pid_file_path="/var/run"):
    pid = pid_read(name, pid_file_path=pid_file_path)

    if pid == -1:
        log.debug("Returned pid not running at %s" % pid)
        return False
    else:
        # check if the process is still running with kill
        try:
            os.kill(pid, 0)
            log.debug("Process running at %d" % pid)
            return True
        except OSError:
            # this means the process is gone
            log.warn("Stale pid file %r has %d pid." % (
                make_pid_file_path(name, pid_file_path), pid))
            return False

def kill_server(name, pid_file_path="/var/run", sig=signal.SIGINT):
    if still_running(name, pid_file_path=pid_file_path):
        pid = pid_read(name, pid_file_path=pid_file_path)
        os.kill(pid, signal.SIGINT)

def reload_server(name, pid_file_path="/var/run"):
    kill_server(name, pid_file_path=pid_file_path, sig=signal.SIGHUP)

def pid_remove_dead(name, pid_file_path="/var/run"):
    if not still_running(name, pid_file_path=pid_file_path):
        pid_file = make_pid_file_path(name, pid_file_path)
        if os.path.exists(pid_file):
            os.remove(pid_file)

def daemonize(prog_name, pid_path="/var/run"):
    if os.fork() == 0:
        os.setsid()
        signal.signal(signal.SIGHUP, signal.SIG_IGN)
        pid = os.fork()

        if pid != 0:
            os._exit(0)
        else:
            pid_remove_dead(prog_name, pid_path)
            pid_store(prog_name, pid_path)
    else:
        os._exit(0)

def chroot_jail(root="/tmp"):
    os.chroot(root)

def drop_privileges(uid_name='nobody', gid_name='nogroup'):
    if os.getuid() != 0:
        return

    # Get the uid/gid from the name
    running_uid = pwd.getpwnam(uid_name).pw_uid
    running_gid = grp.getgrnam(gid_name).gr_gid

    log.info("Dropping pivs to UID %r GID %r" % (running_uid, running_gid))

    # Remove group privileges
    os.setgroups([])

    # Try setting the new uid/gid
    os.setgid(running_gid)
    os.setuid(running_uid)

    # Ensure a very conservative umask
    os.umask(077)

def register_shutdown(handler):
    signal.signal(signal.SIGINT, handler)
    signal.signal(signal.SIGTERM, handler)

<|endoftext|>


<commit_before>"""Robot Scanner XUDD test demo.

Here's the premise.  There's a warehouse full of droids, some
infected, and some not.  The SecurityRobot is being sent in to clean
up the mess.  It's capable of sending a message that infected droids
are susceptible to responding in a predictable way.  Once it has
identified that a droid is infected, it shoots it full of holes till
the droid is terminated.  The SecurityRobot goes from room to room
till things are cleared out.

Overseeing the operation is the "overseer".  The security robot keeps
the overseer up to date on its progress as it goes.  (For this demo,
the overseer is also responsible for initializing the world and
reporting info back to the user.)
"""

from __future__ import print_function
import random

from xudd.hive import Hive
from xudd.actor import Actor

# List of room tuples of (clean_droids, infected_droids)

ROOM_STRUCTURE = [
    (3, 1),
    (0, 2),
    (8, 5),
    (5, 0),
    (2, 1)]

class Overseer(Actor):
    """
    Actor that initializes the world of this demo, starts the mission,
    and sends information about what's going on back to the user.
    """
    def __init__(self, *args, **kwargs):
        super(Overseer, self).__init__(self, *args, **kwargs)

        self.message_routing.extend(
            {"init_world": self.init_world,
             "transmission": self.transmission,
             "compile_and_shutdown": self.compile_and_shutdown})

    def init_world(self):
        """
        Initialize the world we're operating in for this demo.
        """
        # Add rooms and droids
        last_room = None
        first_room = None

        for clean_droids, infected_droids in ROOM_STRUCTURE:
            room = WarehouseRoom(self.hive)

            if last_room:
                self.hive.send_message(
                    to=last_room.id,
                    directive="set_next_room",
                    body={"id": room.id})
                self.hive.send_message(
                    to=room.id,
                    directive="set_previous_room",
                    body={"id": last_room.id})

            for droid_num in range(clean_droids):
                droid = Droid(self.hive, infected=False, room=room.id)
                yield self.wait_on_message(
                    to=droid.id,
                    directive="register_with_room")

            for droid_num in range(infected_droids):
                droid = Droid(self.hive, infected=True, room=room.id)
                yield self.wait_on_message(
                    to=droid.id,
                    directive="register_with_room")

            last_room = room
            if first_room == None:
                first_room = room

        # Add security robot
        security_robot = SecurityRobot(self.hive)

        # Tell the security robot to begin their mission
        self.hive.send_message(
            to=security_robot.id,
            directive="begin_mission",
            body={
                "starting_room": first_room.id})

    def transmission(self, message):
        print_function(message.body['message'])

class WarehouseRoom(Actor):
    """
    A room full of robots.
    """
    def __init__(self, *args, **kwargs):
        super(WarehouseRoom, self).__init__(self, *args, **kwargs)
        self.droids = []
        self.next_room = None
        self.previous_room = None

        self.message_routing.extend(
            {"set_next_room": self.set_next_room,
             "set_previous_room": self.set_previous_room,
             "get_next_room": self.get_next_room,
             "get_previous_room": self.get_previous_room,
             "register_droid": self.register_droid,
             "droids_in_room": self.droids_in_room})

    def set_next_room(self, message):
        self.next_room = message.body['id']

    def set_previous_room(self, message):
        self.previous_room = message.body['id']

    def get_next_room(self, message):
        message.reply(
            body={"id": self.next_room})

    def get_previous_room(self, message):
        message.reply(
            body={"id": self.previous_room})

    def list_droids(self, message):
        message.reply(
            body={"droid_ids": self.droids})

    def register_droid(self, message):
        self.droids.append(message.body['droid_id'])

class Droid(Actor):
    """
    A droid that may or may not be infected!

    What will happen?  Stay tuned!
    """
    def __init__(self, hive, id, infected=False):
        super(Droid, self).__init__(self, hive, id)
        self.infected = infected
        self.hp = 50

        self.message_routing.extend(
            {"infection_expose": self.infection_expose,
             "get_shot": self.get_shot})

    def infection_expose(self, message):
        message.reply(
            body={"is_infected": self.infected})

    def get_shot(self, message):
        damage = random.randrange(0, 60)
        self.hp -= damage
        alive = self.hp > 0

        message.reply(
            body={
                "hp_left": self.hp,
                "damage_taken": damage,
                "alive": alive})

        if not alive:
            self.hive.remove(self.id)

ALIVE_FORMAT = "Droid %s shot; taken %s damage. Still alive... %s hp left."
DEAD_FORMAT = "Droid %s shot; taken %s damage. Terminated."

class SecurityRobot(Actor):
    """
    Security robot... designed to seek out and destroy infected droids.
    """
    def __init__(self, hive, id):
        super(SecurityRobot, self).__init__(self, hive, id)

        # The room we're currently in
        self.room = None

        self.message_routing.extend(
            {"begin_mission": self.begin_mission})

    def __droid_status_format(self, shot_response):
        if shot_response.body["alive"]:
            return ALIVE_FORMAT % (
                shot_response.from_id,
                shot_response.body["damage_taken"],
                shot_response.body["hp_left"])
        else:
            return DEAD_FORMAT % (
                shot_response.from_id,
                shot_response.body["damage_taken"])

    def begin_mission(self, message):
        self.room = message.body['starting_room']

        # Walk through all rooms, clearing out infected droids
        while True:
            self.hive.send_message(
                to="overseer",
                directive="transmission",
                body={
                    "message": "Entering room %s..." % self.room})

            # Find all the droids in this room and exterminate the
            # infected ones.
            response = yield self.wait_on_message(
                to=self.room,
                directive="droids_in_room")
            for droid_id in response.body["droid_ids"]:
                response = yield self.wait_on_message(
                    to=droid_id,
                    directive="infection_expose")

                # If the droid is clean, let the overseer know and move on.
                if not response.body["is_infected"]:
                    transmission = (
                        "%s is clean... moving on." % droid_id)
                    self.hive.send_message(
                        to="overseer",
                        directive="transmission",
                        body={
                            "message": transmission})

                # Let the overseer know we found an infected droid
                # and are engaging
                transmission = (
                    "%s found to be infected... taking out" % droid_id)
                self.hive.send_message(
                    to="overseer",
                    directive="transmission",
                    body={
                        "message": transmission})

                # Keep firing till it's dead.
                infected_droid_alive = True
                while infected_droid_alive:
                    response = yield self.wait_on_message(
                        to=droid_id,
                        directive="get_shot")

                    # Relay the droid status
                    droid_status = self.__droid_status_format(response)
                    self.hive.send_message(
                        to="overseer",
                        directive="transmission",
                        body={
                            "message": droid_status})
                    infected_droid_alive = droid_status.body["alive"]

            # switch to next room, if there is one
            response = yield self.wait_on_message(
                to=self.room,
                directive="get_next_room")
            next_room = response.body["id"]
            if next_room:
                self.room = next_room
            else:
                # We're done scanning rooms finally
                break

        # Good job everyone! Shut down the operation.
        self.hive.send_message(
            to="overseer",
            directive="transmission",
            body={
                "message": "Mission accomplished."})
        self.hive.send_shutdown()

def main():
    hive = Hive()

    # Add overseer, who populates the world and reports things
    Overseer(hive, id="overseer")
    hive.send_message(
        to="overseer",
        directive="init_world")

    hive.workloop()

if __name__ == "__main__":
    main()
<commit_msg>Adding a continue for if a droid is clean<commit_after>"""Robot Scanner XUDD test demo.

Here's the premise.  There's a warehouse full of droids, some
infected, and some not.  The SecurityRobot is being sent in to clean
up the mess.  It's capable of sending a message that infected droids
are susceptible to responding in a predictable way.  Once it has
identified that a droid is infected, it shoots it full of holes till
the droid is terminated.  The SecurityRobot goes from room to room
till things are cleared out.

Overseeing the operation is the "overseer".  The security robot keeps
the overseer up to date on its progress as it goes.  (For this demo,
the overseer is also responsible for initializing the world and
reporting info back to the user.)
"""

from __future__ import print_function
import random

from xudd.hive import Hive
from xudd.actor import Actor

# List of room tuples of (clean_droids, infected_droids)

ROOM_STRUCTURE = [
    (3, 1),
    (0, 2),
    (8, 5),
    (5, 0),
    (2, 1)]

class Overseer(Actor):
    """
    Actor that initializes the world of this demo, starts the mission,
    and sends information about what's going on back to the user.
    """
    def __init__(self, *args, **kwargs):
        super(Overseer, self).__init__(self, *args, **kwargs)

        self.message_routing.extend(
            {"init_world": self.init_world,
             "transmission": self.transmission,
             "compile_and_shutdown": self.compile_and_shutdown})

    def init_world(self):
        """
        Initialize the world we're operating in for this demo.
        """
        # Add rooms and droids
        last_room = None
        first_room = None

        for clean_droids, infected_droids in ROOM_STRUCTURE:
            room = WarehouseRoom(self.hive)

            if last_room:
                self.hive.send_message(
                    to=last_room.id,
                    directive="set_next_room",
                    body={"id": room.id})
                self.hive.send_message(
                    to=room.id,
                    directive="set_previous_room",
                    body={"id": last_room.id})

            for droid_num in range(clean_droids):
                droid = Droid(self.hive, infected=False, room=room.id)
                yield self.wait_on_message(
                    to=droid.id,
                    directive="register_with_room")

            for droid_num in range(infected_droids):
                droid = Droid(self.hive, infected=True, room=room.id)
                yield self.wait_on_message(
                    to=droid.id,
                    directive="register_with_room")

            last_room = room
            if first_room == None:
                first_room = room

        # Add security robot
        security_robot = SecurityRobot(self.hive)

        # Tell the security robot to begin their mission
        self.hive.send_message(
            to=security_robot.id,
            directive="begin_mission",
            body={
                "starting_room": first_room.id})

    def transmission(self, message):
        print_function(message.body['message'])

class WarehouseRoom(Actor):
    """
    A room full of robots.
    """
    def __init__(self, *args, **kwargs):
        super(WarehouseRoom, self).__init__(self, *args, **kwargs)
        self.droids = []
        self.next_room = None
        self.previous_room = None

        self.message_routing.extend(
            {"set_next_room": self.set_next_room,
             "set_previous_room": self.set_previous_room,
             "get_next_room": self.get_next_room,
             "get_previous_room": self.get_previous_room,
             "register_droid": self.register_droid,
             "droids_in_room": self.droids_in_room})

    def set_next_room(self, message):
        self.next_room = message.body['id']

    def set_previous_room(self, message):
        self.previous_room = message.body['id']

    def get_next_room(self, message):
        message.reply(
            body={"id": self.next_room})

    def get_previous_room(self, message):
        message.reply(
            body={"id": self.previous_room})

    def list_droids(self, message):
        message.reply(
            body={"droid_ids": self.droids})

    def register_droid(self, message):
        self.droids.append(message.body['droid_id'])

class Droid(Actor):
    """
    A droid that may or may not be infected!

    What will happen?  Stay tuned!
    """
    def __init__(self, hive, id, infected=False):
        super(Droid, self).__init__(self, hive, id)
        self.infected = infected
        self.hp = 50

        self.message_routing.extend(
            {"infection_expose": self.infection_expose,
             "get_shot": self.get_shot})

    def infection_expose(self, message):
        message.reply(
            body={"is_infected": self.infected})

    def get_shot(self, message):
        damage = random.randrange(0, 60)
        self.hp -= damage
        alive = self.hp > 0

        message.reply(
            body={
                "hp_left": self.hp,
                "damage_taken": damage,
                "alive": alive})

        if not alive:
            self.hive.remove(self.id)

ALIVE_FORMAT = "Droid %s shot; taken %s damage. Still alive... %s hp left."
DEAD_FORMAT = "Droid %s shot; taken %s damage. Terminated."

class SecurityRobot(Actor):
    """
    Security robot... designed to seek out and destroy infected droids.
    """
    def __init__(self, hive, id):
        super(SecurityRobot, self).__init__(self, hive, id)

        # The room we're currently in
        self.room = None

        self.message_routing.extend(
            {"begin_mission": self.begin_mission})

    def __droid_status_format(self, shot_response):
        if shot_response.body["alive"]:
            return ALIVE_FORMAT % (
                shot_response.from_id,
                shot_response.body["damage_taken"],
                shot_response.body["hp_left"])
        else:
            return DEAD_FORMAT % (
                shot_response.from_id,
                shot_response.body["damage_taken"])

    def begin_mission(self, message):
        self.room = message.body['starting_room']

        # Walk through all rooms, clearing out infected droids
        while True:
            self.hive.send_message(
                to="overseer",
                directive="transmission",
                body={
                    "message": "Entering room %s..." % self.room})

            # Find all the droids in this room and exterminate the
            # infected ones.
            response = yield self.wait_on_message(
                to=self.room,
                directive="droids_in_room")
            for droid_id in response.body["droid_ids"]:
                response = yield self.wait_on_message(
                    to=droid_id,
                    directive="infection_expose")

                # If the droid is clean, let the overseer know and move on.
                if not response.body["is_infected"]:
                    transmission = (
                        "%s is clean... moving on." % droid_id)
                    self.hive.send_message(
                        to="overseer",
                        directive="transmission",
                        body={
                            "message": transmission})
                    continue

                # Let the overseer know we found an infected droid
                # and are engaging
                transmission = (
                    "%s found to be infected... taking out" % droid_id)
                self.hive.send_message(
                    to="overseer",
                    directive="transmission",
                    body={
                        "message": transmission})

                # Keep firing till it's dead.
                infected_droid_alive = True
                while infected_droid_alive:
                    response = yield self.wait_on_message(
                        to=droid_id,
                        directive="get_shot")

                    # Relay the droid status
                    droid_status = self.__droid_status_format(response)
                    self.hive.send_message(
                        to="overseer",
                        directive="transmission",
                        body={
                            "message": droid_status})
                    infected_droid_alive = droid_status.body["alive"]

            # switch to next room, if there is one
            response = yield self.wait_on_message(
                to=self.room,
                directive="get_next_room")
            next_room = response.body["id"]
            if next_room:
                self.room = next_room
            else:
                # We're done scanning rooms finally
                break

        # Good job everyone! Shut down the operation.
        self.hive.send_message(
            to="overseer",
            directive="transmission",
            body={
                "message": "Mission accomplished."})
        self.hive.send_shutdown()

def main():
    hive = Hive()

    # Add overseer, who populates the world and reports things
    Overseer(hive, id="overseer")
    hive.send_message(
        to="overseer",
        directive="init_world")

    hive.workloop()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import json
import difflib
import pprint
import pytest
from treeherder.model.derived.base import DatasetNotFoundError
from .sample_data_generator import job_data

slow = pytest.mark.slow

def test_unicode(jm):
    """Unicode representation of a ``JobModel`` is the project name."""
    assert unicode(jm) == unicode(jm.project)

def test_disconnect(jm):
    """test that your model disconnects"""

    # establish the connection to jobs.
    jm._get_last_insert_id()
    # establish the connection to objectstore
    jm.retrieve_job_data(limit=1)

    jm.disconnect()
    assert not jm.get_os_dhub().connection["master_host"]["con_obj"].open
    assert not jm.get_jobs_dhub().connection["master_host"]["con_obj"].open

def test_bad_contenttype(jm):
    """Test trying to get an invalid contenttype"""
    with pytest.raises(DatasetNotFoundError):
        jm.get_dhub("foo")

def do_job_ingestion(jm, job_data):
    """
    Test ingesting job blobs.  ``job_data`` maybe different sizes.

    Process each job structure in the job_data and verify.

    This rebuilds the JSON blob (for the most part) and compares that
    everything was stored correctly.

    """
    for blob in job_data:
        jm.store_job_data(json.dumps(blob))
        jobs = jm.process_objects(1)
        assert len(jobs) == 1
        job_id = jobs[0]

        # import time
        # time.sleep(60)

        # verify the job data
        exp_job = clean_job_blob_dict(blob["job"])
        act_job = JobDictBuilder(jm, job_id).as_dict()
        assert exp_job == act_job, diff_dict(exp_job, act_job)

        # verify the source data
        exp_src = clean_source_blob_dict(blob["sources"][0])
        act_src = SourceDictBuilder(jm, job_id).as_dict()
        assert exp_src == act_src, diff_dict(exp_src, act_src)

    complete_count = jm.get_os_dhub().execute(
        proc="objectstore_test.counts.complete")[0]["complete_count"]
    loading_count = jm.get_os_dhub().execute(
        proc="objectstore_test.counts.loading")[0]["loading_count"]

    assert complete_count == len(job_data)
    assert loading_count == 0

def test_ingest_single_sample_job(jm, sample_data):
    """Process a single job structure in the job_data.txt file"""
    job_data = sample_data.job_data[:1]
    do_job_ingestion(jm, job_data)

@slow
def test_ingest_all_sample_jobs(jm, sample_data):
    """
    Process each job structure in the job_data.txt file and verify.

    """
    job_data = sample_data.job_data
    do_job_ingestion(jm, job_data)

def test_artifact_log_ingestion(jm):
    """
    Test ingesting an artifact with a log

    artifact:{
        type:" json | img | ...",
        name:"",
        log_urls:[
            ]
        blob:""
    },
    """
    artifact = {
        u"type": u"json",
        u"name": u"arti-foo-ct",
        u"log_urls": [
            {
                u"url": u"http://ftp.mozilla.org/arty-facto/...",
                u"name": u"artifact_url"
            }
        ],
        u"blob": ""
    }
    blob = job_data(artifact=artifact)
    jm.store_job_data(json.dumps(blob))
    job_ids = jm.process_objects(1)

    assert get_objectstore_last_error(jm) == u"N"

    # print json.dumps(blob, indent=4)

    job_id = job_ids[0]

    exp_job = clean_job_blob_dict(blob["job"])
    act_job = JobDictBuilder(jm, job_id).as_dict()
    assert exp_job == act_job, diff_dict(exp_job, act_job)

def test_bad_date_value_ingestion(jm):
    """
    Test ingesting an blob with bad date value

    """

    blob = job_data(start_timestamp="foo")
    jm.store_job_data(json.dumps(blob))
    job_ids = jm.process_objects(1)

    assert get_objectstore_last_error(
        jm,
        ) == u"invalid literal for long() with base 10: 'foo'"

class SourceDictBuilder(object):
    """Given a ``job_id``, rebuild the dictionary the source came from."""

    def __init__(self, jm, job_id):
        self.jm = jm
        self.job_id = job_id

    def as_dict(self):
        source = self.jm.get_jobs_dhub().execute(
            proc="jobs_test.selects.job_source",
            placeholders=[self.job_id],
            return_type="iter"
        ).next()

        source["repository"] = self._get_repository(
            source["repository_id"])
        del(source["repository_id"])

        return unicode_keys(source)

    def _get_repository(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "repository",
            obj_id,
        ).get_column_data("name")
        return obj

class JobDictBuilder(object):
    """Given a ``job_id``, rebuild the dictionary the job came from."""

    def __init__(self, jm, job_id):
        self.jm = jm
        self.job_id = job_id

    def as_dict(self):
        job = self.jm.get_job(self.job_id)

        job["artifact"] = self._get_artifact()
        job["log_references"] = self._get_logs()

        job["option_collection"] = self._get_option_collection(
            job["option_collection_hash"])
        del(job["option_collection_hash"])

        job["machine_platform"] = self._get_machine_platform(
            job["machine_platform_id"])
        del(job["machine_platform_id"])

        job["build_platform"] = self._get_build_platform(
            job["build_platform_id"])
        del(job["build_platform_id"])

        job["machine"] = self._get_machine(
            job["machine_id"])
        del(job["machine_id"])

        job["product_name"] = self._get_product(
            job["product_id"])
        del(job["product_id"])

        job["name"] = self._get_name(
            job["job_type_id"])
        del(job["job_type_id"])

        del(job["id"])
        del(job["active_status"])
        del(job["result_set_id"])

        if not job["job_coalesced_to_guid"]:
            del(job["job_coalesced_to_guid"])

        return unicode_keys(job)

    def _get_option_collection(self, option_collection_hash):
        """
        Needs to work with hash.  Get row by id won't work anymore.
        probably need to a new getter where it gets the option id
        but the hash means there's possibly more than one option.
        maybe I need mauro to make a splitter get method?
        """
        option_iter = self.jm.refdata_model.get_option_names(
            option_collection_hash)
        options = {}
        for name_dict in option_iter:
            options[name_dict["name"]] = True

        return options

    def _get_machine_platform(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "machine_platform",
            obj_id,
        ).next()
        del(obj["active_status"])
        del(obj["id"])
        return unicode_keys(obj)

    def _get_build_platform(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "build_platform",
            obj_id,
        ).next()
        del(obj["active_status"])
        del(obj["id"])
        return unicode_keys(obj)

    def _get_machine(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "machine",
            obj_id,
        ).get_column_data("name")
        return obj

    def _get_product(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "product",
            obj_id,
        ).get_column_data("name")
        return obj

    def _get_name(self, obj_id):
        job_type = self.jm.refdata_model.get_row_by_id(
            "job_type",
            obj_id,
        ).next()
        job_group = self.jm.refdata_model.get_row_by_id(
            "job_group",
            job_type["job_group_id"],
        ).get_column_data("name")
        if job_type["name"]:
            return u"{0}-{1}".format(
                job_group,
                job_type["name"],
            )
        else:
            return job_group

    def _get_logs(self):
        logs = self.jm.get_jobs_dhub().execute(
            proc="jobs_test.selects.job_log_urls",
            placeholders=[self.job_id],
            key_column="id",
            return_type='dict',
        )

        log_values = []
        for log in logs.values():
            del(log["active_status"])
            del(log["id"])
            del(log["job_id"])
            log_values.append(unicode_keys(log))

        return log_values

    def _get_artifact(self):
        artifact = self.jm.get_jobs_dhub().execute(
            proc="jobs_test.selects.job_artifact",
            placeholders=[self.job_id],
            key_column="id",
            return_type='dict',
        )
        if not len(artifact):
            artifact = {}
        else:
            artifact = artifact[self.job_id]
            del(artifact["active_status"])
            del(artifact["id"])
            del(artifact["job_id"])

        return unicode_keys(artifact)

def unicode_keys(d):
    return dict([(unicode(k), v) for k, v in d.items()])

def clean_source_blob_dict(src):
    """Fix a few fields so they're easier to compare"""
    src["commit_timestamp"] = long(src["commit_timestamp"])
    src["push_timestamp"] = long(src["push_timestamp"])
    return src

def clean_job_blob_dict(job):
    """Fix a few fields so they're easier to compare"""
    job["start_timestamp"] = long(job["start_timestamp"])
    job["submit_timestamp"] = long(job["submit_timestamp"])
    job["end_timestamp"] = long(job["end_timestamp"])
    job["result"] = unicode(job["result"])

    # @@@ we don't keep track of VM'ness?
    try:
        del(job["machine_platform"]["vm"])
    except KeyError:
        pass  # oh, that's ok
    try:
        del(job["build_platform"]["vm"])
    except KeyError:
        pass  # oh, that's ok

    return job

def diff_dict(d1, d2):
    """Compare two dicts, the same way unittest.assertDictEqual does"""
    diff = ('\n' + '\n'.join(difflib.ndiff(
       pprint.pformat(d1).splitlines(),
       pprint.pformat(d2).splitlines())))
    return diff

def get_objectstore_last_error(jm):
    row_id = jm._get_last_insert_id("objectstore")

    row_data = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
        proc="objectstore_test.selects.row", placeholders=[row_id])[0]

    return row_data['error_msg']
<commit_msg>fixed artifact log test<commit_after>import json
import difflib
import pprint
import pytest
from treeherder.model.derived.base import DatasetNotFoundError
from .sample_data_generator import job_data

slow = pytest.mark.slow

def test_unicode(jm):
    """Unicode representation of a ``JobModel`` is the project name."""
    assert unicode(jm) == unicode(jm.project)

def test_disconnect(jm):
    """test that your model disconnects"""

    # establish the connection to jobs.
    jm._get_last_insert_id()
    # establish the connection to objectstore
    jm.retrieve_job_data(limit=1)

    jm.disconnect()
    assert not jm.get_os_dhub().connection["master_host"]["con_obj"].open
    assert not jm.get_jobs_dhub().connection["master_host"]["con_obj"].open

def test_bad_contenttype(jm):
    """Test trying to get an invalid contenttype"""
    with pytest.raises(DatasetNotFoundError):
        jm.get_dhub("foo")

def do_job_ingestion(jm, job_data):
    """
    Test ingesting job blobs.  ``job_data`` maybe different sizes.

    Process each job structure in the job_data and verify.

    This rebuilds the JSON blob (for the most part) and compares that
    everything was stored correctly.

    """
    for blob in job_data:
        jm.store_job_data(json.dumps(blob))
        jobs = jm.process_objects(1)
        assert len(jobs) == 1
        job_id = jobs[0]

        # import time
        # time.sleep(60)

        # verify the job data
        exp_job = clean_job_blob_dict(blob["job"])
        act_job = JobDictBuilder(jm, job_id).as_dict()
        assert exp_job == act_job, diff_dict(exp_job, act_job)

        # verify the source data
        exp_src = clean_source_blob_dict(blob["sources"][0])
        act_src = SourceDictBuilder(jm, job_id).as_dict()
        assert exp_src == act_src, diff_dict(exp_src, act_src)

    complete_count = jm.get_os_dhub().execute(
        proc="objectstore_test.counts.complete")[0]["complete_count"]
    loading_count = jm.get_os_dhub().execute(
        proc="objectstore_test.counts.loading")[0]["loading_count"]

    assert complete_count == len(job_data)
    assert loading_count == 0

def test_ingest_single_sample_job(jm, sample_data):
    """Process a single job structure in the job_data.txt file"""
    job_data = sample_data.job_data[:1]
    do_job_ingestion(jm, job_data)

@slow
def test_ingest_all_sample_jobs(jm, sample_data):
    """
    Process each job structure in the job_data.txt file and verify.

    """
    job_data = sample_data.job_data
    do_job_ingestion(jm, job_data)

def test_artifact_log_ingestion(jm):
    """
    Test ingesting an artifact with a log

    artifact:{
        type:" json | img | ...",
        name:"",
        log_urls:[
            ]
        blob:""
    },
    """
    artifact = {
        u"type": u"json",
        u"name": u"arti-foo-ct",
        u"log_urls": [
            {
                u"url": u"http://ftp.mozilla.org/arty-facto/...",
                u"name": u"artifact_url"
            }
        ],
        u"blob": ""
    }
    blob = job_data(artifact=artifact)
    jm.store_job_data(json.dumps(blob))
    job_ids = jm.process_objects(1)

    assert get_objectstore_last_error(jm) == u"N"

    # print json.dumps(blob, indent=4)

    job_id = job_ids[0]

    exp_job = clean_job_blob_dict(blob["job"])
    act_job = JobDictBuilder(jm, job_id).as_dict()
    assert exp_job == act_job, diff_dict(exp_job, act_job)

def test_bad_date_value_ingestion(jm):
    """
    Test ingesting an blob with bad date value

    """

    blob = job_data(start_timestamp="foo")
    jm.store_job_data(json.dumps(blob))
    job_ids = jm.process_objects(1)

    assert get_objectstore_last_error(
        jm,
        ) == u"invalid literal for long() with base 10: 'foo'"

class SourceDictBuilder(object):
    """Given a ``job_id``, rebuild the dictionary the source came from."""

    def __init__(self, jm, job_id):
        self.jm = jm
        self.job_id = job_id

    def as_dict(self):
        source = self.jm.get_jobs_dhub().execute(
            proc="jobs_test.selects.job_source",
            placeholders=[self.job_id],
            return_type="iter"
        ).next()

        source["repository"] = self._get_repository(
            source["repository_id"])
        del(source["repository_id"])

        return unicode_keys(source)

    def _get_repository(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "repository",
            obj_id,
        ).get_column_data("name")
        return obj

class JobDictBuilder(object):
    """Given a ``job_id``, rebuild the dictionary the job came from."""

    def __init__(self, jm, job_id):
        self.jm = jm
        self.job_id = job_id

    def as_dict(self):
        job = self.jm.get_job(self.job_id)

        job["artifact"] = self._get_artifact()
        job["log_references"] = self._get_logs()

        job["option_collection"] = self._get_option_collection(
            job["option_collection_hash"])
        del(job["option_collection_hash"])

        job["machine_platform"] = self._get_machine_platform(
            job["machine_platform_id"])
        del(job["machine_platform_id"])

        job["build_platform"] = self._get_build_platform(
            job["build_platform_id"])
        del(job["build_platform_id"])

        job["machine"] = self._get_machine(
            job["machine_id"])
        del(job["machine_id"])

        job["product_name"] = self._get_product(
            job["product_id"])
        del(job["product_id"])

        job["name"] = self._get_name(
            job["job_type_id"])
        del(job["job_type_id"])

        del(job["id"])
        del(job["active_status"])
        del(job["result_set_id"])

        if not job["job_coalesced_to_guid"]:
            del(job["job_coalesced_to_guid"])

        return unicode_keys(job)

    def _get_option_collection(self, option_collection_hash):
        """
        Needs to work with hash.  Get row by id won't work anymore.
        probably need to a new getter where it gets the option id
        but the hash means there's possibly more than one option.
        maybe I need mauro to make a splitter get method?
        """
        option_iter = self.jm.refdata_model.get_option_names(
            option_collection_hash)
        options = {}
        for name_dict in option_iter:
            options[name_dict["name"]] = True

        return options

    def _get_machine_platform(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "machine_platform",
            obj_id,
        ).next()
        del(obj["active_status"])
        del(obj["id"])
        return unicode_keys(obj)

    def _get_build_platform(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "build_platform",
            obj_id,
        ).next()
        del(obj["active_status"])
        del(obj["id"])
        return unicode_keys(obj)

    def _get_machine(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "machine",
            obj_id,
        ).get_column_data("name")
        return obj

    def _get_product(self, obj_id):
        obj = self.jm.refdata_model.get_row_by_id(
            "product",
            obj_id,
        ).get_column_data("name")
        return obj

    def _get_name(self, obj_id):
        job_type = self.jm.refdata_model.get_row_by_id(
            "job_type",
            obj_id,
        ).next()
        job_group = self.jm.refdata_model.get_row_by_id(
            "job_group",
            job_type["job_group_id"],
        ).get_column_data("name")
        if job_type["name"]:
            return u"{0}-{1}".format(
                job_group,
                job_type["name"],
            )
        else:
            return job_group

    def _get_logs(self):
        logs = self.jm.get_jobs_dhub().execute(
            proc="jobs_test.selects.job_log_urls",
            placeholders=[self.job_id],
            key_column="id",
            return_type='dict',
        )

        log_values = []
        for log in logs.values():
            del(log["active_status"])
            del(log["id"])
            del(log["job_id"])
            log_values.append(unicode_keys(log))

        return log_values

    def _get_artifact(self):
        artifact = self.jm.get_jobs_dhub().execute(
            proc="jobs_test.selects.job_artifact",
            placeholders=[self.job_id],
            key_column="id",
            return_type='dict',
        )
        if not len(artifact):
            artifact = {}
        else:
            artifact = artifact[self.job_id]
            del(artifact["active_status"])
            del(artifact["id"])
            del(artifact["job_id"])

        return unicode_keys(artifact)

def unicode_keys(d):
    return dict([(unicode(k), v) for k, v in d.items()])

def clean_source_blob_dict(src):
    """Fix a few fields so they're easier to compare"""
    src["commit_timestamp"] = long(src["commit_timestamp"])
    src["push_timestamp"] = long(src["push_timestamp"])
    return src

def clean_job_blob_dict(job):
    """Fix a few fields so they're easier to compare"""
    job["start_timestamp"] = long(job["start_timestamp"])
    job["submit_timestamp"] = long(job["submit_timestamp"])
    job["end_timestamp"] = long(job["end_timestamp"])
    job["result"] = unicode(job["result"])

    # move artifact logs to log_references area for comparison
    try:
        artlog = job["artifact"]["log_urls"]
        job["log_references"].extend(artlog)
        del(job["artifact"]["log_urls"])
    except KeyError:
        pass  # no problem

    # @@@ we don't keep track of VM'ness?
    try:
        del(job["machine_platform"]["vm"])
    except KeyError:
        pass  # no problem
    try:
        del(job["build_platform"]["vm"])
    except KeyError:
        pass  # no problem

    return job

def diff_dict(d1, d2):
    """Compare two dicts, the same way unittest.assertDictEqual does"""
    diff = ('\n' + '\n'.join(difflib.ndiff(
       pprint.pformat(d1).splitlines(),
       pprint.pformat(d2).splitlines())))
    return diff

def get_objectstore_last_error(jm):
    row_id = jm._get_last_insert_id("objectstore")

    row_data = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
        proc="objectstore_test.selects.row", placeholders=[row_id])[0]

    return row_data['error_msg']
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

class SlidingWindow(object):
    """SlidingWindow

    Sliding window iterator which produces slice objects to slice in a sliding window. This is useful for inference.

    """
    def __init__(self, img_shape, window_shape, striding=None):
        """Constructs a sliding window iterator

        Parameters
        ----------
        img_shape : tuple or list
            shape of the image to slide over
        window_shape : tuple or list
            shape of the window to extract
        striding : tuple or list, optional
            amount to move the window between each position
        """
        self.img_shape = img_shape
        self.window_shape = window_shape
        self.curr_pos = [0, 0, 0]
        self.done = False
        self.striding = window_shape
        if striding:
            self.striding = striding

    def __iter__(self):
        return self

    # py 2.* compatability hack
    def next(self):
        return self.__next__()

    def __next__(self):
        if self.done:
            raise StopIteration()
        slicer = [slice(None)] * 5
        move_dim = True
        for dim, pos in enumerate(self.curr_pos):
            low = pos
            high = pos + self.window_shape[dim]
            if move_dim:
                if high >= self.img_shape[dim]:
                    self.curr_pos[dim] = 0
                    move_dim = True
                else:
                    self.curr_pos[dim] += self.striding[dim]
                    move_dim = False
            if high >= self.img_shape[dim]:
                low = self.img_shape[dim] - self.window_shape[dim]
                high = self.img_shape[dim]
            slicer[dim + 1] = slice(low, high)

        if (np.array(self.curr_pos) == [0, 0, 0]).all():
            self.done = True
        return slicer
<commit_msg>make dimensionality invariant<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

class SlidingWindow(object):
    """SlidingWindow

    Sliding window iterator which produces slice objects to slice in a sliding window. This is useful for inference.

    """
    def __init__(self, img_shape, window_shape, has_batch_dim=True, striding=None):
        """Constructs a sliding window iterator

        Parameters
        ----------
        img_shape : tuple or list
            shape of the image to slide over
        window_shape : tuple or list
            shape of the window to extract
        striding : tuple or list, optional
            amount to move the window between each position
        """
        self.img_shape = img_shape
        self.window_shape = window_shape
        self.rank = len(img_shape)
        self.curr_pos = [0] * self.rank
        self.end_pos = [0] * self.rank
        self.done = False
        self.striding = window_shape
        self.has_batch_dim = has_batch_dim
        if striding:
            self.striding = striding

    def __iter__(self):
        return self

    # py 2.* compatability hack
    def next(self):
        return self.__next__()

    def __next__(self):
        if self.done:
            raise StopIteration()
        slicer = [slice(None)] * (self.rank + 1) if self.has_batch_dim else [slice(None)] * self.rank
        move_dim = True
        for dim, pos in enumerate(self.curr_pos):
            low = pos
            high = pos + self.window_shape[dim]
            if move_dim:
                if high >= self.img_shape[dim]:
                    self.curr_pos[dim] = 0
                    move_dim = True
                else:
                    self.curr_pos[dim] += self.striding[dim]
                    move_dim = False
            if high >= self.img_shape[dim]:
                low = self.img_shape[dim] - self.window_shape[dim]
                high = self.img_shape[dim]

            if self.has_batch_dim:
                slicer[dim + 1] = slice(low, high)
            else:
                slicer[dim] = slice(low, high)

        if (np.array(self.curr_pos) == self.end_pos).all():
            self.done = True
        return slicer
<|endoftext|>


<commit_before>"""
34. Generating HTML forms from models

Django provides shortcuts for creating Form objects from a model class and a
model instance.

The function django.newforms.form_for_model() takes a model class and returns
a Form that is tied to the model. This Form works just like any other Form,
with one additional method: create(). The create() method creates an instance
of the model and returns that newly created instance. It saves the instance to
the database if create(save=True), which is default. If you pass
create(save=False), then you'll get the object without saving it.

The function django.newforms.form_for_instance() takes a model instance and
returns a Form that is tied to the instance. This form works just like any
other Form, with one additional method: apply_changes(). The apply_changes()
method updates the model instance. It saves the changes to the database if
apply_changes(save=True), which is default. If you pass save=False, then you'll
get the object without saving it.
"""

from django.db import models

class Category(models.Model):
    name = models.CharField(maxlength=20)
    url = models.CharField('The URL', maxlength=40)

    def __str__(self):
        return self.name

class Writer(models.Model):
    name = models.CharField(maxlength=50)

    def __str__(self):
        return self.name

class Article(models.Model):
    headline = models.CharField(maxlength=50)
    pub_date = models.DateField()
    writer = models.ForeignKey(Writer)
    categories = models.ManyToManyField(Category, blank=True)

    def __str__(self):
        return self.headline

__test__ = {'API_TESTS': """
>>> from django.newforms import form_for_model, form_for_instance, BaseForm
>>> import datetime

>>> Category.objects.all()
[]

>>> CategoryForm = form_for_model(Category)
>>> f = CategoryForm()
>>> print f

Name:	<input id="id_name" type="text" name="name" maxlength="20" />

The URL:	<input id="id_url" type="text" name="url" maxlength="40" />

>>> print f.as_ul()

Name: <input id="id_name" type="text" name="name" maxlength="20" />

The URL: <input id="id_url" type="text" name="url" maxlength="40" />

>>> print f['name']
<input id="id_name" type="text" name="name" maxlength="20" />

>>> f = CategoryForm(auto_id=False)
>>> print f.as_ul()

Name: <input type="text" name="name" maxlength="20" />

The URL: <input type="text" name="url" maxlength="40" />

>>> f = CategoryForm({'name': 'Entertainment', 'url': 'entertainment'})
>>> f.errors
{}
>>> f.clean_data
{'url': u'entertainment', 'name': u'Entertainment'}
>>> obj = f.create()
>>> obj
<Category: Entertainment>
>>> Category.objects.all()
[<Category: Entertainment>]

>>> f = CategoryForm({'name': "It's a test", 'url': 'test'})
>>> f.errors
{}
>>> f.clean_data
{'url': u'test', 'name': u"It's a test"}
>>> obj = f.create()
>>> obj
<Category: It's a test>
>>> Category.objects.all()
[<Category: Entertainment>, <Category: It's a test>]

If you call create() with save=False, then it will return an object that hasn't
yet been saved. In this case, it's up to you to save it.
>>> f = CategoryForm({'name': 'Third test', 'url': 'third'})
>>> f.errors
{}
>>> f.clean_data
{'url': u'third', 'name': u'Third test'}
>>> obj = f.create(save=False)
>>> obj
<Category: Third test>
>>> Category.objects.all()
[<Category: Entertainment>, <Category: It's a test>]
>>> obj.save()
>>> Category.objects.all()
[<Category: Entertainment>, <Category: It's a test>, <Category: Third test>]

If you call create() with invalid data, you'll get a ValueError.
>>> f = CategoryForm({'name': '', 'url': 'foo'})
>>> f.errors
{'name': [u'This field is required.']}
>>> f.clean_data
>>> f.create()
Traceback (most recent call last):
...
ValueError: The Category could not be created because the data didn't validate.
>>> f = CategoryForm({'name': '', 'url': 'foo'})
>>> f.create()
Traceback (most recent call last):
...
ValueError: The Category could not be created because the data didn't validate.

Create a couple of Writers.
>>> w = Writer(name='Mike Royko')
>>> w.save()
>>> w = Writer(name='Bob Woodward')
>>> w.save()

ManyToManyFields are represented by a MultipleChoiceField, and ForeignKeys are
represented by a ChoiceField.
>>> ArticleForm = form_for_model(Article)
>>> f = ArticleForm(auto_id=False)
>>> print f

Headline:	<input type="text" name="headline" maxlength="50" />

Pub date:	<input type="text" name="pub_date" />

Writer:	<select name="writer">
<option value="" selected="selected">---------</option>
<option value="1">Mike Royko</option>
<option value="2">Bob Woodward</option>
</select>

Categories:	<select multiple="multiple" name="categories">
<option value="1">Entertainment</option>
<option value="2">It's a test</option>
<option value="3">Third test</option>
</select>

You can pass a custom Form class to form_for_model. Make sure it's a
subclass of BaseForm, not Form.
>>> class CustomForm(BaseForm):
...     def say_hello(self):
...         print 'hello'
>>> CategoryForm = form_for_model(Category, form=CustomForm)
>>> f = CategoryForm()
>>> f.say_hello()
hello

Use form_for_instance to create a Form from a model instance. There are two
differences between this Form and one created via form_for_model. First, the
object's current values are inserted as 'initial' data in each Field. Second,
the Form gets an apply_changes() method instead of a create() method.
>>> w = Writer.objects.get(name='Mike Royko')
>>> RoykoForm = form_for_instance(w)
>>> f = RoykoForm(auto_id=False)
>>> print f

Name:	<input type="text" name="name" value="Mike Royko" maxlength="50" />

>>> art = Article(headline='Test article', pub_date=datetime.date(1988, 1, 4), writer=w)
>>> art.save()
>>> art.id
1
>>> TestArticleForm = form_for_instance(art)
>>> f = TestArticleForm(auto_id=False)
>>> print f.as_ul()

Headline: <input type="text" name="headline" value="Test article" maxlength="50" />

Pub date: <input type="text" name="pub_date" value="1988-01-04" />

Writer: <select name="writer">
<option value="">---------</option>
<option value="1" selected="selected">Mike Royko</option>
<option value="2">Bob Woodward</option>
</select>

Categories: <select multiple="multiple" name="categories">
<option value="1">Entertainment</option>
<option value="2">It's a test</option>
<option value="3">Third test</option>
</select>

>>> f = TestArticleForm({'headline': u'New headline', 'pub_date': u'1988-01-04', 'writer': u'1'})
>>> f.is_valid()
True
>>> new_art = f.apply_changes()
>>> new_art.id
1
>>> new_art = Article.objects.get(id=1)
>>> new_art.headline
'New headline'

Add some categories and test the many-to-many form output.
>>> new_art.categories.all()
[]
>>> new_art.categories.add(Category.objects.get(name='Entertainment'))
>>> new_art.categories.all()
[<Category: Entertainment>]
>>> TestArticleForm = form_for_instance(new_art)
>>> f = TestArticleForm(auto_id=False)
>>> print f.as_ul()

Headline: <input type="text" name="headline" value="New headline" maxlength="50" />

Pub date: <input type="text" name="pub_date" value="1988-01-04" />

Writer: <select name="writer">
<option value="">---------</option>
<option value="1" selected="selected">Mike Royko</option>
<option value="2">Bob Woodward</option>
</select>

Categories: <select multiple="multiple" name="categories">
<option value="1" selected="selected">Entertainment</option>
<option value="2">It's a test</option>
<option value="3">Third test</option>
</select>

"""}
<commit_msg>Fixed #3252 -- Fixed bugs in model_forms unit tests, related to recent newforms clean_data change. Thanks for the patch, mir@noris.de<commit_after>"""
34. Generating HTML forms from models

Django provides shortcuts for creating Form objects from a model class and a
model instance.

The function django.newforms.form_for_model() takes a model class and returns
a Form that is tied to the model. This Form works just like any other Form,
with one additional method: create(). The create() method creates an instance
of the model and returns that newly created instance. It saves the instance to
the database if create(save=True), which is default. If you pass
create(save=False), then you'll get the object without saving it.

The function django.newforms.form_for_instance() takes a model instance and
returns a Form that is tied to the instance. This form works just like any
other Form, with one additional method: apply_changes(). The apply_changes()
method updates the model instance. It saves the changes to the database if
apply_changes(save=True), which is default. If you pass save=False, then you'll
get the object without saving it.
"""

from django.db import models

class Category(models.Model):
    name = models.CharField(maxlength=20)
    url = models.CharField('The URL', maxlength=40)

    def __str__(self):
        return self.name

class Writer(models.Model):
    name = models.CharField(maxlength=50)

    def __str__(self):
        return self.name

class Article(models.Model):
    headline = models.CharField(maxlength=50)
    pub_date = models.DateField()
    writer = models.ForeignKey(Writer)
    categories = models.ManyToManyField(Category, blank=True)

    def __str__(self):
        return self.headline

__test__ = {'API_TESTS': """
>>> from django.newforms import form_for_model, form_for_instance, BaseForm
>>> import datetime

>>> Category.objects.all()
[]

>>> CategoryForm = form_for_model(Category)
>>> f = CategoryForm()
>>> print f

Name:	<input id="id_name" type="text" name="name" maxlength="20" />

The URL:	<input id="id_url" type="text" name="url" maxlength="40" />

>>> print f.as_ul()

Name: <input id="id_name" type="text" name="name" maxlength="20" />

The URL: <input id="id_url" type="text" name="url" maxlength="40" />

>>> print f['name']
<input id="id_name" type="text" name="name" maxlength="20" />

>>> f = CategoryForm(auto_id=False)
>>> print f.as_ul()

Name: <input type="text" name="name" maxlength="20" />

The URL: <input type="text" name="url" maxlength="40" />

>>> f = CategoryForm({'name': 'Entertainment', 'url': 'entertainment'})
>>> f.is_valid()
True
>>> f.clean_data
{'url': u'entertainment', 'name': u'Entertainment'}
>>> obj = f.create()
>>> obj
<Category: Entertainment>
>>> Category.objects.all()
[<Category: Entertainment>]

>>> f = CategoryForm({'name': "It's a test", 'url': 'test'})
>>> f.is_valid()
True
>>> f.clean_data
{'url': u'test', 'name': u"It's a test"}
>>> obj = f.create()
>>> obj
<Category: It's a test>
>>> Category.objects.all()
[<Category: Entertainment>, <Category: It's a test>]

If you call create() with save=False, then it will return an object that hasn't
yet been saved. In this case, it's up to you to save it.
>>> f = CategoryForm({'name': 'Third test', 'url': 'third'})
>>> f.is_valid()
True
>>> f.clean_data
{'url': u'third', 'name': u'Third test'}
>>> obj = f.create(save=False)
>>> obj
<Category: Third test>
>>> Category.objects.all()
[<Category: Entertainment>, <Category: It's a test>]
>>> obj.save()
>>> Category.objects.all()
[<Category: Entertainment>, <Category: It's a test>, <Category: Third test>]

If you call create() with invalid data, you'll get a ValueError.
>>> f = CategoryForm({'name': '', 'url': 'foo'})
>>> f.errors
{'name': [u'This field is required.']}
>>> f.clean_data
Traceback (most recent call last):
...
AttributeError: 'CategoryForm' object has no attribute 'clean_data'
>>> f.create()
Traceback (most recent call last):
...
ValueError: The Category could not be created because the data didn't validate.
>>> f = CategoryForm({'name': '', 'url': 'foo'})
>>> f.create()
Traceback (most recent call last):
...
ValueError: The Category could not be created because the data didn't validate.

Create a couple of Writers.
>>> w = Writer(name='Mike Royko')
>>> w.save()
>>> w = Writer(name='Bob Woodward')
>>> w.save()

ManyToManyFields are represented by a MultipleChoiceField, and ForeignKeys are
represented by a ChoiceField.
>>> ArticleForm = form_for_model(Article)
>>> f = ArticleForm(auto_id=False)
>>> print f

Headline:	<input type="text" name="headline" maxlength="50" />

Pub date:	<input type="text" name="pub_date" />

Writer:	<select name="writer">
<option value="" selected="selected">---------</option>
<option value="1">Mike Royko</option>
<option value="2">Bob Woodward</option>
</select>

Categories:	<select multiple="multiple" name="categories">
<option value="1">Entertainment</option>
<option value="2">It's a test</option>
<option value="3">Third test</option>
</select>

You can pass a custom Form class to form_for_model. Make sure it's a
subclass of BaseForm, not Form.
>>> class CustomForm(BaseForm):
...     def say_hello(self):
...         print 'hello'
>>> CategoryForm = form_for_model(Category, form=CustomForm)
>>> f = CategoryForm()
>>> f.say_hello()
hello

Use form_for_instance to create a Form from a model instance. There are two
differences between this Form and one created via form_for_model. First, the
object's current values are inserted as 'initial' data in each Field. Second,
the Form gets an apply_changes() method instead of a create() method.
>>> w = Writer.objects.get(name='Mike Royko')
>>> RoykoForm = form_for_instance(w)
>>> f = RoykoForm(auto_id=False)
>>> print f

Name:	<input type="text" name="name" value="Mike Royko" maxlength="50" />

>>> art = Article(headline='Test article', pub_date=datetime.date(1988, 1, 4), writer=w)
>>> art.save()
>>> art.id
1
>>> TestArticleForm = form_for_instance(art)
>>> f = TestArticleForm(auto_id=False)
>>> print f.as_ul()

Headline: <input type="text" name="headline" value="Test article" maxlength="50" />

Pub date: <input type="text" name="pub_date" value="1988-01-04" />

Writer: <select name="writer">
<option value="">---------</option>
<option value="1" selected="selected">Mike Royko</option>
<option value="2">Bob Woodward</option>
</select>

Categories: <select multiple="multiple" name="categories">
<option value="1">Entertainment</option>
<option value="2">It's a test</option>
<option value="3">Third test</option>
</select>

>>> f = TestArticleForm({'headline': u'New headline', 'pub_date': u'1988-01-04', 'writer': u'1'})
>>> f.is_valid()
True
>>> new_art = f.apply_changes()
>>> new_art.id
1
>>> new_art = Article.objects.get(id=1)
>>> new_art.headline
'New headline'

Add some categories and test the many-to-many form output.
>>> new_art.categories.all()
[]
>>> new_art.categories.add(Category.objects.get(name='Entertainment'))
>>> new_art.categories.all()
[<Category: Entertainment>]
>>> TestArticleForm = form_for_instance(new_art)
>>> f = TestArticleForm(auto_id=False)
>>> print f.as_ul()

Headline: <input type="text" name="headline" value="New headline" maxlength="50" />

Pub date: <input type="text" name="pub_date" value="1988-01-04" />

Writer: <select name="writer">
<option value="">---------</option>
<option value="1" selected="selected">Mike Royko</option>
<option value="2">Bob Woodward</option>
</select>

Categories: <select multiple="multiple" name="categories">
<option value="1" selected="selected">Entertainment</option>
<option value="2">It's a test</option>
<option value="3">Third test</option>
</select>

"""}
<|endoftext|>