<commit_before>from os import getcwd
from sys import path as ppath
ppath.insert(1,getcwd()+'/modules') # TODO: win32 compatibilite (python path)
# TODO: load et launch des tests auto (avec bootstrap contenant ci dessus)

from intelligine.tests.simulation.pheromone.Base import Base
from intelligine.simulation.pheromone.DirectionPheromone import DirectionPheromone
from intelligine.core.Context import Context
from intelligine.cst import PHEROMON_DIRECTION, PHEROMON_DIR_EXPLO

class TestDirection(Base):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._context = Context()

    def setUp(self):
        self._context = Context()

    def _set_up_pheromones(self, pheromones, re_init=True):
        if re_init:
            self._context = Context()
        for position in pheromones:
            self._context.pheromones().set_pheromones(position, pheromones[position])

    def _test_direction_point(self, pheromones, direction, pheromone_type=PHEROMON_DIR_EXPLO, reference_point=(0, 0, 0)):
        """

        :param pheromones:
        :param direction:
        :param pheromone_type:
        :param reference_point:
        :return: void
        """
        self._set_up_pheromones(pheromones)
        direction_tested = DirectionPheromone.get_direction_for_point(self._context, reference_point, pheromone_type)
        self.assertEqual(direction, direction_tested, "Direction must be %s" % direction)

    def test_direct_route(self):
        """
        Test easy direction with 1 best pheromones just near actual position
        :return:
        """
        test_data = {
            10: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            11: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 0, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            12: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            13: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            15: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 1, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            16: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            17: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 0, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            18: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            }
        }

        for direction_wanted in test_data:
            self._test_direction_point(test_data[direction_wanted], direction_wanted)

    def test_with_multiple_same_intensity(self):
        """
        Test find route in middle of multiple pheromones
        :return:
        """
        test_data = {
            10: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}},
                (0, 1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (12, 1)}}
            },
            10: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}},
                (0, 1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (12, 1)}},
                (0, 0, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (12, 1)}}
            },
        }

        for direction_wanted in test_data:
            self._test_direction_point(test_data[direction_wanted], direction_wanted)

    def test_with_multiple_different_intensity(self):
        """
        Test find route in middle of multiple pheromones
        :return:
        """
        test_data = {
            10: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 2)}},
                (0, -1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 2)}},
                (0, 1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (8, 1)}}
            }
        }

        for direction_wanted in test_data:
            self._test_direction_point(test_data[direction_wanted], direction_wanted)<commit_msg>pheromones tests: rename fct<commit_after>from os import getcwd
from sys import path as ppath
ppath.insert(1,getcwd()+'/modules') # TODO: win32 compatibilite (python path)
# TODO: load et launch des tests auto (avec bootstrap contenant ci dessus)

from intelligine.tests.simulation.pheromone.Base import Base
from intelligine.simulation.pheromone.DirectionPheromone import DirectionPheromone
from intelligine.core.Context import Context
from intelligine.cst import PHEROMON_DIRECTION, PHEROMON_DIR_EXPLO

class TestDirection(Base):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._context = Context()

    def setUp(self):
        self._context = Context()

    def _set_up_pheromones(self, pheromones, re_init=True):
        if re_init:
            self._context = Context()
        for position in pheromones:
            self._context.pheromones().set_pheromones(position, pheromones[position])

    def _get_direction_for_point(self, pheromones, direction, pheromone_type=PHEROMON_DIR_EXPLO,
                                 reference_point=(0, 0, 0)):
        """

        :param pheromones:
        :param direction:
        :param pheromone_type:
        :param reference_point:
        :return: void
        """
        self._set_up_pheromones(pheromones)
        direction_tested = DirectionPheromone.get_direction_for_point(self._context, reference_point, pheromone_type)
        self.assertEqual(direction, direction_tested, "Direction must be %s" % direction)

    def test_direct_route(self):
        """
        Test easy direction with 1 best pheromones just near actual position
        :return:
        """
        test_data = {
            10: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            11: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 0, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            12: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            13: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            15: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 1, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            16: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            17: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 0, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            },
            18: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, 1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}}
            }
        }

        for direction_wanted in test_data:
            self._get_direction_for_point(test_data[direction_wanted], direction_wanted)

    def test_with_multiple_same_intensity(self):
        """
        Test find route in middle of multiple pheromones
        :return:
        """
        test_data = {
            10: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}},
                (0, 1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (12, 1)}}
            },
            10: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 1)}},
                (0, -1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 1)}},
                (0, 1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (12, 1)}},
                (0, 0, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (12, 1)}}
            },
        }

        for direction_wanted in test_data:
            self._get_direction_for_point(test_data[direction_wanted], direction_wanted)

    def test_with_multiple_different_intensity(self):
        """
        Test find route in middle of multiple pheromones
        :return:
        """
        test_data = {
            10: {
                (0, 0, 0): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (10, 2)}},
                (0, -1, -1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (9, 2)}},
                (0, 1, 1): {PHEROMON_DIRECTION: {PHEROMON_DIR_EXPLO: (8, 1)}}
            }
        }

        for direction_wanted in test_data:
            self._get_direction_for_point(test_data[direction_wanted], direction_wanted)<|endoftext|>


<commit_before>import time
import requests
from dateutil import parser

from django.contrib.gis.geos import Point
from django.core.management.base import BaseCommand

from inspections.models import Establishment, Inspection, Violation

class Command(BaseCommand):
    """Import saniation data from Durham County API"""
    restaurants_url = 'http://data.wake.opendata.arcgis.com/datasets/124c2187da8c41c59bde04fa67eb2872_0.geojson?where=&geometry={"xmin":-8992487.871106919,"ymin":4226590.155355151,"xmax":-8508182.859892275,"ymax":4318314.589297318,"spatialReference":{"wkid":102100}}'
    inspections_url = 'http://data.wake.opendata.arcgis.com/datasets/ebe3ae7f76954fad81411612d7c4fb17_1.geojson'
    violations_url = 'http://data.wake.opendata.arcgis.com/datasets/9b04d0c39abd4e049cbd4656a0a04ba3_2.geojson'

    def handle(self, *args, **options):
        # restaurants
        restaurants = self.get_county_data(self.restaurants_url)
        self.save_restaurants(restaurants)
        # inspections
        inspections = self.get_county_data(self.inspections_url)
        self.save_inspections(inspections)
        # violations
        violations = self.get_county_data(self.violations_url)
        self.save_violations(violations)

    def get_county_data(self, url):
        resp = requests.get(url)
        json_resp = resp.json()
        if 'features' in json_resp:
            return json_resp['features']
        time.sleep(30)  # wake queues up the response
        return self.get_county_data(url)

    def save_restaurants(self, restaurants):
        for restaurant in restaurants:
            properties = restaurant['properties']
            open_date = properties['RestaurantOpenDate']
            attributes = {
                'external_id': properties['OBJECTID'],
                'state_id': properties['HSISID'],
                'name': properties['Name'],
                # 'type': TODO: figure out a mapping that makes sense
                'address': '{0} {1}'.format(properties['Address1'], properties['Address2']).rstrip(),
                'city': properties['City'],
                'county': 'Wake',
                'state': 'NC',
                'postal_code': properties['PostalCode'],
                'opening_date': parser.parse(open_date) if open_date else None,
                'location': Point(float(properties['X']), float(properties['Y']))
            }
            try:
                restaurant_obj = Establishment.objects.get(state_id=properties['HSISID'])
                print('Already in db')
            except Establishment.DoesNotExist:
                restaurant_obj = Establishment()
                print('New record')
            restaurant_obj.__dict__.update(**attributes)
            restaurant_obj.save()

    def save_inspections(self, inspections):
        for inspection in inspections:
            properties = inspection['properties']
            try:
                establishment = Establishment.objects.get(state_id=int(properties['HSISID']))
            except:
                print('No Establishment with HSISID #' + properties['HSISID'])
                continue
            insp_date = properties['Date']
            attributes = {
                'establishment_id': establishment.id,
                'external_id': properties['OBJECTID'],
                'date': parser.parse(insp_date) if insp_date else None,
                'score': properties['Score'],
                'description': properties['Description'],
                # 'type': TODO: figure out a mapping that makes sense
            }
            try:
                inspection_obj = Inspection.objects.get(establishment_id=establishment.id,
                                                        date=insp_date)
                print('Already in db')
            except Inspection.DoesNotExist:
                inspection_obj = Inspection()
                print('New record')
            inspection_obj.__dict__.update(**attributes)
            inspection_obj.save()

    def save_violations(self, violations):
        for violation in violations:
            properties = violation['properties']
            inspection_date = parser.parse(properties['InspectDate'])
            try:
                establishment = Establishment.objects.get(state_id=int(properties['HSISID']))
            except Establishment.DoesNotExist:
                print('No Establishment with HSISID #' + properties['HSISID'])
                continue
            except TypeError:
                if not properties['HSISID']:
                    print('Object #' + str(properties['OBJECTID']) + ' is NoneType')

                else:
                    print('Object #' + str(properties['OBJECTID']) + ' is  type' + str(type(properties['HSISID'])))
                continue
            try:
                inspection = Inspection.objects.get(establishment_id=establishment.id,
                                                    date=inspection_date)
            except Inspection.DoesNotExist:
                print('No Inspection for HSISID #' + properties['HSISID'] +
                      ' with date ' + str(inspection_date))
                continue
            attributes = {
                'establishment_id': establishment.id,
                'inspection_id': inspection.id,
                'external_id': properties['OBJECTID'],
                'date': inspection_date,
                'code': properties['ViolationCode'],
                'description': properties['shortdesc'],
            }
            try:
                violation_obj = Violation.objects.get(establishment_id=establishment.id,
                                                      date=inspection_date,
                                                      code=properties['ViolationCode'])
                print('ObjectID: ' + str(properties['OBJECTID']) + ' already in db')
            except Violation.DoesNotExist:
                violation_obj = Violation()
                print('New record')
            violation_obj.__dict__.update(**attributes)
            violation_obj.save()
<commit_msg>mapping establishment types<commit_after>import time
import requests
from dateutil import parser

from django.contrib.gis.geos import Point
from django.core.management.base import BaseCommand

from inspections.models import Establishment, Inspection, Violation

class Command(BaseCommand):
    """Import saniation data from Durham County API"""
    restaurants_url = 'http://data.wake.opendata.arcgis.com/datasets/124c2187da8c41c59bde04fa67eb2872_0.geojson?where=&geometry={"xmin":-8992487.871106919,"ymin":4226590.155355151,"xmax":-8508182.859892275,"ymax":4318314.589297318,"spatialReference":{"wkid":102100}}'
    inspections_url = 'http://data.wake.opendata.arcgis.com/datasets/ebe3ae7f76954fad81411612d7c4fb17_1.geojson'
    violations_url = 'http://data.wake.opendata.arcgis.com/datasets/9b04d0c39abd4e049cbd4656a0a04ba3_2.geojson'

    def handle(self, *args, **options):
        # restaurants
        restaurants = self.get_county_data(self.restaurants_url)
        self.save_restaurants(restaurants)
        # inspections
        inspections = self.get_county_data(self.inspections_url)
        self.save_inspections(inspections)
        # violations
        violations = self.get_county_data(self.violations_url)
        self.save_violations(violations)

    def get_county_data(self, url):
        resp = requests.get(url)
        json_resp = resp.json()
        if 'features' in json_resp:
            return json_resp['features']
        time.sleep(30)  # wake queues up the response
        return self.get_county_data(url)

    def get_establishment_type(self, wake_type):
        wake_type_dict = {
            'Meat Market': 30,
            'Private School Lunchrooms': 5,
            'Mobile Food Units': 3,
            'Limited Food Service': 14,
            'Restaurant': 1,
            'Elderly Nutrition Sites (catered)': 9,
            'Institutional Food Service': 16,
            'Public School Lunchrooms': 11,
            'Pushcarts': 4,
            'Food Stand': 2
            }
        try:
            return wake_type_dict[wake_type]
        except KeyError:
            return 0

    def save_restaurants(self, restaurants):
        for restaurant in restaurants:
            properties = restaurant['properties']
            open_date = properties['RestaurantOpenDate']
            attributes = {
                'external_id': properties['OBJECTID'],
                'state_id': properties['HSISID'],
                'name': properties['Name'],
                'type': self.get_establishment_type(properties['FacilityType']),
                'address': '{0} {1}'.format(properties['Address1'], properties['Address2']).rstrip(),
                'city': properties['City'],
                'county': 'Wake',
                'state': 'NC',
                'postal_code': properties['PostalCode'],
                'opening_date': parser.parse(open_date) if open_date else None,
                'location': Point(float(properties['X']), float(properties['Y']))
            }
            try:
                restaurant_obj = Establishment.objects.get(state_id=properties['HSISID'])
                print('Already in db')
            except Establishment.DoesNotExist:
                restaurant_obj = Establishment()
                print('New record')
            restaurant_obj.__dict__.update(**attributes)
            restaurant_obj.save()

    def save_inspections(self, inspections):
        for inspection in inspections:
            properties = inspection['properties']
            try:
                establishment = Establishment.objects.get(state_id=int(properties['HSISID']))
            except:
                print('No Establishment with HSISID #' + properties['HSISID'])
                continue
            insp_date = properties['Date']
            attributes = {
                'establishment_id': establishment.id,
                'external_id': properties['OBJECTID'],
                'date': parser.parse(insp_date) if insp_date else None,
                'score': properties['Score'],
                'description': properties['Description'],
                # 'type': TODO: figure out a mapping that makes sense
            }
            try:
                inspection_obj = Inspection.objects.get(establishment_id=establishment.id,
                                                        date=insp_date)
                print('Already in db')
            except Inspection.DoesNotExist:
                inspection_obj = Inspection()
                print('New record')
            inspection_obj.__dict__.update(**attributes)
            inspection_obj.save()

    def save_violations(self, violations):
        for violation in violations:
            properties = violation['properties']
            inspection_date = parser.parse(properties['InspectDate'])
            try:
                establishment = Establishment.objects.get(state_id=int(properties['HSISID']))
            except Establishment.DoesNotExist:
                print('No Establishment with HSISID #' + properties['HSISID'])
                continue
            except TypeError:
                if not properties['HSISID']:
                    print('Object #' + str(properties['OBJECTID']) + ' is NoneType')

                else:
                    print('Object #' + str(properties['OBJECTID']) + ' is  type' + str(type(properties['HSISID'])))
                continue
            try:
                inspection = Inspection.objects.get(establishment_id=establishment.id,
                                                    date=inspection_date)
            except Inspection.DoesNotExist:
                print('No Inspection for HSISID #' + properties['HSISID'] +
                      ' with date ' + str(inspection_date))
                continue
            attributes = {
                'establishment_id': establishment.id,
                'inspection_id': inspection.id,
                'external_id': properties['OBJECTID'],
                'date': inspection_date,
                'code': properties['ViolationCode'],
                'description': properties['shortdesc'],
            }
            try:
                violation_obj = Violation.objects.get(establishment_id=establishment.id,
                                                      date=inspection_date,
                                                      code=properties['ViolationCode'])
                print('ObjectID: ' + str(properties['OBJECTID']) + ' already in db')
            except Violation.DoesNotExist:
                violation_obj = Violation()
                print('New record')
            violation_obj.__dict__.update(**attributes)
            violation_obj.save()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutArrayAssignments in the Ruby Koans
#

from runner.koan import *

class AboutListAssignments(Koan):
    def test_non_parallel_assignment(self):
        names = ["John", "Smith"]
        self.assertEqual(__, names)

    def test_parallel_assignments(self):
        first_name, last_name = ["John", "Smith"]
        self.assertEqual(__, first_name)
        self.assertEqual(__, last_name)

    def test_parallel_assignments_with_extra_values(self):
        title, *first_names, last_name = ["Sir", "Ricky", "Bobby", "Worthington"]
        self.assertEqual(__, title)
        self.assertEqual(__, first_names)
        self.assertEqual(__, last_name)

    def test_parallel_assignments_with_sublists(self):
        first_name, last_name = [["Willie", "Rae"], "Johnson"]
        self.assertEqual(__, first_name)
        self.assertEqual(__, last_name)

    def test_swapping_with_parallel_assignment(self):
        first_name = "Roy"
        last_name = "Rob"
        first_name, last_name = last_name, first_name
        self.assertEqual(__, first_name)
        self.assertEqual(__, last_name)

<commit_msg>About list assginment<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutArrayAssignments in the Ruby Koans
#

from runner.koan import *

class AboutListAssignments(Koan):
    def test_non_parallel_assignment(self):
        names = ["John", "Smith"]
        self.assertEqual(["John", "Smith"], names)

    def test_parallel_assignments(self):
        first_name, last_name = ["John", "Smith"]
        self.assertEqual("John", first_name)
        self.assertEqual("Smith", last_name)

    def test_parallel_assignments_with_extra_values(self):
        title, *first_names, last_name = ["Sir", "Ricky", "Bobby", "Worthington"]
        self.assertEqual("Sir", title)
        self.assertEqual(["Ricky", "Bobby"], first_names)
        self.assertEqual("Worthington", last_name)

    def test_parallel_assignments_with_sublists(self):
        first_name, last_name = [["Willie", "Rae"], "Johnson"]
        self.assertEqual(["Willie", "Rae"], first_name)
        self.assertEqual("Johnson", last_name)

    def test_swapping_with_parallel_assignment(self):
        first_name = "Roy"
        last_name = "Rob"
        first_name, last_name = last_name, first_name
        self.assertEqual("Rob", first_name)
        self.assertEqual("Roy", last_name)

<|endoftext|>


<commit_before>"""
Some quick and dirty tests for a very small subset of the code.
"""

import pytest

from xenserver.models import XenVM
from xenserver import tasks
from xenserver.tests.helpers import XenServerHelper
from xenserver.tests.matchers import (
    ExpectedXenServerVM, ExpectedXenServerVIF, ExtractValues, MatchSorted)

@pytest.fixture
def xs_helper(monkeypatch):
    """
    Provide a XenServerHelper instance and monkey-patch xenserver.tasks to use
    sessions from that instance instead of making real API calls.
    """
    xshelper = XenServerHelper()
    monkeypatch.setattr(tasks, 'getSession', xshelper.get_session)
    return xshelper

class TaskCatcher(object):
    def __init__(self, monkeypatch):
        self.mp = monkeypatch

    def catch_async(self, task, f=lambda *a: a):
        """
        Return a list that will be populated with the args of any call to the
        given task.
        """
        calls = []
        self.mp.setattr(task, 'apply_async', lambda *a: calls.append(f(*a)))
        return calls

    def catch_updateServer(self):
        """
        Special case of catch_async for updateServer.
        """
        return self.catch_async(
            tasks.updateServer, lambda args, kwargs: args[0].hostname)

def updateServer_argf(args, kwargs):
    return args[0].hostname

@pytest.fixture
def task_catcher(monkeypatch):
    """
    Monkey-patch the given task's apply_async() method to add calls to a
    list instead of queueing the task.
    """
    return TaskCatcher(monkeypatch)

@pytest.mark.django_db
class TestCreateVM(object):
    """
    Test xenserver.tasks.create_vm task.
    """

    def setup_vm(self, name, template, status="Provisioning", **kw):
        params = {
            "sockets": template.cores,
            "memory": template.memory,
        }
        params.update(kw)
        return XenVM.objects.create(
            name="foovm", status="Provisioning", template=template, **params)

    def extract_VIFs(self, xenserver, VM_ref, spec):
        """
        Get the VIFs for the given VM and match them to a list of (network,
        VIF) ref pairs.
        """
        assert MatchSorted(spec) == xenserver.list_network_VIFs_for_VM(VM_ref)

    def extract_VBDs(self, xenserver, VM_ref, spec):
        """
        Get the VBDs for the given VM and match them to a list of (SR, VBD)
        ref pairs.
        """
        assert MatchSorted(spec) == xenserver.list_SR_VBDs_for_VM(VM_ref)

    def expected_vm(self, template, local_SR, VIFs, VBDs, **kw):
        """
        Build an ExpectedXenServerVM object with some default parameters.
        """
        params = {
            "PV_args": " -- quiet console=hvc0",
            "name_label": "None.None",
            "VCPUs_max": "1",
            "VCPUs_at_startup": "1",
            "memory_static_max": str(template.memory*1024*1024),
            "memory_dynamic_max": str(template.memory*1024*1024),
            "suspend_SR": local_SR,
        }
        return ExpectedXenServerVM(
            VIFs=MatchSorted(VIFs), VBDs=MatchSorted(VBDs), **params)

    xapi_versions = pytest.mark.parametrize('xapi_version', [(1, 1), (1, 2)])

    @xapi_versions
    def test_create_vm_simple(self, xapi_version, xs_helper):
        """
        We can create a new VM using mostly default values.
        """
        xsh, xs = xs_helper.new_host('xenserver01.local', xapi_version)
        template = xs_helper.db_template("footempl")
        vm = self.setup_vm("foovm", template)

        assert vm.xsref == ''
        assert xsh.api.VMs == {}

        tasks.create_vm.apply(
            [vm, xs, template, None, None, None, None, None, None],
            {'extra_network_bridges': []})

        vm.refresh_from_db()
        assert vm.xsref != ''

        # Make sure the right VIFs and VBDs were created and extract their
        # reference values.
        ev = ExtractValues("VIF", "iso_VBD", "local_VBD")
        self.extract_VIFs(xsh.api, vm.xsref, [(xsh.net['eth0'], ev.VIF)])
        self.extract_VBDs(xsh.api, vm.xsref, [
            (xsh.sr['iso'], ev.iso_VBD),
            (xsh.sr['local'], ev.local_VBD),
        ])

        # The VM data structure should match the values we passed to
        # create_vm().
        assert xsh.api.VMs.keys() == [vm.xsref]
        assert self.expected_vm(
            template, xsh.sr['local'], VIFs=[ev.VIF],
            VBDs=[ev.iso_VBD, ev.local_VBD],
        ) == xsh.api.VMs[vm.xsref]

        # The VIF data structures should match the values we passed to
        # create_vm().
        assert xsh.api.VIFs.keys() == [ev.VIF.value]
        assert ExpectedXenServerVIF(
            device="0", VM=vm.xsref, network=xsh.net['eth0'],
        ) == xsh.api.VIFs[ev.VIF.value]

        # The VM should be started.
        assert xsh.api.VM_operations == [(vm.xsref, "start")]

    @xapi_versions
    def test_create_vm_second_vif(self, xapi_version, xs_helper):
        """
        We can create a new VM with a second VIF.
        """
        xsh, xs = xs_helper.new_host('xenserver01.local', xapi_version)
        template = xs_helper.db_template("footempl")
        vm = self.setup_vm("foovm", template)

        assert vm.xsref == ''
        assert xsh.api.VMs == {}

        tasks.create_vm.apply(
            [vm, xs, template, None, None, None, None, None, None],
            {'extra_network_bridges': ['xenbr1']})

        vm.refresh_from_db()
        assert vm.xsref != ''

        # Make sure the right VIFs and VBDs were created and extract their
        # reference values.
        ev = ExtractValues("pub_VIF", "prv_VIF", "iso_VBD", "local_VBD")
        self.extract_VIFs(xsh.api, vm.xsref, [
            (xsh.net['eth0'], ev.pub_VIF),
            (xsh.net['eth1'], ev.prv_VIF),
        ])
        self.extract_VBDs(xsh.api, vm.xsref, [
            (xsh.sr['iso'], ev.iso_VBD),
            (xsh.sr['local'], ev.local_VBD),
        ])

        # The VM data structure should match the values we passed to
        # create_vm().
        assert xsh.api.VMs.keys() == [vm.xsref]
        assert self.expected_vm(
            template, xsh.sr['local'], VIFs=[ev.pub_VIF, ev.prv_VIF],
            VBDs=[ev.iso_VBD, ev.local_VBD],
        ) == xsh.api.VMs[vm.xsref]

        # The VIF data structures should match the values we passed to
        # create_vm().
        assert MatchSorted([ev.pub_VIF, ev.prv_VIF]) == xsh.api.VIFs.keys()
        assert ExpectedXenServerVIF(
            device="0", VM=vm.xsref, network=xsh.net['eth0'],
        ) == xsh.api.VIFs[ev.pub_VIF.value]
        assert ExpectedXenServerVIF(
            device="1", VM=vm.xsref, network=xsh.net['eth1'],
        ) == xsh.api.VIFs[ev.prv_VIF.value]

        # The VM should be started.
        assert xsh.api.VM_operations == [(vm.xsref, "start")]

@pytest.mark.django_db
class TestUpdateVms(object):
    """
    Test xenserver.tasks.updateVms task.
    """

    def test_no_servers(self, xs_helper, task_catcher):
        """
        Nothing to do if we have no servers.
        """
        us_calls = task_catcher.catch_updateServer()
        tasks.updateVms.apply()
        assert us_calls == []

    def test_one_server(self, xs_helper, task_catcher):
        """
        A single server will be updated.
        """
        xs_helper.new_host('xs01.local')
        us_calls = task_catcher.catch_updateServer()
        tasks.updateVms.apply()
        assert us_calls == ['xs01.local']

    def test_three_servers(self, xs_helper, task_catcher):
        """
        Multiple servers will be updated.
        """
        xs_helper.new_host('xs01.local')
        xs_helper.new_host('xs02.local')
        xs_helper.new_host('xs03.local')
        us_calls = task_catcher.catch_updateServer()
        tasks.updateVms.apply()
        assert sorted(us_calls) == ['xs01.local', 'xs02.local', 'xs03.local']
<commit_msg>Remove unused function.<commit_after>"""
Some quick and dirty tests for a very small subset of the code.
"""

import pytest

from xenserver.models import XenVM
from xenserver import tasks
from xenserver.tests.helpers import XenServerHelper
from xenserver.tests.matchers import (
    ExpectedXenServerVM, ExpectedXenServerVIF, ExtractValues, MatchSorted)

@pytest.fixture
def xs_helper(monkeypatch):
    """
    Provide a XenServerHelper instance and monkey-patch xenserver.tasks to use
    sessions from that instance instead of making real API calls.
    """
    xshelper = XenServerHelper()
    monkeypatch.setattr(tasks, 'getSession', xshelper.get_session)
    return xshelper

class TaskCatcher(object):
    def __init__(self, monkeypatch):
        self.mp = monkeypatch

    def catch_async(self, task, f=lambda *a: a):
        """
        Return a list that will be populated with the args of any call to the
        given task.
        """
        calls = []
        self.mp.setattr(task, 'apply_async', lambda *a: calls.append(f(*a)))
        return calls

    def catch_updateServer(self):
        """
        Special case of catch_async for updateServer.
        """
        return self.catch_async(
            tasks.updateServer, lambda args, kwargs: args[0].hostname)

@pytest.fixture
def task_catcher(monkeypatch):
    """
    Monkey-patch the given task's apply_async() method to add calls to a
    list instead of queueing the task.
    """
    return TaskCatcher(monkeypatch)

@pytest.mark.django_db
class TestCreateVM(object):
    """
    Test xenserver.tasks.create_vm task.
    """

    def setup_vm(self, name, template, status="Provisioning", **kw):
        params = {
            "sockets": template.cores,
            "memory": template.memory,
        }
        params.update(kw)
        return XenVM.objects.create(
            name="foovm", status="Provisioning", template=template, **params)

    def extract_VIFs(self, xenserver, VM_ref, spec):
        """
        Get the VIFs for the given VM and match them to a list of (network,
        VIF) ref pairs.
        """
        assert MatchSorted(spec) == xenserver.list_network_VIFs_for_VM(VM_ref)

    def extract_VBDs(self, xenserver, VM_ref, spec):
        """
        Get the VBDs for the given VM and match them to a list of (SR, VBD)
        ref pairs.
        """
        assert MatchSorted(spec) == xenserver.list_SR_VBDs_for_VM(VM_ref)

    def expected_vm(self, template, local_SR, VIFs, VBDs, **kw):
        """
        Build an ExpectedXenServerVM object with some default parameters.
        """
        params = {
            "PV_args": " -- quiet console=hvc0",
            "name_label": "None.None",
            "VCPUs_max": "1",
            "VCPUs_at_startup": "1",
            "memory_static_max": str(template.memory*1024*1024),
            "memory_dynamic_max": str(template.memory*1024*1024),
            "suspend_SR": local_SR,
        }
        return ExpectedXenServerVM(
            VIFs=MatchSorted(VIFs), VBDs=MatchSorted(VBDs), **params)

    xapi_versions = pytest.mark.parametrize('xapi_version', [(1, 1), (1, 2)])

    @xapi_versions
    def test_create_vm_simple(self, xapi_version, xs_helper):
        """
        We can create a new VM using mostly default values.
        """
        xsh, xs = xs_helper.new_host('xenserver01.local', xapi_version)
        template = xs_helper.db_template("footempl")
        vm = self.setup_vm("foovm", template)

        assert vm.xsref == ''
        assert xsh.api.VMs == {}

        tasks.create_vm.apply(
            [vm, xs, template, None, None, None, None, None, None],
            {'extra_network_bridges': []})

        vm.refresh_from_db()
        assert vm.xsref != ''

        # Make sure the right VIFs and VBDs were created and extract their
        # reference values.
        ev = ExtractValues("VIF", "iso_VBD", "local_VBD")
        self.extract_VIFs(xsh.api, vm.xsref, [(xsh.net['eth0'], ev.VIF)])
        self.extract_VBDs(xsh.api, vm.xsref, [
            (xsh.sr['iso'], ev.iso_VBD),
            (xsh.sr['local'], ev.local_VBD),
        ])

        # The VM data structure should match the values we passed to
        # create_vm().
        assert xsh.api.VMs.keys() == [vm.xsref]
        assert self.expected_vm(
            template, xsh.sr['local'], VIFs=[ev.VIF],
            VBDs=[ev.iso_VBD, ev.local_VBD],
        ) == xsh.api.VMs[vm.xsref]

        # The VIF data structures should match the values we passed to
        # create_vm().
        assert xsh.api.VIFs.keys() == [ev.VIF.value]
        assert ExpectedXenServerVIF(
            device="0", VM=vm.xsref, network=xsh.net['eth0'],
        ) == xsh.api.VIFs[ev.VIF.value]

        # The VM should be started.
        assert xsh.api.VM_operations == [(vm.xsref, "start")]

    @xapi_versions
    def test_create_vm_second_vif(self, xapi_version, xs_helper):
        """
        We can create a new VM with a second VIF.
        """
        xsh, xs = xs_helper.new_host('xenserver01.local', xapi_version)
        template = xs_helper.db_template("footempl")
        vm = self.setup_vm("foovm", template)

        assert vm.xsref == ''
        assert xsh.api.VMs == {}

        tasks.create_vm.apply(
            [vm, xs, template, None, None, None, None, None, None],
            {'extra_network_bridges': ['xenbr1']})

        vm.refresh_from_db()
        assert vm.xsref != ''

        # Make sure the right VIFs and VBDs were created and extract their
        # reference values.
        ev = ExtractValues("pub_VIF", "prv_VIF", "iso_VBD", "local_VBD")
        self.extract_VIFs(xsh.api, vm.xsref, [
            (xsh.net['eth0'], ev.pub_VIF),
            (xsh.net['eth1'], ev.prv_VIF),
        ])
        self.extract_VBDs(xsh.api, vm.xsref, [
            (xsh.sr['iso'], ev.iso_VBD),
            (xsh.sr['local'], ev.local_VBD),
        ])

        # The VM data structure should match the values we passed to
        # create_vm().
        assert xsh.api.VMs.keys() == [vm.xsref]
        assert self.expected_vm(
            template, xsh.sr['local'], VIFs=[ev.pub_VIF, ev.prv_VIF],
            VBDs=[ev.iso_VBD, ev.local_VBD],
        ) == xsh.api.VMs[vm.xsref]

        # The VIF data structures should match the values we passed to
        # create_vm().
        assert MatchSorted([ev.pub_VIF, ev.prv_VIF]) == xsh.api.VIFs.keys()
        assert ExpectedXenServerVIF(
            device="0", VM=vm.xsref, network=xsh.net['eth0'],
        ) == xsh.api.VIFs[ev.pub_VIF.value]
        assert ExpectedXenServerVIF(
            device="1", VM=vm.xsref, network=xsh.net['eth1'],
        ) == xsh.api.VIFs[ev.prv_VIF.value]

        # The VM should be started.
        assert xsh.api.VM_operations == [(vm.xsref, "start")]

@pytest.mark.django_db
class TestUpdateVms(object):
    """
    Test xenserver.tasks.updateVms task.
    """

    def test_no_servers(self, xs_helper, task_catcher):
        """
        Nothing to do if we have no servers.
        """
        us_calls = task_catcher.catch_updateServer()
        tasks.updateVms.apply()
        assert us_calls == []

    def test_one_server(self, xs_helper, task_catcher):
        """
        A single server will be updated.
        """
        xs_helper.new_host('xs01.local')
        us_calls = task_catcher.catch_updateServer()
        tasks.updateVms.apply()
        assert us_calls == ['xs01.local']

    def test_three_servers(self, xs_helper, task_catcher):
        """
        Multiple servers will be updated.
        """
        xs_helper.new_host('xs01.local')
        xs_helper.new_host('xs02.local')
        xs_helper.new_host('xs03.local')
        us_calls = task_catcher.catch_updateServer()
        tasks.updateVms.apply()
        assert sorted(us_calls) == ['xs01.local', 'xs02.local', 'xs03.local']
<|endoftext|>


<commit_before>from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.core.checks import Error
from django.core.exceptions import FieldDoesNotExist, FieldError
from django.db import models
from django.test import SimpleTestCase
from django.test.utils import isolate_apps

@isolate_apps('model_inheritance')
class AbstractInheritanceTests(SimpleTestCase):
    def test_single_parent(self):
        class AbstractBase(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class AbstractDescendant(AbstractBase):
            name = models.CharField(max_length=50)

            class Meta:
                abstract = True

        class DerivedChild(AbstractBase):
            name = models.CharField(max_length=50)

        class DerivedGrandChild(AbstractDescendant):
            pass

        self.assertEqual(AbstractDescendant._meta.get_field('name').max_length, 50)
        self.assertEqual(DerivedChild._meta.get_field('name').max_length, 50)
        self.assertEqual(DerivedGrandChild._meta.get_field('name').max_length, 50)

    def test_multiple_parents_mro(self):
        class AbstractBaseOne(models.Model):
            class Meta:
                abstract = True

        class AbstractBaseTwo(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class DescendantOne(AbstractBaseOne, AbstractBaseTwo):
            class Meta:
                abstract = True

        class DescendantTwo(AbstractBaseOne, AbstractBaseTwo):
            name = models.CharField(max_length=50)

            class Meta:
                abstract = True

        class Derived(DescendantOne, DescendantTwo):
            pass

        self.assertEqual(DescendantOne._meta.get_field('name').max_length, 30)
        self.assertEqual(DescendantTwo._meta.get_field('name').max_length, 50)
        self.assertEqual(Derived._meta.get_field('name').max_length, 50)

    def test_multiple_inheritance_cannot_shadow_concrete_inherited_field(self):
        class ConcreteParent(models.Model):
            name = models.CharField(max_length=255)

        class AbstractParent(models.Model):
            name = models.IntegerField()

            class Meta:
                abstract = True

        class FirstChild(ConcreteParent, AbstractParent):
            pass

        class AnotherChild(AbstractParent, ConcreteParent):
            pass

        self.assertIsInstance(FirstChild._meta.get_field('name'), models.CharField)
        self.assertEqual(
            AnotherChild.check(),
            [Error(
                "The field 'name' clashes with the field 'name' "
                "from model 'model_inheritance.concreteparent'.",
                obj=AnotherChild._meta.get_field('name'),
                id="models.E006",
            )]
        )

    def test_virtual_field(self):
        class RelationModel(models.Model):
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey('content_type', 'object_id')

        class RelatedModelAbstract(models.Model):
            field = GenericRelation(RelationModel)

            class Meta:
                abstract = True

        class ModelAbstract(models.Model):
            field = models.CharField(max_length=100)

            class Meta:
                abstract = True

        class OverrideRelatedModelAbstract(RelatedModelAbstract):
            field = models.CharField(max_length=100)

        class ExtendModelAbstract(ModelAbstract):
            field = GenericRelation(RelationModel)

        self.assertIsInstance(OverrideRelatedModelAbstract._meta.get_field('field'), models.CharField)
        self.assertIsInstance(ExtendModelAbstract._meta.get_field('field'), GenericRelation)

    def test_cannot_override_indirect_abstract_field(self):
        class AbstractBase(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class ConcreteDescendant(AbstractBase):
            pass

        msg = (
            "Local field 'name' in class 'Descendant' clashes with field of "
            "the same name from base class 'ConcreteDescendant'."
        )
        with self.assertRaisesMessage(FieldError, msg):
            class Descendant(ConcreteDescendant):
                name = models.IntegerField()

    def test_override_field_with_attr(self):
        class AbstractBase(models.Model):
            first_name = models.CharField(max_length=50)
            last_name = models.CharField(max_length=50)
            middle_name = models.CharField(max_length=30)
            full_name = models.CharField(max_length=150)

            class Meta:
                abstract = True

        class Descendant(AbstractBase):
            middle_name = None

            def full_name(self):
                return self.first_name + self.last_name

        msg = "Descendant has no field named %r"
        with self.assertRaisesMessage(FieldDoesNotExist, msg % 'middle_name'):
            Descendant._meta.get_field('middle_name')

        with self.assertRaisesMessage(FieldDoesNotExist, msg % 'full_name'):
            Descendant._meta.get_field('full_name')

    def test_overriding_field_removed_by_concrete_model(self):
        class AbstractModel(models.Model):
            foo = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class RemovedAbstractModelField(AbstractModel):
            foo = None

        class OverrideRemovedFieldByConcreteModel(RemovedAbstractModelField):
            foo = models.CharField(max_length=50)

        self.assertEqual(OverrideRemovedFieldByConcreteModel._meta.get_field('foo').max_length, 50)

    def test_shadowed_fkey_id(self):
        class Foo(models.Model):
            pass

        class AbstractBase(models.Model):
            foo = models.ForeignKey(Foo, models.CASCADE)

            class Meta:
                abstract = True

        class Descendant(AbstractBase):
            foo_id = models.IntegerField()

        self.assertEqual(
            Descendant.check(),
            [Error(
                "The field 'foo_id' clashes with the field 'foo' "
                "from model 'model_inheritance.descendant'.",
                obj=Descendant._meta.get_field('foo_id'),
                id='models.E006',
            )]
        )

    def test_shadow_related_name_when_set_to_none(self):
        class AbstractBase(models.Model):
            bar = models.IntegerField()

            class Meta:
                abstract = True

        class Foo(AbstractBase):
            bar = None
            foo = models.IntegerField()

        class Bar(models.Model):
            bar = models.ForeignKey(Foo, models.CASCADE, related_name='bar')

        self.assertEqual(Bar.check(), [])

    def test_reverse_foreign_key(self):
        class AbstractBase(models.Model):
            foo = models.CharField(max_length=100)

            class Meta:
                abstract = True

        class Descendant(AbstractBase):
            pass

        class Foo(models.Model):
            foo = models.ForeignKey(Descendant, models.CASCADE, related_name='foo')

        self.assertEqual(
            Foo._meta.get_field('foo').check(),
            [
                Error(
                    "Reverse accessor for 'Foo.foo' clashes with field name 'Descendant.foo'.",
                    hint=(
                        "Rename field 'Descendant.foo', or add/change a related_name "
                        "argument to the definition for field 'Foo.foo'."
                    ),
                    obj=Foo._meta.get_field('foo'),
                    id='fields.E302',
                ),
                Error(
                    "Reverse query name for 'Foo.foo' clashes with field name 'Descendant.foo'.",
                    hint=(
                        "Rename field 'Descendant.foo', or add/change a related_name "
                        "argument to the definition for field 'Foo.foo'."
                    ),
                    obj=Foo._meta.get_field('foo'),
                    id='fields.E303',
                ),
            ]
        )

    def test_multi_inheritance_field_clashes(self):
        class AbstractBase(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class ConcreteBase(AbstractBase):
            pass

        class AbstractDescendant(ConcreteBase):
            class Meta:
                abstract = True

        class ConcreteDescendant(AbstractDescendant):
            name = models.CharField(max_length=100)

        self.assertEqual(
            ConcreteDescendant.check(),
            [Error(
                "The field 'name' clashes with the field 'name' from "
                "model 'model_inheritance.concretebase'.",
                obj=ConcreteDescendant._meta.get_field('name'),
                id="models.E006",
            )]
        )

    def test_override_one2one_relation_auto_field_clashes(self):
        class ConcreteParent(models.Model):
            name = models.CharField(max_length=255)

        class AbstractParent(models.Model):
            name = models.IntegerField()

            class Meta:
                abstract = True

        msg = (
            "Auto-generated field 'concreteparent_ptr' in class 'Descendant' "
            "for parent_link to base class 'ConcreteParent' clashes with "
            "declared field of the same name."
        )
        with self.assertRaisesMessage(FieldError, msg):
            class Descendant(ConcreteParent, AbstractParent):
                concreteparent_ptr = models.CharField(max_length=30)

    def test_abstract_model_with_regular_python_mixin_mro(self):
        class AbstractModel(models.Model):
            name = models.CharField(max_length=255)
            age = models.IntegerField()

            class Meta:
                abstract = True

        class Mixin:
            age = None

        class Mixin2:
            age = 2

        class DescendantMixin(Mixin):
            pass

        class ConcreteModel(models.Model):
            foo = models.IntegerField()

        class ConcreteModel2(ConcreteModel):
            age = models.SmallIntegerField()

        def fields(model):
            if not hasattr(model, '_meta'):
                return []
            return [(f.name, f.__class__) for f in model._meta.get_fields()]

        model_dict = {'__module__': 'model_inheritance'}
        model1 = type('Model1', (AbstractModel, Mixin), model_dict.copy())
        model2 = type('Model2', (Mixin2, AbstractModel), model_dict.copy())
        model3 = type('Model3', (DescendantMixin, AbstractModel), model_dict.copy())
        model4 = type('Model4', (Mixin2, Mixin, AbstractModel), model_dict.copy())
        model5 = type('Model5', (Mixin2, ConcreteModel2, Mixin, AbstractModel), model_dict.copy())

        self.assertEqual(
            fields(model1),
            [('id', models.AutoField), ('name', models.CharField), ('age', models.IntegerField)]
        )

        self.assertEqual(fields(model2), [('id', models.AutoField), ('name', models.CharField)])
        self.assertEqual(getattr(model2, 'age'), 2)

        self.assertEqual(fields(model3), [('id', models.AutoField), ('name', models.CharField)])

        self.assertEqual(fields(model4), [('id', models.AutoField), ('name', models.CharField)])
        self.assertEqual(getattr(model4, 'age'), 2)

        self.assertEqual(
            fields(model5),
            [
                ('id', models.AutoField), ('foo', models.IntegerField),
                ('concretemodel_ptr', models.OneToOneField),
                ('age', models.SmallIntegerField), ('concretemodel2_ptr', models.OneToOneField),
                ('name', models.CharField),
            ]
        )
<commit_msg>Fixed #31437 -- Corrected tests to show abstract multiple inheritance system check error.<commit_after>from django.contrib.contenttypes.fields import (
    GenericForeignKey, GenericRelation,
)
from django.contrib.contenttypes.models import ContentType
from django.core.checks import Error
from django.core.exceptions import FieldDoesNotExist, FieldError
from django.db import models
from django.test import SimpleTestCase
from django.test.utils import isolate_apps

@isolate_apps('model_inheritance')
class AbstractInheritanceTests(SimpleTestCase):
    def test_single_parent(self):
        class AbstractBase(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class AbstractDescendant(AbstractBase):
            name = models.CharField(max_length=50)

            class Meta:
                abstract = True

        class DerivedChild(AbstractBase):
            name = models.CharField(max_length=50)

        class DerivedGrandChild(AbstractDescendant):
            pass

        self.assertEqual(AbstractDescendant._meta.get_field('name').max_length, 50)
        self.assertEqual(DerivedChild._meta.get_field('name').max_length, 50)
        self.assertEqual(DerivedGrandChild._meta.get_field('name').max_length, 50)

    def test_multiple_inheritance_cannot_shadow_inherited_field(self):
        class ParentA(models.Model):
            name = models.CharField(max_length=255)

            class Meta:
                abstract = True

        class ParentB(models.Model):
            name = models.IntegerField()

            class Meta:
                abstract = True

        class Child(ParentA, ParentB):
            pass

        self.assertEqual(Child.check(), [
            Error(
                "The field 'name' clashes with the field 'name' from model "
                "'model_inheritance.child'.",
                obj=Child._meta.get_field('name'),
                id='models.E006',
            ),
        ])

    def test_multiple_inheritance_cannot_shadow_concrete_inherited_field(self):
        class ConcreteParent(models.Model):
            name = models.CharField(max_length=255)

        class AbstractParent(models.Model):
            name = models.IntegerField()

            class Meta:
                abstract = True

        class FirstChild(ConcreteParent, AbstractParent):
            pass

        class AnotherChild(AbstractParent, ConcreteParent):
            pass

        self.assertIsInstance(FirstChild._meta.get_field('name'), models.CharField)
        self.assertEqual(
            AnotherChild.check(),
            [Error(
                "The field 'name' clashes with the field 'name' "
                "from model 'model_inheritance.concreteparent'.",
                obj=AnotherChild._meta.get_field('name'),
                id="models.E006",
            )]
        )

    def test_virtual_field(self):
        class RelationModel(models.Model):
            content_type = models.ForeignKey(ContentType, models.CASCADE)
            object_id = models.PositiveIntegerField()
            content_object = GenericForeignKey('content_type', 'object_id')

        class RelatedModelAbstract(models.Model):
            field = GenericRelation(RelationModel)

            class Meta:
                abstract = True

        class ModelAbstract(models.Model):
            field = models.CharField(max_length=100)

            class Meta:
                abstract = True

        class OverrideRelatedModelAbstract(RelatedModelAbstract):
            field = models.CharField(max_length=100)

        class ExtendModelAbstract(ModelAbstract):
            field = GenericRelation(RelationModel)

        self.assertIsInstance(OverrideRelatedModelAbstract._meta.get_field('field'), models.CharField)
        self.assertIsInstance(ExtendModelAbstract._meta.get_field('field'), GenericRelation)

    def test_cannot_override_indirect_abstract_field(self):
        class AbstractBase(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class ConcreteDescendant(AbstractBase):
            pass

        msg = (
            "Local field 'name' in class 'Descendant' clashes with field of "
            "the same name from base class 'ConcreteDescendant'."
        )
        with self.assertRaisesMessage(FieldError, msg):
            class Descendant(ConcreteDescendant):
                name = models.IntegerField()

    def test_override_field_with_attr(self):
        class AbstractBase(models.Model):
            first_name = models.CharField(max_length=50)
            last_name = models.CharField(max_length=50)
            middle_name = models.CharField(max_length=30)
            full_name = models.CharField(max_length=150)

            class Meta:
                abstract = True

        class Descendant(AbstractBase):
            middle_name = None

            def full_name(self):
                return self.first_name + self.last_name

        msg = "Descendant has no field named %r"
        with self.assertRaisesMessage(FieldDoesNotExist, msg % 'middle_name'):
            Descendant._meta.get_field('middle_name')

        with self.assertRaisesMessage(FieldDoesNotExist, msg % 'full_name'):
            Descendant._meta.get_field('full_name')

    def test_overriding_field_removed_by_concrete_model(self):
        class AbstractModel(models.Model):
            foo = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class RemovedAbstractModelField(AbstractModel):
            foo = None

        class OverrideRemovedFieldByConcreteModel(RemovedAbstractModelField):
            foo = models.CharField(max_length=50)

        self.assertEqual(OverrideRemovedFieldByConcreteModel._meta.get_field('foo').max_length, 50)

    def test_shadowed_fkey_id(self):
        class Foo(models.Model):
            pass

        class AbstractBase(models.Model):
            foo = models.ForeignKey(Foo, models.CASCADE)

            class Meta:
                abstract = True

        class Descendant(AbstractBase):
            foo_id = models.IntegerField()

        self.assertEqual(
            Descendant.check(),
            [Error(
                "The field 'foo_id' clashes with the field 'foo' "
                "from model 'model_inheritance.descendant'.",
                obj=Descendant._meta.get_field('foo_id'),
                id='models.E006',
            )]
        )

    def test_shadow_related_name_when_set_to_none(self):
        class AbstractBase(models.Model):
            bar = models.IntegerField()

            class Meta:
                abstract = True

        class Foo(AbstractBase):
            bar = None
            foo = models.IntegerField()

        class Bar(models.Model):
            bar = models.ForeignKey(Foo, models.CASCADE, related_name='bar')

        self.assertEqual(Bar.check(), [])

    def test_reverse_foreign_key(self):
        class AbstractBase(models.Model):
            foo = models.CharField(max_length=100)

            class Meta:
                abstract = True

        class Descendant(AbstractBase):
            pass

        class Foo(models.Model):
            foo = models.ForeignKey(Descendant, models.CASCADE, related_name='foo')

        self.assertEqual(
            Foo._meta.get_field('foo').check(),
            [
                Error(
                    "Reverse accessor for 'Foo.foo' clashes with field name 'Descendant.foo'.",
                    hint=(
                        "Rename field 'Descendant.foo', or add/change a related_name "
                        "argument to the definition for field 'Foo.foo'."
                    ),
                    obj=Foo._meta.get_field('foo'),
                    id='fields.E302',
                ),
                Error(
                    "Reverse query name for 'Foo.foo' clashes with field name 'Descendant.foo'.",
                    hint=(
                        "Rename field 'Descendant.foo', or add/change a related_name "
                        "argument to the definition for field 'Foo.foo'."
                    ),
                    obj=Foo._meta.get_field('foo'),
                    id='fields.E303',
                ),
            ]
        )

    def test_multi_inheritance_field_clashes(self):
        class AbstractBase(models.Model):
            name = models.CharField(max_length=30)

            class Meta:
                abstract = True

        class ConcreteBase(AbstractBase):
            pass

        class AbstractDescendant(ConcreteBase):
            class Meta:
                abstract = True

        class ConcreteDescendant(AbstractDescendant):
            name = models.CharField(max_length=100)

        self.assertEqual(
            ConcreteDescendant.check(),
            [Error(
                "The field 'name' clashes with the field 'name' from "
                "model 'model_inheritance.concretebase'.",
                obj=ConcreteDescendant._meta.get_field('name'),
                id="models.E006",
            )]
        )

    def test_override_one2one_relation_auto_field_clashes(self):
        class ConcreteParent(models.Model):
            name = models.CharField(max_length=255)

        class AbstractParent(models.Model):
            name = models.IntegerField()

            class Meta:
                abstract = True

        msg = (
            "Auto-generated field 'concreteparent_ptr' in class 'Descendant' "
            "for parent_link to base class 'ConcreteParent' clashes with "
            "declared field of the same name."
        )
        with self.assertRaisesMessage(FieldError, msg):
            class Descendant(ConcreteParent, AbstractParent):
                concreteparent_ptr = models.CharField(max_length=30)

    def test_abstract_model_with_regular_python_mixin_mro(self):
        class AbstractModel(models.Model):
            name = models.CharField(max_length=255)
            age = models.IntegerField()

            class Meta:
                abstract = True

        class Mixin:
            age = None

        class Mixin2:
            age = 2

        class DescendantMixin(Mixin):
            pass

        class ConcreteModel(models.Model):
            foo = models.IntegerField()

        class ConcreteModel2(ConcreteModel):
            age = models.SmallIntegerField()

        def fields(model):
            if not hasattr(model, '_meta'):
                return []
            return [(f.name, f.__class__) for f in model._meta.get_fields()]

        model_dict = {'__module__': 'model_inheritance'}
        model1 = type('Model1', (AbstractModel, Mixin), model_dict.copy())
        model2 = type('Model2', (Mixin2, AbstractModel), model_dict.copy())
        model3 = type('Model3', (DescendantMixin, AbstractModel), model_dict.copy())
        model4 = type('Model4', (Mixin2, Mixin, AbstractModel), model_dict.copy())
        model5 = type('Model5', (Mixin2, ConcreteModel2, Mixin, AbstractModel), model_dict.copy())

        self.assertEqual(
            fields(model1),
            [('id', models.AutoField), ('name', models.CharField), ('age', models.IntegerField)]
        )

        self.assertEqual(fields(model2), [('id', models.AutoField), ('name', models.CharField)])
        self.assertEqual(getattr(model2, 'age'), 2)

        self.assertEqual(fields(model3), [('id', models.AutoField), ('name', models.CharField)])

        self.assertEqual(fields(model4), [('id', models.AutoField), ('name', models.CharField)])
        self.assertEqual(getattr(model4, 'age'), 2)

        self.assertEqual(
            fields(model5),
            [
                ('id', models.AutoField), ('foo', models.IntegerField),
                ('concretemodel_ptr', models.OneToOneField),
                ('age', models.SmallIntegerField), ('concretemodel2_ptr', models.OneToOneField),
                ('name', models.CharField),
            ]
        )
<|endoftext|>


<commit_before>#!/usr/bin/python

import Axon.LikeFile
import pprocess
import time

# --- Support code, this will go back into the library. ---------------------------------------------------------
class ProcessWrapComponent(object):
    def __init__(self, somecomponent):
        self.exchange = pprocess.Exchange()
        self.channel = None
        self.inbound = []
        self.thecomponent = somecomponent
        self.ce = None

    def run(self, channel):
        from Axon.LikeFile import likefile, background
        background().start()
        self.exchange.add(channel)
        self.channel = channel
        self.ce = likefile(self.thecomponent)
        for i in self.main():
            pass

    def activate(self):
        channel = pprocess.start(self.run)
        return channel

    def main(self):
        t = 0
        while 1:
            if time.time() - t > 0.2:
                t = time.time()

            if self.exchange.ready(0):
                chan = self.exchange.ready(0)[0]
                D = chan._receive()
                self.ce.put(*D)

            D = self.ce.anyReady()
            if D:
                for boxname in D:
                    D = self.ce.get(boxname)
                    self.channel._send((D, boxname))
            yield 1

def ProcessPipeline(component_one, component_two, component_three, component_four):
    X = ProcessWrapComponent(component_one)
    Y = ProcessWrapComponent( component_two)

    A = ProcessWrapComponent( component_three )
    B = ProcessWrapComponent( component_four)

    exchange = pprocess.Exchange()
    chan1 = X.activate()
    chan2 = Y.activate()
    chan3 = A.activate()
    chan4 = B.activate()

    exchange.add(chan1)
    exchange.add(chan2)
    exchange.add(chan3)
    exchange.add(chan4)

    mappings = {
         (chan1, "outbox") : (chan2, "inbox"),
         (chan1, "signal") : (chan2, "control"),

         (chan2, "outbox") : (chan3, "inbox"),
         (chan2, "signal") : (chan3, "control"),

         (chan3, "outbox") : (chan4, "inbox"),
         (chan3, "signal") : (chan4, "control"),

    }
    while 1:
        for chan in exchange.ready(0):
            D = chan._receive()
            dest = mappings[ ( chan, D[1] ) ]
            dest[0]._send( (D[0], dest[1] ) )
        time.sleep(0.1)

# --- End Support code ---------------------------------------------------------

# Client code that uses the support code

from Kamaelia.UI.Pygame.Text import TextDisplayer, Textbox

from Kamaelia.Chassis.Pipeline import Pipeline
import sys

if len(sys.argv) <2:
    ProcessPipeline(
                Textbox(position=(20, 340),
                                 text_height=36,
                                 screen_width=900,
                                 screen_height=200,
                                 background_color=(130,0,70),
                                 text_color=(255,255,255)),
                TextDisplayer(position=(20, 90),
                                        text_height=36,
                                        screen_width=900,
                                        screen_height=200,
                                        background_color=(130,0,70),
                                        text_color=(255,255,255)),
                Textbox(position=(20, 340),
                                 text_height=36,
                                 screen_width=900,
                                 screen_height=200,
                                 background_color=(130,0,70),
                                 text_color=(255,255,255)),
                TextDisplayer(position=(20, 90),
                                        text_height=36,
                                        screen_width=900,
                                        screen_height=200,
                                        background_color=(130,0,70),
                                        text_color=(255,255,255))
                )
else:
    Pipeline(
                Textbox(position=(20, 340),
                                 text_height=36,
                                 screen_width=900,
                                 screen_height=200,
                                 background_color=(130,0,70),
                                 text_color=(255,255,255)),
                TextDisplayer(position=(20, 90),
                                        text_height=36,
                                        screen_width=900,
                                        screen_height=200,
                                        background_color=(130,0,70),
                                        text_color=(255,255,255))
                ).run()

<commit_msg>Half way transformed.<commit_after>#!/usr/bin/python

import Axon.LikeFile
import pprocess
import time

# --- Support code, this will go back into the library. ---------------------------------------------------------
class ProcessWrapComponent(object):
    def __init__(self, somecomponent):
        self.exchange = pprocess.Exchange()
        self.channel = None
        self.inbound = []
        self.thecomponent = somecomponent
        self.ce = None

    def run(self, channel):
        from Axon.LikeFile import likefile, background
        background().start()
        self.exchange.add(channel)
        self.channel = channel
        self.ce = likefile(self.thecomponent)
        for i in self.main():
            pass

    def activate(self):
        channel = pprocess.start(self.run)
        return channel

    def main(self):
        t = 0
        while 1:
            if time.time() - t > 0.2:
                t = time.time()

            if self.exchange.ready(0):
                chan = self.exchange.ready(0)[0]
                D = chan._receive()
                self.ce.put(*D)

            D = self.ce.anyReady()
            if D:
                for boxname in D:
                    D = self.ce.get(boxname)
                    self.channel._send((D, boxname))
            yield 1

def ProcessPipeline(*components):
    exchange = pprocess.Exchange()

    chans = []

    for comp in components:
        A = ProcessWrapComponent( comp )
        chan = A.activate()
        chans.append( chan )
        exchange.add(chan )

    mappings = {
         (chans[0], "outbox") : (chans[1], "inbox"),
         (chans[0], "signal") : (chans[1], "control"),

         (chans[1], "outbox") : (chans[2], "inbox"),
         (chans[1], "signal") : (chans[2], "control"),

         (chans[2], "outbox") : (chans[3], "inbox"),
         (chans[2], "signal") : (chans[3], "control"),

    }
    while 1:
        for chan in exchange.ready(0):
            D = chan._receive()
            dest = mappings[ ( chan, D[1] ) ]
            dest[0]._send( (D[0], dest[1] ) )
        time.sleep(0.1)

# --- End Support code ---------------------------------------------------------

# Client code that uses the support code

from Kamaelia.UI.Pygame.Text import TextDisplayer, Textbox

from Kamaelia.Chassis.Pipeline import Pipeline
import sys

if len(sys.argv) <2:
    ProcessPipeline(
                Textbox(position=(20, 340),
                                 text_height=36,
                                 screen_width=900,
                                 screen_height=200,
                                 background_color=(130,0,70),
                                 text_color=(255,255,255)),
                TextDisplayer(position=(20, 90),
                                        text_height=36,
                                        screen_width=900,
                                        screen_height=200,
                                        background_color=(130,0,70),
                                        text_color=(255,255,255)),
                Textbox(position=(20, 340),
                                 text_height=36,
                                 screen_width=900,
                                 screen_height=200,
                                 background_color=(130,0,70),
                                 text_color=(255,255,255)),
                TextDisplayer(position=(20, 90),
                                        text_height=36,
                                        screen_width=900,
                                        screen_height=200,
                                        background_color=(130,0,70),
                                        text_color=(255,255,255))
                )
else:
    Pipeline(
                Textbox(position=(20, 340),
                                 text_height=36,
                                 screen_width=900,
                                 screen_height=200,
                                 background_color=(130,0,70),
                                 text_color=(255,255,255)),
                TextDisplayer(position=(20, 90),
                                        text_height=36,
                                        screen_width=900,
                                        screen_height=200,
                                        background_color=(130,0,70),
                                        text_color=(255,255,255))
                ).run()

<|endoftext|>


<commit_before>'''

'''

__author__ = 'R Devon Hjelm'
__author_email__ = 'erroneus@gmail.com'

import ast
import copy
import logging
import os
from os import path
from shutil import copyfile, rmtree

import torch

from lib import config, models, exp
from lib.log_utils import set_file_logger, set_stream_logger
from lib.utils import convert_nested_dict_to_handler, make_argument_parser, _protected_args
from lib import data, optimizer, train
from lib.viz import init as viz_init

logger = logging.getLogger('cortex.init')

_args = dict(data=data._args, optimizer=optimizer._args, train=train._args)
_args_help = dict(data=data._args_help, optimizer=optimizer._args_help, train=train._args_help)

def setup_out_dir(out_path, name=None, clean=False):
    '''Sets up the output directory of an experiment.

    '''

    if out_path is None:
        if name is None:
            raise ValueError('If `out_path` (-o) argument is not set, you '
                             'must set the `name` (-n)')
        out_path = config.OUT_PATH
        if out_path is None:
            raise ValueError('If `--out_path` (`-o`) argument is not set, you '
                             'must set both the name argument and configure '
                             'the out_path entry in `config.yaml`')

    if name is not None: out_path = path.join(out_path, name)

    if not path.isdir(out_path):
        logger.info('Creating out path `{}`'.format(out_path))
        os.mkdir(out_path)

    binary_dir = path.join(out_path, 'binaries')
    image_dir = path.join(out_path, 'images')

    if clean:
        logger.warning('Cleaning directory (cannot be undone)')
        if path.isdir(binary_dir):
            rmtree(binary_dir)
        if path.isdir(image_dir):
            rmtree(image_dir)

    if not path.isdir(binary_dir): os.mkdir(binary_dir)
    if not path.isdir(image_dir): os.mkdir(image_dir)

    logger.info('Setting out path to `{}`'.format(out_path))
    logger.info('Logging to `{}`'.format(path.join(out_path, 'out.log')))
    set_file_logger(path.join(out_path, 'out.log'))

    exp.OUT_DIRS.update(binary_dir=binary_dir, image_dir=image_dir)

_known_args = dict((k, list(v.keys())) for k, v in _args.items())

def update_args(args, **kwargs):
    if args is not None:
        a = args.split(',')
        for a_ in a:
            k, v = a_.split('=')

            try:
                v = ast.literal_eval(v)
            except ValueError:
                pass

            k_split = k.split('.')
            kw = kwargs
            k_base = None
            for i, k_ in enumerate(k_split):
                if i < len(k_split) - 1:
                    if k_base in _known_args and k_ not in kw:
                        if k_ in _known_args[k_base]:
                            kw[k_] = {}
                    if k_ in kw:
                        kw = kw[k_]
                        k_base = k_
                    else:
                        raise ValueError('Unknown arg {}'.format(k))
                else:
                    if kw is None:
                        kw = dict(k_=v)
                    else:
                        kw[k_] = v

def handle_deprecated(data=None, model=None, optimizer=None, routines=None, test_routines=None, train=None):
    if 'noise_variables' in data:
        for k, v in data['noise_variables'].items():
            if isinstance(v, tuple):
                DeprecationWarning('Old-stype tuple found in noise argument. Converting.')
                data['noise_variables'][k] = dict(dist=v[0], size=v[1])

    if 'updates_per_model' in optimizer:
        DeprecationWarning('Old-stype `updates_per_model` found. Use `updates_per_routine`')
        optimizer['updates_per_routine'] = optimizer.pop('updates_per_model')

def setup_reload(arch, use_cuda, exp_file):
    exp.USE_CUDA = use_cuda

    if exp.USE_CUDA:
        logger.info('Using GPU')
    else:
        logger.info('Using CPU')

    config_file_path = path.join(path.dirname(
        path.abspath(__file__)), 'config.yaml')
    if not path.isfile(config_file_path): config_file_path = None
    config.update_config(config_file_path)
    viz_init()

    arch.setup(arch)

    logger.info('Reloading from {}'.format(exp_file))
    d = torch.load(exp_file)
    exp.INFO.update(**d['info'])
    exp.NAME = d['info']['name']
    exp.SUMMARY.update(**d['summary'])
    exp.ARGS.update(**d['args'])
    reloads = d['arch'].keys()
    for k in reloads:
        exp.MODEL_PARAMS_RELOAD.update(**{k: d['arch'][k]})

def reload_experiment(args):
    exp_file = args.reload
    reloads = args.reloads
    name = args.name

    d = torch.load(exp_file)
    exp.INFO.update(**d['info'])
    exp.NAME = d['info']['name']
    exp.SUMMARY.update(**d['summary'])
    kwargs = convert_nested_dict_to_handler(d['args'])
    handle_deprecated(**kwargs)
    exp.ARGS.update(**kwargs)
    reloads = reloads or d['models'].keys()
    for k in reloads:
        exp.MODEL_PARAMS_RELOAD.update(**{k: d['models'][k]})
    out_dirs = d['out_dirs']

    if name:
        exp.NAME = name
        exp.INFO['name'] = name
        setup_out_dir(args.out_path, name, clean=args.clean)
    out_path = path.dirname(path.dirname(exp_file))
    out_dirs = dict((k, path.join(out_path, path.basename(v))) for k, v in out_dirs.items())
    exp.OUT_DIRS.update(**out_dirs)

def setup(use_cuda):

    # Parse args and set logger, cuda
    parser = make_argument_parser()
    for arg_k in _args:
        args = _args[arg_k]
        for k, v in args.items():
            if isinstance(v, dict):
                pass
            type_ = type(v) if v is not None else str
            parser.add_argument('--' + arg_k + '.' + k, default=None, type=type_, help=_args_help[arg_k][k])
    args = parser.parse_args()

    set_stream_logger(args.verbosity)
    exp.USE_CUDA = use_cuda
    if exp.USE_CUDA:
        logger.info('Using GPU')
    else:
        logger.info('Using CPU')

    # Setup file paths
    config_file_path = path.join(path.dirname(
        path.abspath(__file__)), 'config.yaml')
    if not path.isfile(config_file_path): config_file_path = None

    # User config
    config.update_config(config_file_path)

    # Initialize visualizer
    viz_init()

    # Set up the architecture, build the models
    arch = models.setup(args.arch)

    if args.reload:
        # Reload the old experiment
        if not path.isfile(args.reload):
            raise ValueError('Cannot find {}'.format(args.reload))
        logger.info('Reloading from {}'.format(args.reload))
        copyfile(args.reload, args.reload + '.bak')
        reload_experiment(args)
        for k, v in vars(args).items():
            if v is not None:
                if '.' in k:
                    head, tail = k.split('.')
                    exp.ARGS[head][tail] = v
        update_args(args.args, **exp.ARGS)

    else:
        # Make a new experiment
        kwargs = copy.deepcopy(_args)
        for k, v in arch.DEFAULT_CONFIG.items():
            if k not in kwargs:
                kwargs[k] = {}
            kwargs[k].update(**v)
        for k, v in vars(args).items():
            if v is not None:
                if '.' in k:
                    head, tail = k.split('.')
                    kwargs[head][tail] = v

        kwargs = convert_nested_dict_to_handler(kwargs)

        if 'test_routines' not in kwargs.keys():
            kwargs['test_routines'] = {}

        update_args(args.args, **kwargs)

        name = args.name
        if name is None:
            name = args.arch
        exp.NAME = name
        exp.INFO['name'] = name
        exp.configure_experiment(config_file=args.config_file, **kwargs)
        exp.ARGS.update(**kwargs)
        setup_out_dir(args.out_path, name, clean=args.clean)

    if hasattr(arch, 'SETUP'):
        getattr(arch, 'SETUP')(**exp.ARGS)

    if hasattr(arch, 'DataLoader'):
        DataLoader = getattr(arch, 'DataLoader')
        logger.info('Loading custom DataLoader class, {}'.format(DataLoader))
        exp.ARGS['data']['DataLoader'] = DataLoader

    if hasattr(arch, 'Dataset'):
        Dataset = getattr('Dataset')
        logger.info('Loading custom Dataset class, {}'.format(Dataset))
        exp.ARGS['data']['Dataset'] = Dataset

    if hasattr(arch, 'transform'):
        transform = getattr('transform')
        logger.info('Loading custom transform function, {}'.format(transform))
        exp.ARGS['data']['transform'] = transform

    exp.ARGS['data']['copy_to_local'] = args.copy_to_local

    exp.ARGS['train']['test_mode'] = args.test
<commit_msg>small patch to fix default args<commit_after>'''

'''

__author__ = 'R Devon Hjelm'
__author_email__ = 'erroneus@gmail.com'

import ast
import copy
import logging
import os
from os import path
from shutil import copyfile, rmtree

import torch

from lib import config, models, exp
from lib.log_utils import set_file_logger, set_stream_logger
from lib.utils import convert_nested_dict_to_handler, make_argument_parser, _protected_args
from lib import data, optimizer, train
from lib.viz import init as viz_init

logger = logging.getLogger('cortex.init')

_args = dict(data=data._args, optimizer=optimizer._args, train=train._args)
_args_help = dict(data=data._args_help, optimizer=optimizer._args_help, train=train._args_help)

def setup_out_dir(out_path, name=None, clean=False):
    '''Sets up the output directory of an experiment.

    '''

    if out_path is None:
        if name is None:
            raise ValueError('If `out_path` (-o) argument is not set, you '
                             'must set the `name` (-n)')
        out_path = config.OUT_PATH
        if out_path is None:
            raise ValueError('If `--out_path` (`-o`) argument is not set, you '
                             'must set both the name argument and configure '
                             'the out_path entry in `config.yaml`')

    if name is not None: out_path = path.join(out_path, name)

    if not path.isdir(out_path):
        logger.info('Creating out path `{}`'.format(out_path))
        os.mkdir(out_path)

    binary_dir = path.join(out_path, 'binaries')
    image_dir = path.join(out_path, 'images')

    if clean:
        logger.warning('Cleaning directory (cannot be undone)')
        if path.isdir(binary_dir):
            rmtree(binary_dir)
        if path.isdir(image_dir):
            rmtree(image_dir)

    if not path.isdir(binary_dir): os.mkdir(binary_dir)
    if not path.isdir(image_dir): os.mkdir(image_dir)

    logger.info('Setting out path to `{}`'.format(out_path))
    logger.info('Logging to `{}`'.format(path.join(out_path, 'out.log')))
    set_file_logger(path.join(out_path, 'out.log'))

    exp.OUT_DIRS.update(binary_dir=binary_dir, image_dir=image_dir)

_known_args = dict((k, list(v.keys())) for k, v in _args.items())

def update_args(args, **kwargs):
    if args is not None:
        a = args.split(',')
        for a_ in a:
            k, v = a_.split('=')

            try:
                v = ast.literal_eval(v)
            except ValueError:
                pass

            k_split = k.split('.')
            kw = kwargs
            k_base = None
            for i, k_ in enumerate(k_split):
                if i < len(k_split) - 1:
                    if k_base in _known_args and k_ not in kw:
                        if k_ in _known_args[k_base]:
                            kw[k_] = {}
                    if k_ in kw:
                        kw = kw[k_]
                        k_base = k_
                    else:
                        raise ValueError('Unknown arg {}'.format(k))
                else:
                    if kw is None:
                        kw = dict(k_=v)
                    else:
                        kw[k_] = v

def handle_deprecated(data=None, model=None, optimizer=None, routines=None, test_routines=None, train=None):
    if 'noise_variables' in data:
        for k, v in data['noise_variables'].items():
            if isinstance(v, tuple):
                DeprecationWarning('Old-stype tuple found in noise argument. Converting.')
                data['noise_variables'][k] = dict(dist=v[0], size=v[1])

    if 'updates_per_model' in optimizer:
        DeprecationWarning('Old-stype `updates_per_model` found. Use `updates_per_routine`')
        optimizer['updates_per_routine'] = optimizer.pop('updates_per_model')

def setup_reload(arch, use_cuda, exp_file):
    exp.USE_CUDA = use_cuda

    if exp.USE_CUDA:
        logger.info('Using GPU')
    else:
        logger.info('Using CPU')

    config_file_path = path.join(path.dirname(
        path.abspath(__file__)), 'config.yaml')
    if not path.isfile(config_file_path): config_file_path = None
    config.update_config(config_file_path)
    viz_init()

    arch.setup(arch)

    logger.info('Reloading from {}'.format(exp_file))
    d = torch.load(exp_file)
    exp.INFO.update(**d['info'])
    exp.NAME = d['info']['name']
    exp.SUMMARY.update(**d['summary'])
    exp.ARGS.update(**d['args'])
    reloads = d['arch'].keys()
    for k in reloads:
        exp.MODEL_PARAMS_RELOAD.update(**{k: d['arch'][k]})

def reload_experiment(args):
    exp_file = args.reload
    reloads = args.reloads
    name = args.name

    d = torch.load(exp_file)
    exp.INFO.update(**d['info'])
    exp.NAME = d['info']['name']
    exp.SUMMARY.update(**d['summary'])
    kwargs = copy.deepcopy(_args)
    for k in d['args']:
        if k in kwargs:
            kwargs[k].update(**d['args'][k])
        else:
            kwargs[k] = d['args'][k]
    kwargs = convert_nested_dict_to_handler(kwargs)
    handle_deprecated(**kwargs)
    exp.ARGS.update(**kwargs)

    reloads = reloads or d['models'].keys()
    for k in reloads:
        exp.MODEL_PARAMS_RELOAD.update(**{k: d['models'][k]})
    out_dirs = d['out_dirs']

    if name:
        exp.NAME = name
        exp.INFO['name'] = name
        setup_out_dir(args.out_path, name, clean=args.clean)
    out_path = path.dirname(path.dirname(exp_file))
    out_dirs = dict((k, path.join(out_path, path.basename(v))) for k, v in out_dirs.items())
    exp.OUT_DIRS.update(**out_dirs)

def setup(use_cuda):

    # Parse args and set logger, cuda
    parser = make_argument_parser()
    for arg_k in _args:
        args = _args[arg_k]
        for k, v in args.items():
            if isinstance(v, dict):
                pass
            type_ = type(v) if v is not None else str
            parser.add_argument('--' + arg_k + '.' + k, default=None, type=type_, help=_args_help[arg_k][k])
    args = parser.parse_args()

    set_stream_logger(args.verbosity)
    exp.USE_CUDA = use_cuda
    if exp.USE_CUDA:
        logger.info('Using GPU')
    else:
        logger.info('Using CPU')

    # Setup file paths
    config_file_path = path.join(path.dirname(
        path.abspath(__file__)), 'config.yaml')
    if not path.isfile(config_file_path): config_file_path = None

    # User config
    config.update_config(config_file_path)

    # Initialize visualizer
    viz_init()

    # Set up the architecture, build the models
    arch = models.setup(args.arch)

    if args.reload:
        # Reload the old experiment
        if not path.isfile(args.reload):
            raise ValueError('Cannot find {}'.format(args.reload))
        logger.info('Reloading from {}'.format(args.reload))
        copyfile(args.reload, args.reload + '.bak')
        reload_experiment(args)
        for k, v in vars(args).items():
            if v is not None:
                if '.' in k:
                    head, tail = k.split('.')
                    exp.ARGS[head][tail] = v
        update_args(args.args, **exp.ARGS)

    else:
        # Make a new experiment
        kwargs = copy.deepcopy(_args)
        for k, v in arch.DEFAULT_CONFIG.items():
            if k not in kwargs:
                kwargs[k] = {}
            kwargs[k].update(**v)
        for k, v in vars(args).items():
            if v is not None:
                if '.' in k:
                    head, tail = k.split('.')
                    kwargs[head][tail] = v

        kwargs = convert_nested_dict_to_handler(kwargs)

        if 'test_routines' not in kwargs.keys():
            kwargs['test_routines'] = {}

        update_args(args.args, **kwargs)

        name = args.name
        if name is None:
            name = args.arch
        exp.NAME = name
        exp.INFO['name'] = name
        exp.configure_experiment(config_file=args.config_file, **kwargs)
        exp.ARGS.update(**kwargs)
        setup_out_dir(args.out_path, name, clean=args.clean)

    if hasattr(arch, 'SETUP'):
        getattr(arch, 'SETUP')(**exp.ARGS)

    if hasattr(arch, 'DataLoader'):
        DataLoader = getattr(arch, 'DataLoader')
        logger.info('Loading custom DataLoader class, {}'.format(DataLoader))
        exp.ARGS['data']['DataLoader'] = DataLoader

    if hasattr(arch, 'Dataset'):
        Dataset = getattr('Dataset')
        logger.info('Loading custom Dataset class, {}'.format(Dataset))
        exp.ARGS['data']['Dataset'] = Dataset

    if hasattr(arch, 'transform'):
        transform = getattr('transform')
        logger.info('Loading custom transform function, {}'.format(transform))
        exp.ARGS['data']['transform'] = transform

    exp.ARGS['data']['copy_to_local'] = args.copy_to_local

    exp.ARGS['train']['test_mode'] = args.test
<|endoftext|>