<commit_before>#
# HSC_cfg.py -- HSC configuration information
#
#  Russell Kackley (rkackley@naoj.org)
#

# Complete list of all HSC filters
all_filters =  [f.upper() for f in ('g', 'r', 'r2', 'i', 'i2', 'z', 'Y', 'NB387', 'NB400', 'NB468', 'NB515', 'NB527', 'NB656', 'NB718', 'NB816', 'NB921', 'NB926', 'IB945', 'NB973', 'NB1010')]

# List of filters available for each semester
semester_filters = {
    'S16A': [f.upper() for f in ('g', 'r', 'i', 'z', 'Y', 'NB921', 'NB816', 'NB515', 'NB468', 'NB527', 'NB656')],
    'S16B': [f.upper() for f in ('g', 'r', 'i', 'i2', 'z', 'Y', 'NB387', 'NB515', 'NB527', 'NB816', 'NB921')],
    'S17A': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'NB387', 'NB527', 'NB656', 'NB718', 'NB816', 'NB921', 'NB926')],
    'S17B': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'NB387', 'NB527', 'NB816', 'NB926')],
    'S18A': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'IB945', 'NB101', 'NB816')],
    'S18B': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'IB945', 'NB387', 'NB468', 'NB527', 'NB816', 'NB921', 'NB1010')],
    'S19A': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'IB945', 'NB387', 'NB527', 'NB816', 'NB921')],
    'S19B': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'NB400', 'NB468', 'NB515', 'NB816', 'NB926')],
    'S20A': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'NB387', 'NB816', 'NB921', 'NB926')],
    }
<commit_msg>HSC S20A filter list updated<commit_after>#
# HSC_cfg.py -- HSC configuration information
#
#  Russell Kackley (rkackley@naoj.org)
#

# Complete list of all HSC filters
all_filters =  [f.upper() for f in ('g', 'r', 'r2', 'i', 'i2', 'z', 'Y', 'NB387', 'NB400', 'NB468', 'NB515', 'NB527', 'NB656', 'NB718', 'NB816', 'NB921', 'NB926', 'IB945', 'NB973', 'NB1010')]

# List of filters available for each semester
semester_filters = {
    'S16A': [f.upper() for f in ('g', 'r', 'i', 'z', 'Y', 'NB921', 'NB816', 'NB515', 'NB468', 'NB527', 'NB656')],
    'S16B': [f.upper() for f in ('g', 'r', 'i', 'i2', 'z', 'Y', 'NB387', 'NB515', 'NB527', 'NB816', 'NB921')],
    'S17A': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'NB387', 'NB527', 'NB656', 'NB718', 'NB816', 'NB921', 'NB926')],
    'S17B': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'NB387', 'NB527', 'NB816', 'NB926')],
    'S18A': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'IB945', 'NB101', 'NB816')],
    'S18B': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'IB945', 'NB387', 'NB468', 'NB527', 'NB816', 'NB921', 'NB1010')],
    'S19A': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'IB945', 'NB387', 'NB527', 'NB816', 'NB921')],
    'S19B': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'NB400', 'NB468', 'NB515', 'NB816', 'NB926')],
    'S20A': [f.upper() for f in ('g', 'r2', 'i2', 'z', 'Y', 'NB400', 'NB816', 'NB921', 'NB926')],
    }
<|endoftext|>


<commit_before>from collections import defaultdict
import json
import os
import re

from SPARQLWrapper import SPARQLWrapper, JSON

def query_people(offset=0, limit=100):
    client = SPARQLWrapper("http://dbpedia.org/sparql")
    client.setQuery('''
    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
    PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
    SELECT DISTINCT
        ?person
        ?personType
        ?name
        ?description
        ?url
        ?teamName as ?team
        ?positionName as ?position
        ?birthDate as ?birth_date
        ?draftYear as ?draft_year
        ?collegeName as ?college
        ?highSchoolName as ?high_school
        ?birthPlaceName as ?birth_place
    WHERE {
      ?person rdf:type ?personType ;
              dbo:team|dbp:team _:team ;
              dbo:position _:position ;
              foaf:name ?name ;
              foaf:isPrimaryTopicOf ?url ;
              rdfs:comment ?description ;
              dbo:birthDate ?birthDate .
      _:team rdf:type _:teamType .
      FILTER (
        (!regex(?name, "^.+,.+$", "i")) &&
        (LANG(?description) = 'en') &&
        (DATATYPE(?birthDate) = xsd:date) &&
        (
            (
                (?personType = dbo:BaseballPlayer) &&
                (_:teamType = yago:MajorLeagueBaseballTeams)
            ) ||
            (
                (?personType = dbo:BasketballPlayer) &&
                (_:teamType = yago:NationalBasketballAssociationTeams)
            ) ||
            (
                (?personType = dbo:AmericanFootballPlayer) &&
                (_:teamType = yago:NationalFootballLeagueTeams)
            )
        )
    )

      _:team rdfs:label ?teamName .
      FILTER (LANG(?teamName) = 'en')

      _:position rdfs:label ?positionName .
      FILTER (LANG(?positionName) = 'en')

      OPTIONAL {
        ?person dbo:draftYear ?draftYear .
      }
      OPTIONAL {
        ?person dbp:college ?college .
        ?college rdfs:label ?collegeName .
        FILTER(LANG(?collegeName) = 'en')
      }
      OPTIONAL {
        ?person dbp:highschool ?highschool .
        ?highschool rdfs:label ?highSchoolName .
        FILTER(LANG(?highSchoolName) = 'en')
      }
      OPTIONAL {
        ?person dbo:birthPlace _:birthPlace .
        _:birthPlace dbo:country _:birthCountry .
        _:birthCountry rdfs:label ?birthPlaceName .
        FILTER(LANG(?birthPlaceName) = 'en')
      }
    } OFFSET %(offset)d LIMIT %(limit)d
    ''' % {'offset': offset, 'limit': limit})
    client.setReturnFormat(JSON)

    print 'grabbing rows %d-%d' % (offset, offset + limit)
    import datetime
    start = datetime.datetime.now()

    result = client.query().convert()

    dt = datetime.datetime.now() - start
    print ' ' * 4 + 'took %.1f seconds' % dt.total_seconds()
    return result['results']['bindings']

def get_people(max_limit=None):
    limit = 2000
    n_offset = 0
    people = []
    while True:
        offset = n_offset * limit
        results = query_people(offset=offset, limit=limit)
        if len(results) == 0:
            break
        people.extend(results)
        n_offset += 1
    return people

def iterate_fuse_objects(max_limit=None):
    """ Generate all Fuse objects from the entire list of people.

    We'll:
    1. Sort all the rows returned by the URI.
    2. Group each set of URIs into a single Fuse object.
    3. yield the last Fuse object when the new URI comes.

    If you are not familiar with Python yield statements, see:

        http://stackoverflow.com/questions/231767/what-does-the-yield-keyword-do-in-python

    """
    people = get_people(max_limit=max_limit)
    people.sort(key=lambda x: x['person']['value'])

    fuseobj = None
    for athlete in people:
        athlete_uri = athlete['person']['value']
        if fuseobj is None or fuseobj['fuse:id'] != athlete_uri:
            # if this is the initial iteration or the fuse:id is new, yield
            # the fuseobj and create a new one
            if fuseobj is not None:
                # set is not JSON serializable - convert list before yielding
                fuseobj['team'] = list(fuseobj['team'])
                fuseobj['position'] = list(fuseobj['position'])
                fuseobj['high_school'] = list(fuseobj['high_school'])
                fuseobj['college'] = list(fuseobj['college'])
                yield fuseobj
            fuseobj = defaultdict(set)
            fuseobj['fuse:type'] = 'athlete'
            fuseobj['fuse:id'] = athlete_uri
            dbo = 'http://dbpedia.org/ontology/'
            fuseobj['sport'] = {
                dbo + 'BaseballPlayer': 'Baseball',
                dbo + 'BasketballPlayer': 'Basketball',
                dbo + 'AmericanFootballPlayer': 'Football',
            }[convert_value(athlete['personType'])]
            for key in ('name', 'description', 'birth_date', 'sport',
                        'draft_year', 'url', 'birth_place'):
                try:
                    fuseobj[key] = convert_value(athlete[key])
                except KeyError:
                    pass
        for key in ('position', 'high_school', 'college', 'team'):
            try:
                value = convert_value(athlete[key])
            except KeyError:
                pass
            else:
                # Clean up the strings
                if isinstance(value, basestring):
                    value = re.sub('\(.*?\)', '', value).strip()
                    if key == 'college':
                        flags = re.X | re.I
                        exp = r'''
                            ((wo)?men\'s)?.*
                            (football|basketball|baseball).*$
                        '''
                        value = re.sub(exp, '', value, flags=flags).strip()
                fuseobj[key].add(value)

def convert_value(value_obj):
    """ Convert the SPARQL json object dictionary into a Fuse type.
    """
    valuetype = value_obj['type']
    value = value_obj['value']
    if valuetype == 'typed-literal':
        typ = value_obj['datatype']
        if typ == 'http://www.w3.org/2001/XMLSchema#integer':
            value = int(value)
        elif typ == 'http://www.w3.org/2001/XMLSchema#float':
            value = float(value)
        elif typ == 'http://www.w3.org/2001/XMLSchema#date':
            value = {'_date': value}
    return value

def main():
    # Get some custom variables from the environment (mostly for testing)
    # PEOPLE_DUMP_LIMIT decides how many people we'll dump to file.
    # PEOPLE_JSON specifies a custom athletes file location.
    max_limit = int(os.environ.get('PEOPLE_DUMP_LIMIT'))
    filename = os.environ.get('PEOPLE_JSON', 'var/athletes.json')
    items = []
    for person in iterate_fuse_objects(max_limit=max_limit):
        items.append(person)
    with open(filename, 'w') as f:
        json.dump({'items': items}, f, indent=2)

if __name__ == '__main__':
    main()
<commit_msg>refactor: remove all yield statements<commit_after>from collections import defaultdict
import json
import os
import re

from SPARQLWrapper import SPARQLWrapper, JSON

def query_people(offset=0, limit=100):
    client = SPARQLWrapper("http://dbpedia.org/sparql")
    client.setQuery('''
    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
    PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
    SELECT DISTINCT
        ?person
        ?personType
        ?name
        ?description
        ?url
        ?teamName as ?team
        ?positionName as ?position
        ?birthDate as ?birth_date
        ?draftYear as ?draft_year
        ?collegeName as ?college
        ?highSchoolName as ?high_school
        ?birthPlaceName as ?birth_place
    WHERE {
      ?person rdf:type ?personType ;
              dbo:team|dbp:team _:team ;
              dbo:position _:position ;
              foaf:name ?name ;
              foaf:isPrimaryTopicOf ?url ;
              rdfs:comment ?description ;
              dbo:birthDate ?birthDate .
      _:team rdf:type _:teamType .
      FILTER (
        (!regex(?name, "^.+,.+$", "i")) &&
        (LANG(?description) = 'en') &&
        (DATATYPE(?birthDate) = xsd:date) &&
        (
            (
                (?personType = dbo:BaseballPlayer) &&
                (_:teamType = yago:MajorLeagueBaseballTeams)
            ) ||
            (
                (?personType = dbo:BasketballPlayer) &&
                (_:teamType = yago:NationalBasketballAssociationTeams)
            ) ||
            (
                (?personType = dbo:AmericanFootballPlayer) &&
                (_:teamType = yago:NationalFootballLeagueTeams)
            )
        )
    )

      _:team rdfs:label ?teamName .
      FILTER (LANG(?teamName) = 'en')

      _:position rdfs:label ?positionName .
      FILTER (LANG(?positionName) = 'en')

      OPTIONAL {
        ?person dbo:draftYear ?draftYear .
      }
      OPTIONAL {
        ?person dbp:college ?college .
        ?college rdfs:label ?collegeName .
        FILTER(LANG(?collegeName) = 'en')
      }
      OPTIONAL {
        ?person dbp:highschool ?highschool .
        ?highschool rdfs:label ?highSchoolName .
        FILTER(LANG(?highSchoolName) = 'en')
      }
      OPTIONAL {
        ?person dbo:birthPlace _:birthPlace .
        _:birthPlace dbo:country _:birthCountry .
        _:birthCountry rdfs:label ?birthPlaceName .
        FILTER(LANG(?birthPlaceName) = 'en')
      }
    } OFFSET %(offset)d LIMIT %(limit)d
    ''' % {'offset': offset, 'limit': limit})
    client.setReturnFormat(JSON)

    print 'grabbing rows %d-%d' % (offset, offset + limit)
    import datetime
    start = datetime.datetime.now()

    result = client.query().convert()

    dt = datetime.datetime.now() - start
    print ' ' * 4 + 'took %.1f seconds' % dt.total_seconds()
    return result['results']['bindings']

def get_people(max_limit=None):
    limit = 2000
    n_offset = 0
    people = []
    while True:
        offset = n_offset * limit
        results = query_people(offset=offset, limit=limit)
        if len(results) == 0:
            break
        people.extend(results)
        n_offset += 1
    return people

def get_fuse_objects(max_limit=None):
    """ Generate all Fuse objects from the entire list of people.

    We'll:
    1. Sort all the rows returned by the URI.
    2. Group each set of URIs into a single Fuse object.
    3. Store the last Fuse object when the new URI comes.
    """
    people = get_people(max_limit=max_limit)
    people.sort(key=lambda x: x['person']['value'])

    fuseobjs = []
    fuseobj = None
    for athlete in people:
        athlete_uri = athlete['person']['value']
        if fuseobj is None or fuseobj['fuse:id'] != athlete_uri:
            # if this is the initial iteration or the fuse:id is new, yield
            # the fuseobj and create a new one
            if fuseobj is not None:
                # set is not JSON serializable - convert list before storing
                fuseobj['team'] = list(fuseobj['team'])
                fuseobj['position'] = list(fuseobj['position'])
                fuseobj['high_school'] = list(fuseobj['high_school'])
                fuseobj['college'] = list(fuseobj['college'])
                fuseobjs.append(fuseobj)
            fuseobj = defaultdict(set)
            fuseobj['fuse:type'] = 'athlete'
            fuseobj['fuse:id'] = athlete_uri
            dbo = 'http://dbpedia.org/ontology/'
            fuseobj['sport'] = {
                dbo + 'BaseballPlayer': 'Baseball',
                dbo + 'BasketballPlayer': 'Basketball',
                dbo + 'AmericanFootballPlayer': 'Football',
            }[convert_value(athlete['personType'])]
            for key in ('name', 'description', 'birth_date', 'sport',
                        'draft_year', 'url', 'birth_place'):
                try:
                    fuseobj[key] = convert_value(athlete[key])
                except KeyError:
                    pass
        for key in ('position', 'high_school', 'college', 'team'):
            try:
                value = convert_value(athlete[key])
            except KeyError:
                pass
            else:
                # Clean up the strings
                if isinstance(value, basestring):
                    value = re.sub('\(.*?\)', '', value).strip()
                    if key == 'college':
                        flags = re.X | re.I
                        exp = r'''
                            ((wo)?men\'s)?.*
                            (football|basketball|baseball).*$
                        '''
                        value = re.sub(exp, '', value, flags=flags).strip()
                fuseobj[key].add(value)
    return fuseobjs

def convert_value(value_obj):
    """ Convert the SPARQL json object dictionary into a Fuse type.
    """
    valuetype = value_obj['type']
    value = value_obj['value']
    if valuetype == 'typed-literal':
        typ = value_obj['datatype']
        if typ == 'http://www.w3.org/2001/XMLSchema#integer':
            value = int(value)
        elif typ == 'http://www.w3.org/2001/XMLSchema#float':
            value = float(value)
        elif typ == 'http://www.w3.org/2001/XMLSchema#date':
            value = {'_date': value}
    return value

def main():
    # Get some custom variables from the environment (mostly for testing)
    # PEOPLE_DUMP_LIMIT decides how many people we'll dump to file.
    # PEOPLE_JSON specifies a custom athletes file location.
    max_limit = int(os.environ.get('PEOPLE_DUMP_LIMIT'))
    filename = os.environ.get('PEOPLE_JSON', 'var/athletes.json')
    items = get_fuse_objects(max_limit=max_limit)
    with open(filename, 'w') as f:
        json.dump({'items': items}, f, indent=2)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import unittest
from mbed_greentea.tests_spec import TestSpec, TestBinary

simple_test_spec = {
    "builds": {
        "K64F-ARM": {
            "platform": "K64F",
            "toolchain": "ARM",
            "base_path": "./.build/K64F/ARM",
            "baud_rate": 115200,
            "tests": {
                "mbed-drivers-test-generic_tests":{
                    "binaries":[
                        {
                            "binary_type": "bootable",
                            "path": "./.build/K64F/ARM/mbed-drivers-test-generic_tests.bin"
                        }
                    ]
                },
                "mbed-drivers-test-c_strings":{
                    "binaries":[
                        {
                            "binary_type": "bootable",
                            "path": "./.build/K64F/ARM/mbed-drivers-test-c_strings.bin"
                        }
                    ]
                }
            }
        },
        "K64F-GCC": {
            "platform": "K64F",
            "toolchain": "GCC_ARM",
            "base_path": "./.build/K64F/GCC_ARM",
            "baud_rate": 9600,
            "tests": {
                "mbed-drivers-test-generic_tests":{
                    "binaries":[
                        {
                            "binary_type": "bootable",
                            "path": "./.build/K64F/GCC_ARM/mbed-drivers-test-generic_tests.bin"
                        }
                    ]
                }

            }
        }

    }
}

class TestsSpecFunctionality(unittest.TestCase):

    def setUp(self):
        self.ts_2_builds = simple_test_spec

    def tearDown(self):
        pass

    def test_example(self):
        self.assertEqual(True, True)
        self.assertNotEqual(True, False)

    def test_get_test_builds(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()

        self.assertIs(type(test_builds), list)
        self.assertEqual(len(test_builds), 2)

    def test_get_test_builds_names(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()
        test_builds_names = [x.get_name() for x in self.test_spec.get_test_builds()]

        self.assertEqual(len(test_builds_names), 2)
        self.assertIs(type(test_builds_names), list)

        self.assertIn('K64F-ARM', test_builds_names)
        self.assertIn('K64F-GCC', test_builds_names)

    def test_get_test_build(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()
        test_builds_names = [x.get_name() for x in self.test_spec.get_test_builds()]

        self.assertEqual(len(test_builds_names), 2)
        self.assertIs(type(test_builds_names), list)

        self.assertNotEqual(None, self.test_spec.get_test_build('K64F-ARM'))
        self.assertNotEqual(None, self.test_spec.get_test_build('K64F-GCC'))

    def test_get_build_properties(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()
        test_builds_names = [x.get_name() for x in self.test_spec.get_test_builds()]

        self.assertEqual(len(test_builds_names), 2)
        self.assertIs(type(test_builds_names), list)

        k64f_arm = self.test_spec.get_test_build('K64F-ARM')
        k64f_gcc = self.test_spec.get_test_build('K64F-GCC')

        self.assertNotEqual(None, k64f_arm)
        self.assertNotEqual(None, k64f_gcc)

        self.assertEqual('K64F', k64f_arm.get_platform())
        self.assertEqual('ARM', k64f_arm.get_toolchain())
        self.assertEqual(115200, k64f_arm.get_baudrate())

        self.assertEqual('K64F', k64f_gcc.get_platform())
        self.assertEqual('GCC_ARM', k64f_gcc.get_toolchain())
        self.assertEqual(9600, k64f_gcc.get_baudrate())

    def test_get_test_builds_properties(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()
        test_builds_names = [x.get_name() for x in self.test_spec.get_test_builds()]

        self.assertIn('K64F-ARM', test_builds_names)
        self.assertIn('K64F-GCC', test_builds_names)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Add tests for TestSpec.get_test_builds() parameter 'filter_by_names'<commit_after>#!/usr/bin/env python
"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import unittest
from mbed_greentea.tests_spec import TestSpec, TestBinary

simple_test_spec = {
    "builds": {
        "K64F-ARM": {
            "platform": "K64F",
            "toolchain": "ARM",
            "base_path": "./.build/K64F/ARM",
            "baud_rate": 115200,
            "tests": {
                "mbed-drivers-test-generic_tests":{
                    "binaries":[
                        {
                            "binary_type": "bootable",
                            "path": "./.build/K64F/ARM/mbed-drivers-test-generic_tests.bin"
                        }
                    ]
                },
                "mbed-drivers-test-c_strings":{
                    "binaries":[
                        {
                            "binary_type": "bootable",
                            "path": "./.build/K64F/ARM/mbed-drivers-test-c_strings.bin"
                        }
                    ]
                }
            }
        },
        "K64F-GCC": {
            "platform": "K64F",
            "toolchain": "GCC_ARM",
            "base_path": "./.build/K64F/GCC_ARM",
            "baud_rate": 9600,
            "tests": {
                "mbed-drivers-test-generic_tests":{
                    "binaries":[
                        {
                            "binary_type": "bootable",
                            "path": "./.build/K64F/GCC_ARM/mbed-drivers-test-generic_tests.bin"
                        }
                    ]
                }

            }
        }

    }
}

class TestsSpecFunctionality(unittest.TestCase):

    def setUp(self):
        self.ts_2_builds = simple_test_spec

    def tearDown(self):
        pass

    def test_example(self):
        self.assertEqual(True, True)
        self.assertNotEqual(True, False)

    def test_get_test_builds(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()

        self.assertIs(type(test_builds), list)
        self.assertEqual(len(test_builds), 2)

    def test_get_test_builds_names(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()
        test_builds_names = [x.get_name() for x in self.test_spec.get_test_builds()]

        self.assertEqual(len(test_builds_names), 2)
        self.assertIs(type(test_builds_names), list)

        self.assertIn('K64F-ARM', test_builds_names)
        self.assertIn('K64F-GCC', test_builds_names)

    def test_get_test_build(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()
        test_builds_names = [x.get_name() for x in self.test_spec.get_test_builds()]

        self.assertEqual(len(test_builds_names), 2)
        self.assertIs(type(test_builds_names), list)

        self.assertNotEqual(None, self.test_spec.get_test_build('K64F-ARM'))
        self.assertNotEqual(None, self.test_spec.get_test_build('K64F-GCC'))

    def test_get_build_properties(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()
        test_builds_names = [x.get_name() for x in self.test_spec.get_test_builds()]

        self.assertEqual(len(test_builds_names), 2)
        self.assertIs(type(test_builds_names), list)

        k64f_arm = self.test_spec.get_test_build('K64F-ARM')
        k64f_gcc = self.test_spec.get_test_build('K64F-GCC')

        self.assertNotEqual(None, k64f_arm)
        self.assertNotEqual(None, k64f_gcc)

        self.assertEqual('K64F', k64f_arm.get_platform())
        self.assertEqual('ARM', k64f_arm.get_toolchain())
        self.assertEqual(115200, k64f_arm.get_baudrate())

        self.assertEqual('K64F', k64f_gcc.get_platform())
        self.assertEqual('GCC_ARM', k64f_gcc.get_toolchain())
        self.assertEqual(9600, k64f_gcc.get_baudrate())

    def test_get_test_builds_properties(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)
        test_builds = self.test_spec.get_test_builds()
        test_builds_names = [x.get_name() for x in self.test_spec.get_test_builds()]

        self.assertIn('K64F-ARM', test_builds_names)
        self.assertIn('K64F-GCC', test_builds_names)

    def test_get_test_builds_names_filter_by_names(self):
        self.test_spec = TestSpec()
        self.test_spec.parse(self.ts_2_builds)

        filter_by_names = ['K64F-ARM']
        test_builds = self.test_spec.get_test_builds(filter_by_names=filter_by_names)
        test_builds_names = [x.get_name() for x in test_builds]
        self.assertEqual(len(test_builds_names), 1)
        self.assertIn('K64F-ARM', test_builds_names)

        filter_by_names = ['K64F-GCC']
        test_builds = self.test_spec.get_test_builds(filter_by_names=filter_by_names)
        test_builds_names = [x.get_name() for x in test_builds]
        self.assertEqual(len(test_builds_names), 1)
        self.assertIn('K64F-GCC', test_builds_names)

        filter_by_names = ['SOME-PLATFORM-NAME']
        test_builds = self.test_spec.get_test_builds(filter_by_names=filter_by_names)
        test_builds_names = [x.get_name() for x in test_builds]
        self.assertEqual(len(test_builds_names), 0)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from multiprocessing import Process, Queue

class InvalidArguments(ValueError):
    pass

class Plugin(Process):
    """A Plugin is a self-contained bit of functionality that runs in it's own
    process, and runs via listening for messages and sending messages through
    Queues.
    """
    listeners = set(['SHUTDOWN'])
    messengers = set([])
    name = 'Base Plugin'

    def __new__(cls, *args, **kwargs):
        plug = super(Plugin, cls).__new__(cls, *args, **kwargs)
        plug.listeners.update(Plugin.listeners)
        return plug

    def __init__(self):
        """Set us up to run as a separate process, initialze our listener Queue,
        and set our runnable attribute.
        """
        super(Plugin, self).__init__()
        self.listener = Queue()
        self.runnable = True

    # def set_messenger(self, messenger):
    #     """Set our messenger"""
    #     self.messenger = messenger

    def send(self, message, payload=None):
        """Send a message through our messenger Queue.
        Messages are presumably descriptions of a task that just got completed,
        or a notification of status, or whatnot.
        """
        self.messenger.put((message, payload))

    def recieve(self, message, payload=None):
        """Get a message from our listener Queue.
        This should currently be used in a subclasses self.run loop.
        """
        self.listener.put((message, payload))

    def SHUTDOWN(self, payload):
        """Set self.runnable to false.
        This should cause a subclass to break out of it's run loop.
        """
        self.runnable=False

    def pre_run(self):
        """Code to be run before our run loop starts"""
        pass

    def pre_call_message(self):
        """Code to be run before calling a message handler"""
        pass

    def pre_first_call_message(self):
        """Code to be run before calling the first message handler"""

    def post_first_call_message(self):
        """Code to be run after the first message has been handled"""
        pass

    def post_call_message(self):
        """Code to be run after a message has been handled"""
        pass

    def post_run(self):
        """Code to be run after our run loop terminates"""
        pass

    def run(self):
        """Run our loop, and any defined hooks...
        """
        self.pre_run()
        first = True
        while self.runnable:
            self.pre_call_message()

            if first:
                self.pre_first_call_message()

            message, payload = self.listener.get()
            getattr(self, message)(payload)

            if first:
                first = False
                self.post_first_call_message()

            self.post_call_message()

        self.post_run()

<commit_msg>remove some dead code<commit_after>from multiprocessing import Process, Queue

class InvalidArguments(ValueError):
    pass

class Plugin(Process):
    """A Plugin is a self-contained bit of functionality that runs in it's own
    process, and runs via listening for messages and sending messages through
    Queues.
    """
    listeners = set(['SHUTDOWN'])
    messengers = set([])
    name = 'Base Plugin'

    def __new__(cls, *args, **kwargs):
        plug = super(Plugin, cls).__new__(cls, *args, **kwargs)
        plug.listeners.update(Plugin.listeners)
        return plug

    def __init__(self):
        """Set us up to run as a separate process, initialze our listener Queue,
        and set our runnable attribute.
        """
        super(Plugin, self).__init__()
        self.listener = Queue()
        self.runnable = True

    def send(self, message, payload=None):
        """Send a message through our messenger Queue.
        Messages are presumably descriptions of a task that just got completed,
        or a notification of status, or whatnot.
        """
        self.messenger.put((message, payload))

    def recieve(self, message, payload=None):
        """Get a message from our listener Queue.
        This should currently be used in a subclasses self.run loop.
        """
        self.listener.put((message, payload))

    def SHUTDOWN(self, payload):
        """Set self.runnable to false.
        This should cause a subclass to break out of it's run loop.
        """
        self.runnable=False

    def pre_run(self):
        """Code to be run before our run loop starts"""
        pass

    def pre_call_message(self):
        """Code to be run before calling a message handler"""
        pass

    def pre_first_call_message(self):
        """Code to be run before calling the first message handler"""

    def post_first_call_message(self):
        """Code to be run after the first message has been handled"""
        pass

    def post_call_message(self):
        """Code to be run after a message has been handled"""
        pass

    def post_run(self):
        """Code to be run after our run loop terminates"""
        pass

    def run(self):
        """Run our loop, and any defined hooks...
        """
        self.pre_run()
        first = True
        while self.runnable:
            self.pre_call_message()

            if first:
                self.pre_first_call_message()

            message, payload = self.listener.get()
            getattr(self, message)(payload)

            if first:
                first = False
                self.post_first_call_message()

            self.post_call_message()

        self.post_run()

<|endoftext|>


<commit_before>from django.conf import settings
from django.db import models
from django.db.models import Q
from django.http import Http404
from django.utils import translation
from django.utils.translation import ugettext_lazy as _

import mptt

def get_object(path, fail_silently=False):
    dot = path.rindex('.')
    try:
        return getattr(__import__(path[:dot], {}, {}, ['']), path[dot+1:])
    except ImportError:
        if not fail_silently:
            raise

    return None

class TypeRegistryMetaClass(type):
    """
    You can access the list of subclasses as <BaseClass>.types
    """

    def __init__(cls, name, bases, attrs):
        if not hasattr(cls, 'types'):
            cls.types = []
        else:
            cls.types.append(cls)

class Region(models.Model):
    """
    A template region which will be a container for several page contents.

    Often used regions might be "main" and "sidebar"
    """

    key = models.CharField(_('key'), max_length=20, unique=True)
    title = models.CharField(_('title'), max_length=50, unique=True)
    inherited = models.BooleanField(_('inherited'), default=False,
        help_text=_('Should the content be inherited by subpages if they do not define any content for this region?'))

    class Meta:
        verbose_name = _('region')
        verbose_name_plural = _('regions')

    def __unicode__(self):
        return self.key

class Template(models.Model):
    """
    A template file on the disk which can be used by pages to render themselves.
    """

    title = models.CharField(max_length=200)
    path = models.CharField(max_length=200)
    regions = models.ManyToManyField(Region, related_name='templates')

    class Meta:
        ordering = ['title']
        verbose_name = _('template')
        verbose_name_plural = _('templates')

    def __unicode__(self):
        return self.title

class PagePretender(object):
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

    def get_absolute_url(self):
        return self.url

class NavigationExtension(object):
    __metaclass__ = TypeRegistryMetaClass
    name = _('navigation extension')

    def children(self, page, **kwargs):
        raise NotImplementedError

class PageManager(models.Manager):
    def active(self):
        return self.filter(active=True)

    def page_for_path(self, path, raise404=False):
        """
        Return a page for a path.

        Example:
        Page.objects.page_for_path(request.path)
        """

        stripped = path.strip('/')

        try:
            return self.active().filter(override_url='/%s/' % stripped)[0]
        except IndexError:
            pass

        tokens = stripped.split('/')

        count = len(tokens)

        filters = {'%sisnull' % ('parent__' * count): True}

        for n, token in enumerate(tokens):
            filters['%sslug' % ('parent__' * (count-n-1))] = token

        try:
            return self.active().filter(**filters)[0]
        except IndexError:
            if raise404:
                raise Http404
            raise self.model.DoesNotExist

    def page_for_path_or_404(self, path):
        """
        Wrapper for page_for_path which raises a Http404 if no page
        has been found for the passed path.
        """
        return self.page_for_path(path, raise404=True)

    def best_match_for_path(self, path, raise404=False):
        """
        Return the best match for a path.
        """

        tokens = path.strip('/').split('/')

        for count in range(len(tokens), -1, -1):
            try:
                return self.page_for_path('/'.join(tokens[:count]))
            except self.model.DoesNotExist:
                pass

        if raise404:
            raise Http404
        return None

    def in_navigation(self):
        return self.active().filter(in_navigation=True)

    def toplevel_navigation(self):
        return self.in_navigation().filter(parent__isnull=True)

    def from_request(self, request, raise404=False):
        page = self.page_for_path(request.path, raise404)
        page.setup_request(request)
        return page

    def from_request_or_404(self, request):
        return self.page_for_path_or_404(request.path, raise404=True)

    def best_match_from_request(self, request, raise404=False):
        page = self.best_match_for_path(request.path, raise404)
        page.setup_request(request)
        return page

class Page(models.Model):
    active = models.BooleanField(_('active'), default=False)
    template = models.ForeignKey(Template)

    # structure and navigation
    title = models.CharField(_('title'), max_length=100,
        help_text=_('This is used for the generated navigation too.'))
    slug = models.SlugField()
    parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
    in_navigation = models.BooleanField(_('in navigation'), default=True)
    override_url = models.CharField(_('override URL'), max_length=200, blank=True,
        help_text=_('Override the target URL for the navigation.'))
    redirect_to = models.CharField(_('redirect to'), max_length=200, blank=True,
        help_text=_('Target URL for automatic redirects.'))

    # navigation extensions
    NE_CHOICES = [(
        '%s.%s' % (cls.__module__, cls.__name__), cls.name) for cls in NavigationExtension.types]
    navigation_extension = models.CharField(_('navigation extension'),
        choices=NE_CHOICES, blank=True, max_length=50,
        help_text=_('Select the module providing subpages for this page if you need to customize the navigation.'))

    # content
    _content_title = models.TextField(_('content title'), blank=True,
        help_text=_('The first line is the main title, the following lines are subtitles.'))

    # meta stuff TODO keywords and description?
    _page_title = models.CharField(_('page title'), max_length=100, blank=True,
        help_text=_('Page title for browser window. Same as title by default.'))
    meta_keywords = models.TextField(_('meta keywords'), blank=True,
        help_text=_('This will be prepended to the default keyword list.'))
    meta_description = models.TextField(_('meta description'), blank=True,
        help_text=_('This will be prepended to the default description.'))

    # language
    language = models.CharField(_('language'), max_length=10,
        choices=settings.LANGUAGES)
    translations = models.ManyToManyField('self', blank=True)

    class Meta:
        ordering = ['tree_id', 'lft']
        verbose_name = _('page')
        verbose_name_plural = _('pages')

    objects = PageManager()

    def __unicode__(self):
        return u'%s (%s)' % (self.title, self.get_absolute_url())

    def get_absolute_url(self):
        if self.override_url:
            return self.override_url
        if self.is_root_node():
            return u'/%s/' % (self.slug)
        else:
            return u'/%s/%s/' % ('/'.join([page.slug for page in self.get_ancestors()]), self.slug)

    @property
    def content(self):
        if not hasattr(self, '_content_proxy'):
            self._content_proxy = ContentProxy(self, self.template)

        return self._content_proxy

    @property
    def page_title(self):
        if self._page_title:
            return self._page_title
        return self.content_title

    @property
    def content_title(self):
        if not self._content_title:
            return self.title

        try:
            return self._content_title.splitlines()[0]
        except IndexError:
            return u''

    @property
    def content_subtitle(self):
        return u'\n'.join(self._content_title.splitlines()[1:])

    def setup_request(self, request):
        translation.activate(self.language)
        request.LANGUAGE_CODE = translation.get_language()

    def extended_navigation(self):
        if not self.navigation_extension:
            return []

        cls = get_object(self.navigation_extension, fail_silently=True)
        if not cls:
            return []

        return cls().children(self)

mptt.register(Page)

class ContentProxy(object):
    """
    This proxy offers attribute-style access to the page contents of regions.

    Example:
    >>> page = Page.objects.all()[0]
    >>> page.content.main
    [A list of all page contents which are assigned to the region with key 'main']
    """

    def __init__(self, page, template):
        self.page = page
        self.template = template

    def __getattr__(self, attr):
        """
        Get all page content instances for the specified page and region

        If no page contents could be found for the current page and the region
        has the inherited flag set, this method will go up the ancestor chain
        until either some page contents have found or no ancestors are left.
        """

        try:
            region = self.__dict__['template'].regions.get(key=attr)
        except Region.DoesNotExist:
            return []

        def collect_items(page):
            contents = []
            for cls in PageContent.types:
                queryset = getattr(page, '%s_set' % cls.__name__.lower())
                contents += list(queryset.filter(region=region).select_related(
                    'page', 'region'))

            if not contents and page.parent_id and region.inherited:
                return collect_items(page.parent)

            return contents

        contents = collect_items(self.__dict__['page'])
        contents.sort(key=lambda c: c.ordering)
        return contents

class PageContentRegistry(models.base.ModelBase, TypeRegistryMetaClass):
    pass

class PageContent(models.Model):
    __metaclass__ = PageContentRegistry

    page = models.ForeignKey(Page, related_name='%(class)s_set')
    region = models.ForeignKey(Region)
    ordering = models.IntegerField(_('ordering'), default=0)

    class Meta:
        abstract = True
        ordering = ['ordering']

    def __unicode__(self):
        return u'%s on %s, ordering %s' % (self.region, self.page, self.ordering)

    def render(self, **kwargs):
        render_fn = getattr(self, 'render_%s' % self.region.key, None)

        if render_fn:
            return render_fn(**kwargs)

        raise NotImplementedError

<commit_msg>Reorder fields in Region model<commit_after>from django.conf import settings
from django.db import models
from django.db.models import Q
from django.http import Http404
from django.utils import translation
from django.utils.translation import ugettext_lazy as _

import mptt

def get_object(path, fail_silently=False):
    dot = path.rindex('.')
    try:
        return getattr(__import__(path[:dot], {}, {}, ['']), path[dot+1:])
    except ImportError:
        if not fail_silently:
            raise

    return None

class TypeRegistryMetaClass(type):
    """
    You can access the list of subclasses as <BaseClass>.types
    """

    def __init__(cls, name, bases, attrs):
        if not hasattr(cls, 'types'):
            cls.types = []
        else:
            cls.types.append(cls)

class Region(models.Model):
    """
    A template region which will be a container for several page contents.

    Often used regions might be "main" and "sidebar"
    """

    title = models.CharField(_('title'), max_length=50, unique=True)
    key = models.CharField(_('key'), max_length=20, unique=True)
    inherited = models.BooleanField(_('inherited'), default=False,
        help_text=_('Should the content be inherited by subpages if they do not define any content for this region?'))

    class Meta:
        verbose_name = _('region')
        verbose_name_plural = _('regions')

    def __unicode__(self):
        return self.title

class Template(models.Model):
    """
    A template file on the disk which can be used by pages to render themselves.
    """

    title = models.CharField(max_length=200)
    path = models.CharField(max_length=200)
    regions = models.ManyToManyField(Region, related_name='templates')

    class Meta:
        ordering = ['title']
        verbose_name = _('template')
        verbose_name_plural = _('templates')

    def __unicode__(self):
        return self.title

class PagePretender(object):
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

    def get_absolute_url(self):
        return self.url

class NavigationExtension(object):
    __metaclass__ = TypeRegistryMetaClass
    name = _('navigation extension')

    def children(self, page, **kwargs):
        raise NotImplementedError

class PageManager(models.Manager):
    def active(self):
        return self.filter(active=True)

    def page_for_path(self, path, raise404=False):
        """
        Return a page for a path.

        Example:
        Page.objects.page_for_path(request.path)
        """

        stripped = path.strip('/')

        try:
            return self.active().filter(override_url='/%s/' % stripped)[0]
        except IndexError:
            pass

        tokens = stripped.split('/')

        count = len(tokens)

        filters = {'%sisnull' % ('parent__' * count): True}

        for n, token in enumerate(tokens):
            filters['%sslug' % ('parent__' * (count-n-1))] = token

        try:
            return self.active().filter(**filters)[0]
        except IndexError:
            if raise404:
                raise Http404
            raise self.model.DoesNotExist

    def page_for_path_or_404(self, path):
        """
        Wrapper for page_for_path which raises a Http404 if no page
        has been found for the passed path.
        """
        return self.page_for_path(path, raise404=True)

    def best_match_for_path(self, path, raise404=False):
        """
        Return the best match for a path.
        """

        tokens = path.strip('/').split('/')

        for count in range(len(tokens), -1, -1):
            try:
                return self.page_for_path('/'.join(tokens[:count]))
            except self.model.DoesNotExist:
                pass

        if raise404:
            raise Http404
        return None

    def in_navigation(self):
        return self.active().filter(in_navigation=True)

    def toplevel_navigation(self):
        return self.in_navigation().filter(parent__isnull=True)

    def from_request(self, request, raise404=False):
        page = self.page_for_path(request.path, raise404)
        page.setup_request(request)
        return page

    def from_request_or_404(self, request):
        return self.page_for_path_or_404(request.path, raise404=True)

    def best_match_from_request(self, request, raise404=False):
        page = self.best_match_for_path(request.path, raise404)
        page.setup_request(request)
        return page

class Page(models.Model):
    active = models.BooleanField(_('active'), default=False)
    template = models.ForeignKey(Template)

    # structure and navigation
    title = models.CharField(_('title'), max_length=100,
        help_text=_('This is used for the generated navigation too.'))
    slug = models.SlugField()
    parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
    in_navigation = models.BooleanField(_('in navigation'), default=True)
    override_url = models.CharField(_('override URL'), max_length=200, blank=True,
        help_text=_('Override the target URL for the navigation.'))
    redirect_to = models.CharField(_('redirect to'), max_length=200, blank=True,
        help_text=_('Target URL for automatic redirects.'))

    # navigation extensions
    NE_CHOICES = [(
        '%s.%s' % (cls.__module__, cls.__name__), cls.name) for cls in NavigationExtension.types]
    navigation_extension = models.CharField(_('navigation extension'),
        choices=NE_CHOICES, blank=True, max_length=50,
        help_text=_('Select the module providing subpages for this page if you need to customize the navigation.'))

    # content
    _content_title = models.TextField(_('content title'), blank=True,
        help_text=_('The first line is the main title, the following lines are subtitles.'))

    # meta stuff TODO keywords and description?
    _page_title = models.CharField(_('page title'), max_length=100, blank=True,
        help_text=_('Page title for browser window. Same as title by default.'))
    meta_keywords = models.TextField(_('meta keywords'), blank=True,
        help_text=_('This will be prepended to the default keyword list.'))
    meta_description = models.TextField(_('meta description'), blank=True,
        help_text=_('This will be prepended to the default description.'))

    # language
    language = models.CharField(_('language'), max_length=10,
        choices=settings.LANGUAGES)
    translations = models.ManyToManyField('self', blank=True)

    class Meta:
        ordering = ['tree_id', 'lft']
        verbose_name = _('page')
        verbose_name_plural = _('pages')

    objects = PageManager()

    def __unicode__(self):
        return u'%s (%s)' % (self.title, self.get_absolute_url())

    def get_absolute_url(self):
        if self.override_url:
            return self.override_url
        if self.is_root_node():
            return u'/%s/' % (self.slug)
        else:
            return u'/%s/%s/' % ('/'.join([page.slug for page in self.get_ancestors()]), self.slug)

    @property
    def content(self):
        if not hasattr(self, '_content_proxy'):
            self._content_proxy = ContentProxy(self, self.template)

        return self._content_proxy

    @property
    def page_title(self):
        if self._page_title:
            return self._page_title
        return self.content_title

    @property
    def content_title(self):
        if not self._content_title:
            return self.title

        try:
            return self._content_title.splitlines()[0]
        except IndexError:
            return u''

    @property
    def content_subtitle(self):
        return u'\n'.join(self._content_title.splitlines()[1:])

    def setup_request(self, request):
        translation.activate(self.language)
        request.LANGUAGE_CODE = translation.get_language()

    def extended_navigation(self):
        if not self.navigation_extension:
            return []

        cls = get_object(self.navigation_extension, fail_silently=True)
        if not cls:
            return []

        return cls().children(self)

mptt.register(Page)

class ContentProxy(object):
    """
    This proxy offers attribute-style access to the page contents of regions.

    Example:
    >>> page = Page.objects.all()[0]
    >>> page.content.main
    [A list of all page contents which are assigned to the region with key 'main']
    """

    def __init__(self, page, template):
        self.page = page
        self.template = template

    def __getattr__(self, attr):
        """
        Get all page content instances for the specified page and region

        If no page contents could be found for the current page and the region
        has the inherited flag set, this method will go up the ancestor chain
        until either some page contents have found or no ancestors are left.
        """

        try:
            region = self.__dict__['template'].regions.get(key=attr)
        except Region.DoesNotExist:
            return []

        def collect_items(page):
            contents = []
            for cls in PageContent.types:
                queryset = getattr(page, '%s_set' % cls.__name__.lower())
                contents += list(queryset.filter(region=region).select_related(
                    'page', 'region'))

            if not contents and page.parent_id and region.inherited:
                return collect_items(page.parent)

            return contents

        contents = collect_items(self.__dict__['page'])
        contents.sort(key=lambda c: c.ordering)
        return contents

class PageContentRegistry(models.base.ModelBase, TypeRegistryMetaClass):
    pass

class PageContent(models.Model):
    __metaclass__ = PageContentRegistry

    page = models.ForeignKey(Page, related_name='%(class)s_set')
    region = models.ForeignKey(Region)
    ordering = models.IntegerField(_('ordering'), default=0)

    class Meta:
        abstract = True
        ordering = ['ordering']

    def __unicode__(self):
        return u'%s on %s, ordering %s' % (self.region, self.page, self.ordering)

    def render(self, **kwargs):
        render_fn = getattr(self, 'render_%s' % self.region.key, None)

        if render_fn:
            return render_fn(**kwargs)

        raise NotImplementedError

<|endoftext|>


<commit_before>from form_designer import settings as app_settings
from django.core.files.base import File
from django.utils.translation import ugettext_lazy as _
from django.db.models.fields.files import FieldFile
from django.template.defaultfilters import filesizeformat
import os
import hashlib, uuid

def get_storage():
    return app_settings.FILE_STORAGE_CLASS()

def clean_files(form):
    for field in form.file_fields:
        uploaded_file = form.cleaned_data[field.name]
        msg = None
        if not os.path.splitext(uploaded_file.name)[1].lstrip('.').lower() in  \
            app_settings.ALLOWED_FILE_TYPES:
                msg = _('This file type is not allowed.')
        elif uploaded_file._size > app_settings.MAX_UPLOAD_SIZE:
            msg = _('Please keep file size under %s. Current size is %s.') %  \
                (filesizeformat(app_settings.MAX_UPLOAD_SIZE), filesizeformat(uploaded_file._size))
        if msg:
            form._errors[field.name] = form.error_class([msg])

    return form.cleaned_data

def handle_uploaded_files(form_definition, form):
    files = []
    if form_definition.save_uploaded_files and len(form.file_fields):
        storage = get_storage()
        secret_hash = hashlib.sha1(str(uuid.uuid4())).hexdigest()[:10]
        for field in form.file_fields:
            uploaded_file = form.cleaned_data[field.name]
            root, ext = os.path.splitext(uploaded_file.name)
            filename = storage.get_available_name(
                os.path.join(app_settings.FILE_STORAGE_DIR,
                form_definition.name,
                '%s_%s%s' % (root, secret_hash, ext)))
            storage.save(filename, uploaded_file)
            file_obj = StoredUploadedFile(filename)
            form.cleaned_data[field.name] = StoredUploadedFile(filename)
            files.append(storage.path(filename))
    return files

class StoredUploadedFile(FieldFile):
    """
    A wrapper for uploaded files that is compatible to the FieldFile class, i.e.
    you can use instances of this class in templates just like you use the value
    of FileFields (e.g. `{{ my_file.url }}`)
    """
    def __init__(self, name):
        File.__init__(self, None, name)
        self.field = self
        self.is_icon = True

    @property
    def storage(self):
        return get_storage()

    def save(self, *args, **kwargs):
        raise NotImplementedError('Static files are read-only')

    def delete(self, *args, **kwargs):
        raise NotImplementedError('Static files are read-only')

    def __unicode__(self):
        return self.name
<commit_msg>Make sure submitting an empty FileField works and display an error if its required.<commit_after>from form_designer import settings as app_settings
from django.core.files.base import File
from django.utils.translation import ugettext_lazy as _
from django.db.models.fields.files import FieldFile
from django.template.defaultfilters import filesizeformat
import os
import hashlib, uuid

def get_storage():
    return app_settings.FILE_STORAGE_CLASS()

def clean_files(form):
    for field in form.file_fields:
        uploaded_file = form.cleaned_data.get(field.name, None)
        msg = None
        if uploaded_file is None:
            if field.required:
                msg = _('This field is required.')
            else:
                continue
        elif not os.path.splitext(uploaded_file.name)[1].lstrip('.').lower() in  \
            app_settings.ALLOWED_FILE_TYPES:
                msg = _('This file type is not allowed.')
        elif uploaded_file._size > app_settings.MAX_UPLOAD_SIZE:
            msg = _('Please keep file size under %s. Current size is %s.') %  \
                (filesizeformat(app_settings.MAX_UPLOAD_SIZE), filesizeformat(uploaded_file._size))
        if msg:
            form._errors[field.name] = form.error_class([msg])

    return form.cleaned_data

def handle_uploaded_files(form_definition, form):
    files = []
    if form_definition.save_uploaded_files and len(form.file_fields):
        storage = get_storage()
        secret_hash = hashlib.sha1(str(uuid.uuid4())).hexdigest()[:10]
        for field in form.file_fields:
            uploaded_file = form.cleaned_data.get(field.name, None)
            if uploaded_file is None:
                continue
            root, ext = os.path.splitext(uploaded_file.name)
            filename = storage.get_available_name(
                os.path.join(app_settings.FILE_STORAGE_DIR,
                form_definition.name,
                '%s_%s%s' % (root, secret_hash, ext)))
            storage.save(filename, uploaded_file)
            form.cleaned_data[field.name] = StoredUploadedFile(filename)
            files.append(storage.path(filename))
    return files

class StoredUploadedFile(FieldFile):
    """
    A wrapper for uploaded files that is compatible to the FieldFile class, i.e.
    you can use instances of this class in templates just like you use the value
    of FileFields (e.g. `{{ my_file.url }}`)
    """
    def __init__(self, name):
        File.__init__(self, None, name)
        self.field = self
        self.is_icon = True

    @property
    def storage(self):
        return get_storage()

    def save(self, *args, **kwargs):
        raise NotImplementedError('Static files are read-only')

    def delete(self, *args, **kwargs):
        raise NotImplementedError('Static files are read-only')

    def __unicode__(self):
        return self.name
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Validators
==========

.. autoclass: GTEValidator
    :members:

"""

from django.core import validators
from django.utils.translation import ugettext_lazy as _

class GTValidator(validators.BaseValidator):

    message = _("Ensure this field is greater than %(limit_value)s")

    def clean(self, x):
        return int(x)

    def compare(self, cleaned, limit_value):
        return cleaned <= limit_value

class LTValidator(validators.BaseValidator):

    message = _("Ensure this field is lesser than %(limit_value)s")

    def clean(self, x):
        return int(x)

    def compare(self, cleaned, limit_value):
        return cleaned >= limit_value

class ValidatorFactory(object):

    maps = {
        'MINLENGTH': lambda self, x, y=None: self.min_length(x, y),
        'MAXLENGTH': lambda self, x, y=None: self.max_length(x, y),
        'REGEXP': lambda self, x, y=None: self.regexp(x, y),
        'GT': lambda self, x, y=None: self.gt(x, y),
        'GTE': lambda self, x, y=None: self.gte(x, y),
        'LT': lambda self, x, y=None: self.lt(x, y),
        'LTE': lambda self, x, y=None: self.lte(x, y),
        'EQ': lambda self, x, y=None: self.eq(x, y),
        'NEQ': lambda self, x, y=None: self.neq(x, y),
    }

    def min_length(self, limit_value, message):
        limit_value = int(limit_value)
        return validators.MinLengthValidator(limit_value, message)

    def max_length(self, limit_value, message):
        limit_value = int(limit_value)
        return validators.MaxLengthValidator(limit_value, message)

    def regexp(self, value, message):
        return validators.RegexValidator(regex=value, message=message)

    def gt(self, value, message):
        return GTValidator(int(value), message)

    def gte(self, value, message):
        return validators.MinValueValidator(int(value), message)

    def lt(self, value, message):
        return LTValidator(int(value), message)

    def lte(self, value, message):
        return validators.MaxValueValidator(int(value), message)

    def produce(self, validation):
        meth = self.maps[validation.type]
        msg = validation.message or None
        return meth(self, validation.value, msg)
<commit_msg>refs #41, added DateValidatorFactory<commit_after># -*- coding: utf-8 -*-
"""
Validators
==========

.. autoclass: GTEValidator
    :members:

"""

from dateutil.parser import parse

from django.core import validators
from django.utils.translation import ugettext_lazy as _

class GTValidator(validators.BaseValidator):

    message = _("Ensure this field is greater than %(limit_value)s")

    def clean(self, x):
        return x

    def compare(self, cleaned, limit_value):
        return cleaned <= limit_value

class LTValidator(validators.BaseValidator):

    message = _("Ensure this field is lesser than %(limit_value)s")

    def clean(self, x):
        return x

    def compare(self, cleaned, limit_value):
        return cleaned >= limit_value

class DateValidator(object):

    def __init__(self, limit_value, message=None):
        return super(DateValidator, self).__init__(
            parse(limit_value).date(), message
        )

class DateGTValidator(DateValidator, GTValidator):
    pass

class DateLTValidator(DateValidator, LTValidator):
    pass

class DateMaxValueValidator(DateValidator, validators.MaxValueValidator):
    pass

class DateMinValueValidator(DateValidator, validators.MinValueValidator):
    pass

class ValidatorFactory(object):

    maps = {
        'MINLENGTH': lambda self, x, y=None: self.min_length(x, y),
        'MAXLENGTH': lambda self, x, y=None: self.max_length(x, y),
        'REGEXP': lambda self, x, y=None: self.regexp(x, y),
        'GT': lambda self, x, y=None: self.gt(x, y),
        'GTE': lambda self, x, y=None: self.gte(x, y),
        'LT': lambda self, x, y=None: self.lt(x, y),
        'LTE': lambda self, x, y=None: self.lte(x, y),
        'EQ': lambda self, x, y=None: self.eq(x, y),
        'NEQ': lambda self, x, y=None: self.neq(x, y),
    }

    def min_length(self, limit_value, message):
        limit_value = int(limit_value)
        return validators.MinLengthValidator(limit_value, message)

    def max_length(self, limit_value, message):
        limit_value = int(limit_value)
        return validators.MaxLengthValidator(limit_value, message)

    def regexp(self, value, message):
        return validators.RegexValidator(regex=value, message=message)

    def gt(self, value, message):
        return GTValidator(int(value), message)

    def gte(self, value, message):
        return validators.MinValueValidator(int(value), message)

    def lt(self, value, message):
        return LTValidator(int(value), message)

    def lte(self, value, message):
        return validators.MaxValueValidator(int(value), message)

    def produce(self, validation):
        meth = self.maps[validation.type]
        msg = validation.message or None
        return meth(self, validation.value, msg)

class DateValidatorFactory(ValidatorFactory):

    def lt(self, value, message):
        return DateLTValidator(value, message)

    def lte(self, value, message):
        return DateMaxValueValidator(value, message)
<|endoftext|>


<commit_before>""" The aim of this file is to contain all the function
and the main which have to do with emotion recognition, especially
with the Kanade database."""

import glob
import argparse
# import DimensionalityReduction
import cPickle as pickle
from sklearn import cross_validation

import os
import fnmatch
import matplotlib.image as io
import matplotlib.pyplot as plt
import numpy as np
from skimage.transform import resize
from skimage import io
from skimage import color
import os

import facedetection

import deepbelief as db
import restrictedBoltzmannMachine as rbm
from common import *

parser = argparse.ArgumentParser(description='digit recognition')
parser.add_argument('--save',dest='save',action='store_true', default=False,
                    help="if true, the network is serialized and saved")
parser.add_argument('--train',dest='train',action='store_true', default=False,
                    help=("if true, the network is trained from scratch from the"
                          "traning data"))
parser.add_argument('--rbm', dest='rbm',action='store_true', default=False,
                    help=("if true, the code for traning an rbm on the data is run"))
parser.add_argument('--db', dest='db',action='store_true', default=False,
                    help=("if true, the code for traning a deepbelief net on the"
                          "data is run"))
parser.add_argument('--trainSize', type=int, default=10000,
                    help='the number of tranining cases to be considered')
parser.add_argument('--testSize', type=int, default=1000,
                    help='the number of testing cases to be considered')
parser.add_argument('netFile', help="file where the serialized network should be saved")
parser.add_argument('--nesterov', dest='nesterov',action='store_true', default=False,
                    help=("if true, the deep belief net is trained using nesterov momentum"))
parser.add_argument('--rmsprop', dest='rmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the deep belief net."))
parser.add_argument('--cv', dest='cv',action='store_true', default=False,
                    help=("if true, do cross validation"))
parser.add_argument('--facedetection', dest='facedetection',action='store_true', default=False,
                    help=("if true, do face detection"))

# DEBUG mode?
parser.add_argument('--debug', dest='debug',action='store_false', default=False,
                    help=("if true, the deep belief net is ran in DEBUG mode"))

# Get the arguments of the program
args = parser.parse_args()

# Set the debug mode in the deep belief net
db.DEBUG = args.debug

SMALL_SIZE = ((40, 30))

def rbmEmotions(big=False, reconstructRandom=False):
  data, labels = readKanade(big)
  print "data.shape"
  print data.shape

  trainData = data[0:-1, :]

  # Train the network
  if args.train:
    # The number of hidden units is taken from a deep learning tutorial
    # The data are the values of the images have to be normalized before being
    # presented to the network
    nrVisible = len(data[0])
    nrHidden = 800
    # use 1 dropout to test the rbm for now
    net = rbm.RBM(nrVisible, nrHidden, 0.01, 1, 1)
    net.train(trainData)
    t = visualizeWeights(net.weights.T, SMALL_SIZE, (10,10))
  else:
    # Take the saved network and use that for reconstructions
    f = open(args.netFile, "rb")
    t = pickle.load(f)
    net = pickle.load(f)
    f.close()

  # get a random image and see it looks like
  # if reconstructRandom:
  #   test = np.random.random_sample(test.shape)

  # Show the initial image first
  test = data[-1, :]
  print "test.shape"
  print test.shape

  plt.imshow(vectorToImage(test, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.show()

  recon = net.reconstruct(test.reshape(1, test.shape[0]))
  print recon.shape

  plt.imshow(vectorToImage(recon, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.show()

  # Show the weights and their form in a tile fashion
  # Plot the weights
  plt.imshow(t, cmap=plt.cm.gray)
  plt.axis('off')
  plt.show()
  print "done"

  if args.save:
    f = open(args.netFile, "wb")
    pickle.dump(t, f)
    pickle.dump(net, f)

"""
  Arguments:
    big: should the big or small images be used?
    folds: which folds should be used (1,..5) (a list). If None is passed all
    folds are used
"""
def deepBeliefKanadeCV(big=False, folds=None):
  data, labels = readKanade(big, folds)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  unsupervisedData = buildUnsupervisedDataSet()

  kf = cross_validation.KFold(n=len(data), k=len(folds))
  bestCorrect = 0
  bestProbs = 0

  # bestParam
  # 0.001
  # bestProbs
  # 0.3703703703
  # TODO: try boosting for CV in order to increase the number of folds
  params = [15, 20, 25, 30, 40]
  fold = 0
  for train, test in kf:

    trainData = data[train]
    trainLabels = labels[train]

    # TODO: this might require more thought
    net = db.DBN(5, [1200, 1000, 1000, 1000, 7],
               unsupervisedLearningRate=0.01,
               supervisedLearningRate=0.001,
               nesterovMomentum=args.nesterov,
               rmsprop=args.rmsprop,
               hiddenDropout=0.5, rbmHiddenDropout=0.5, visibleDropout=0.8,
               rbmVisibleDropout=1,
               normConstraint=params[i])

    net.train(trainData, trainLabels, unsupervisedData=unsupervisedData)

    probs, predicted = net.classify(data[test])

    actualLabels = labels[test]
    correct = 0
    errorCases = []

    for i in xrange(len(test)):
      print "predicted"
      print "probs"
      print probs[i]
      print predicted[i]
      print "actual"
      actual = actualLabels[i]
      print actual
      if predicted[i] == np.argmax(actual):
        correct += 1
      else:
        errorCases.append(i)

    print "correct for " + str(params[fold])
    print correct

    if bestCorrect < correct:
      bestCorrect = correct
      bestParam = params[fold]
      bestProbs = correct * 1.0 / len(test)

    fold += 1

  print "bestParam"
  print bestParam

  print "bestProbs"
  print bestProbs

def deepBeliefKanade(big=False):
  data, labels = readKanade(big, None)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), n_folds=5)
  for train, test in kf:
    break

  trainData = data[train]
  trainLabels = labels[train]

  # TODO: this might require more thought
  net = db.DBN(5, [1200, 1000, 1000, 1000, 7],
             unsupervisedLearningRate=0.01,
             # is this not a bad learning rate?
             supervisedLearningRate=0.001,
             nesterovMomentum=args.nesterov,
             rmsprop=args.rmsprop,
             hiddenDropout=0.5, rbmHiddenDropout=0.5, visibleDropout=0.8,
             rbmVisibleDropout=1)

  unsupervisedData = buildUnsupervisedDataSet()

  net.train(trainData, trainLabels, unsupervisedData=unsupervisedData)

  probs, predicted = net.classify(data[test])

  actualLabels = labels[test]
  correct = 0
  errorCases = []

  for i in xrange(len(test)):
    print "predicted"
    print "probs"
    print probs[i]
    print predicted[i]
    print "actual"
    actual = actualLabels[i]
    print actual
    if predicted[i] == np.argmax(actual):
      correct += 1
    else:
      errorCases.append(i)

  print "correct"
  print correct

  print "percentage correct"
  print correct  * 1.0/ len(test)

def buildUnsupervisedDataSet():
  return np.vstack((
    # readCroppedYale(),
    readAttData(),
    readJaffe(),
    # readNottingham(),
    readAberdeen()))

def readKanade(big=False, folds=None):
  if big:
    files = glob.glob('kanade_150*.pickle')
  else:
    files = glob.glob('kanade_f*.pickle')

  if not folds:
    folds = range(1, 6)

  # Read the data from them. Sort out the files that do not have
  # the folds that we want
  # TODO: do this better (with regex in the file name)
  # DO not reply on the order returned
  files = [ files[x -1] for x in folds]

  data = np.array([])
  labels = np.array([])
  # TODO: do LDA on the training data

  # TODO: do proper CV in which you use 4 folds for training and one for testing
  # at that time
  dataFolds = []
  labelFolds = []
  for filename in files:
    with open(filename, "rb") as  f:
      # Sort out the labels from the data
      # TODO: run the readKanade again tomorrow and change these idnices here
      dataAndLabels = pickle.load(f)
      foldData = dataAndLabels[:, 0:-1]
      print "foldData.shape"
      print foldData.shape
      foldLabels = dataAndLabels[:,-1]
      dataFolds.append(foldData)
      foldLabels = np.array(map(int, foldLabels))

      vectorLabels = labelsToVectors(foldLabels -1, 7)
      labelFolds.append(vectorLabels)

      print "foldLabels.shape"
      print vectorLabels.shape

  data =  np.vstack(tuple(dataFolds))
  labels = np.vstack(tuple(labelFolds))
  return data, labels

# TODO: get big, small as argument in order to be able to fit the resizing
def readCroppedYale():
  # PATH = "/data/mcr10/yaleb/CroppedYale"
  PATH = "/home/aela/uni/project/CroppedYale"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  # Filter out the ones that containt "amyes bient"
  imageFiles = [ x for x in imageFiles if not "Ambient" in x]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)

    images += [img.reshape(-1)]

  return np.array(images)

def readAttData():
  PATH = "/data/mcr10/att"
  # PATH = "/home/aela/uni/project/code/pics/cambrdige_pics"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)
    images += [img.reshape(-1)]

  return np.array(images)

def readAndCrop(path, extension, doRecognition, isColoured=False):
  dirforres = "detection-cropped"
  pathForCropped = os.path.join(path, dirforres)

  if doRecognition:
    if not os.path.exists(pathForCropped):
      os.makedirs(pathForCropped)

    imageFiles = [(os.path.join(dirpath, f), f)
      for dirpath, dirnames, files in os.walk(path)
      for f in fnmatch.filter(files, '*.' + extension)]

    images = []

    for fullPath, shortPath in imageFiles:
      # Do not do this for already cropped images
      if pathForCropped in fullPath:
        continue

      print fullPath
      img = io.imread(fullPath)
      if isColoured:
        img = color.rgb2gray(img)
        img = np.array(img * 255, dtype='uint8')

      face = facedetection.cropFace(img)
      if not face == None:
        # Only do the resizing once you are done with the cropping of the faces
        face = resize(face, SMALL_SIZE)
        # Check that you are always saving them in the right format
        print "face.min"
        print face.min()

        print "face.max"
        print face.max()

        assert face.min() >=0 and face.max() <=1
        images += [face.reshape(-1)]

        # Save faces as files
        croppedFileName = os.path.join(pathForCropped, shortPath)
        io.imsave(croppedFileName, face)

  else:
    images = []
    imageFiles = [os.path.join(dirpath, f)
      for dirpath, dirnames, files in os.walk(pathForCropped)
      for f in fnmatch.filter(files, '*.' + extension)]

    for f in imageFiles:
      img = io.imread(f)
      if not (img.min() >=0 and img.max()<=1.000008):
        img = img / 255.0
      images += [img.reshape(-1)]

  print len(images)
  return np.array(images)

# This needs some thought: remove the cropped folder from path?

def readJaffe():
  PATH = "/data/mcr10/jaffe"
  # PATH = "/home/aela/uni/project/jaffe"
  return readAndCrop(PATH , "tiff", args.facedetection, isColoured=False)

def readNottingham():
  PATH = "/home/aela/uni/project/nottingham"
  # PATH = "/data/mcr10/nottingham"
  return readAndCrop(PATH, "gif", args.facedetection, isColoured=False)

def readAberdeen():
  PATH = "/data/mcr10/Aberdeen"
  # PATH = "/home/aela/uni/project/Aberdeen"
  return readAndCrop(PATH, "jpg", args.facedetection, isColoured=True)

def main():
  # deepBeliefKanade()

  # readNottingham()
  # readCroppedYale()
  # readJaffe()
  # readAttData()
  # readAberdeen()
  if args.rbm:
    rbmEmotions()
  elif args.cv:
    deepBeliefKanadeCV()
  elif args.db:
    deepBeliefKanade()

# You can also group the emotions into positive and negative to see
# if you can get better results (probably yes)
if __name__ == '__main__':
  import random
  print "FIXING RANDOMNESS"
  random.seed(6)
  np.random.seed(6)
  main()<commit_msg>fixed cv for emotions<commit_after>""" The aim of this file is to contain all the function
and the main which have to do with emotion recognition, especially
with the Kanade database."""

import glob
import argparse
# import DimensionalityReduction
import cPickle as pickle
from sklearn import cross_validation

import os
import fnmatch
import matplotlib.image as io
import matplotlib.pyplot as plt
import numpy as np
from skimage.transform import resize
from skimage import io
from skimage import color
import os

import facedetection

import deepbelief as db
import restrictedBoltzmannMachine as rbm
from common import *

parser = argparse.ArgumentParser(description='digit recognition')
parser.add_argument('--save',dest='save',action='store_true', default=False,
                    help="if true, the network is serialized and saved")
parser.add_argument('--train',dest='train',action='store_true', default=False,
                    help=("if true, the network is trained from scratch from the"
                          "traning data"))
parser.add_argument('--rbm', dest='rbm',action='store_true', default=False,
                    help=("if true, the code for traning an rbm on the data is run"))
parser.add_argument('--db', dest='db',action='store_true', default=False,
                    help=("if true, the code for traning a deepbelief net on the"
                          "data is run"))
parser.add_argument('--trainSize', type=int, default=10000,
                    help='the number of tranining cases to be considered')
parser.add_argument('--testSize', type=int, default=1000,
                    help='the number of testing cases to be considered')
parser.add_argument('netFile', help="file where the serialized network should be saved")
parser.add_argument('--nesterov', dest='nesterov',action='store_true', default=False,
                    help=("if true, the deep belief net is trained using nesterov momentum"))
parser.add_argument('--rmsprop', dest='rmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the deep belief net."))
parser.add_argument('--cv', dest='cv',action='store_true', default=False,
                    help=("if true, do cross validation"))
parser.add_argument('--facedetection', dest='facedetection',action='store_true', default=False,
                    help=("if true, do face detection"))

# DEBUG mode?
parser.add_argument('--debug', dest='debug',action='store_false', default=False,
                    help=("if true, the deep belief net is ran in DEBUG mode"))

# Get the arguments of the program
args = parser.parse_args()

# Set the debug mode in the deep belief net
db.DEBUG = args.debug

SMALL_SIZE = ((40, 30))

def rbmEmotions(big=False, reconstructRandom=False):
  data, labels = readKanade(big)
  print "data.shape"
  print data.shape

  trainData = data[0:-1, :]

  # Train the network
  if args.train:
    # The number of hidden units is taken from a deep learning tutorial
    # The data are the values of the images have to be normalized before being
    # presented to the network
    nrVisible = len(data[0])
    nrHidden = 800
    # use 1 dropout to test the rbm for now
    net = rbm.RBM(nrVisible, nrHidden, 0.01, 1, 1)
    net.train(trainData)
    t = visualizeWeights(net.weights.T, SMALL_SIZE, (10,10))
  else:
    # Take the saved network and use that for reconstructions
    f = open(args.netFile, "rb")
    t = pickle.load(f)
    net = pickle.load(f)
    f.close()

  # get a random image and see it looks like
  # if reconstructRandom:
  #   test = np.random.random_sample(test.shape)

  # Show the initial image first
  test = data[-1, :]
  print "test.shape"
  print test.shape

  plt.imshow(vectorToImage(test, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.show()

  recon = net.reconstruct(test.reshape(1, test.shape[0]))
  print recon.shape

  plt.imshow(vectorToImage(recon, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.show()

  # Show the weights and their form in a tile fashion
  # Plot the weights
  plt.imshow(t, cmap=plt.cm.gray)
  plt.axis('off')
  plt.show()
  print "done"

  if args.save:
    f = open(args.netFile, "wb")
    pickle.dump(t, f)
    pickle.dump(net, f)

"""
  Arguments:
    big: should the big or small images be used?
"""
def deepBeliefKanadeCV(big=False):
  data, labels = readKanade(big)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape
  # bestParam
  # 0.001
  # bestProbs
  # 0.3703703703
  # TODO: try boosting for CV in order to increase the number of folds
  params = [15, 20, 25, 30, 40]

  unsupervisedData = buildUnsupervisedDataSet()

  kf = cross_validation.KFold(n=len(data), k=len(params))
  bestCorrect = 0
  bestProbs = 0

  fold = 0
  for train, test in kf:

    trainData = data[train]
    trainLabels = labels[train]

    # TODO: this might require more thought
    net = db.DBN(5, [1200, 1000, 1000, 1000, 7],
               unsupervisedLearningRate=0.01,
               supervisedLearningRate=0.001,
               nesterovMomentum=args.nesterov,
               rmsprop=args.rmsprop,
               hiddenDropout=0.5, rbmHiddenDropout=0.5, visibleDropout=0.8,
               rbmVisibleDropout=1,
               normConstraint=params[i])

    net.train(trainData, trainLabels, unsupervisedData=unsupervisedData)

    probs, predicted = net.classify(data[test])

    actualLabels = labels[test]
    correct = 0
    errorCases = []

    for i in xrange(len(test)):
      print "predicted"
      print "probs"
      print probs[i]
      print predicted[i]
      print "actual"
      actual = actualLabels[i]
      print actual
      if predicted[i] == np.argmax(actual):
        correct += 1
      else:
        errorCases.append(i)

    print "correct for " + str(params[fold])
    print correct

    if bestCorrect < correct:
      bestCorrect = correct
      bestParam = params[fold]
      bestProbs = correct * 1.0 / len(test)

    fold += 1

  print "bestParam"
  print bestParam

  print "bestProbs"
  print bestProbs

def deepBeliefKanade(big=False):
  data, labels = readKanade(big, None)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), n_folds=5)
  for train, test in kf:
    break

  trainData = data[train]
  trainLabels = labels[train]

  # TODO: this might require more thought
  net = db.DBN(5, [1200, 1000, 1000, 1000, 7],
             unsupervisedLearningRate=0.01,
             # is this not a bad learning rate?
             supervisedLearningRate=0.001,
             nesterovMomentum=args.nesterov,
             rmsprop=args.rmsprop,
             hiddenDropout=0.5, rbmHiddenDropout=0.5, visibleDropout=0.8,
             rbmVisibleDropout=1)

  unsupervisedData = buildUnsupervisedDataSet()

  net.train(trainData, trainLabels, unsupervisedData=unsupervisedData)

  probs, predicted = net.classify(data[test])

  actualLabels = labels[test]
  correct = 0
  errorCases = []

  for i in xrange(len(test)):
    print "predicted"
    print "probs"
    print probs[i]
    print predicted[i]
    print "actual"
    actual = actualLabels[i]
    print actual
    if predicted[i] == np.argmax(actual):
      correct += 1
    else:
      errorCases.append(i)

  print "correct"
  print correct

  print "percentage correct"
  print correct  * 1.0/ len(test)

def buildUnsupervisedDataSet():
  return np.vstack((
    # readCroppedYale(),
    readAttData(),
    readJaffe(),
    # readNottingham(),
    readAberdeen()))

def readKanade(big=False, folds=None):
  if big:
    files = glob.glob('kanade_150*.pickle')
  else:
    files = glob.glob('kanade_f*.pickle')

  if not folds:
    folds = range(1, 6)

  # Read the data from them. Sort out the files that do not have
  # the folds that we want
  # TODO: do this better (with regex in the file name)
  # DO not reply on the order returned
  files = [ files[x -1] for x in folds]

  data = np.array([])
  labels = np.array([])
  # TODO: do LDA on the training data

  # TODO: do proper CV in which you use 4 folds for training and one for testing
  # at that time
  dataFolds = []
  labelFolds = []
  for filename in files:
    with open(filename, "rb") as  f:
      # Sort out the labels from the data
      # TODO: run the readKanade again tomorrow and change these idnices here
      dataAndLabels = pickle.load(f)
      foldData = dataAndLabels[:, 0:-1]
      print "foldData.shape"
      print foldData.shape
      foldLabels = dataAndLabels[:,-1]
      dataFolds.append(foldData)
      foldLabels = np.array(map(int, foldLabels))

      vectorLabels = labelsToVectors(foldLabels -1, 7)
      labelFolds.append(vectorLabels)

      print "foldLabels.shape"
      print vectorLabels.shape

  data =  np.vstack(tuple(dataFolds))
  labels = np.vstack(tuple(labelFolds))
  return data, labels

# TODO: get big, small as argument in order to be able to fit the resizing
def readCroppedYale():
  # PATH = "/data/mcr10/yaleb/CroppedYale"
  PATH = "/home/aela/uni/project/CroppedYale"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  # Filter out the ones that containt "amyes bient"
  imageFiles = [ x for x in imageFiles if not "Ambient" in x]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)

    images += [img.reshape(-1)]

  return np.array(images)

def readAttData():
  PATH = "/data/mcr10/att"
  # PATH = "/home/aela/uni/project/code/pics/cambrdige_pics"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)
    images += [img.reshape(-1)]

  return np.array(images)

def readAndCrop(path, extension, doRecognition, isColoured=False):
  dirforres = "detection-cropped"
  pathForCropped = os.path.join(path, dirforres)

  if doRecognition:
    if not os.path.exists(pathForCropped):
      os.makedirs(pathForCropped)

    imageFiles = [(os.path.join(dirpath, f), f)
      for dirpath, dirnames, files in os.walk(path)
      for f in fnmatch.filter(files, '*.' + extension)]

    images = []

    for fullPath, shortPath in imageFiles:
      # Do not do this for already cropped images
      if pathForCropped in fullPath:
        continue

      print fullPath
      img = io.imread(fullPath)
      if isColoured:
        img = color.rgb2gray(img)
        img = np.array(img * 255, dtype='uint8')

      face = facedetection.cropFace(img)
      if not face == None:
        # Only do the resizing once you are done with the cropping of the faces
        face = resize(face, SMALL_SIZE)
        # Check that you are always saving them in the right format
        print "face.min"
        print face.min()

        print "face.max"
        print face.max()

        assert face.min() >=0 and face.max() <=1
        images += [face.reshape(-1)]

        # Save faces as files
        croppedFileName = os.path.join(pathForCropped, shortPath)
        io.imsave(croppedFileName, face)

  else:
    images = []
    imageFiles = [os.path.join(dirpath, f)
      for dirpath, dirnames, files in os.walk(pathForCropped)
      for f in fnmatch.filter(files, '*.' + extension)]

    for f in imageFiles:
      img = io.imread(f)
      if not (img.min() >=0 and img.max()<=1.000008):
        img = img / 255.0
      images += [img.reshape(-1)]

  print len(images)
  return np.array(images)

# This needs some thought: remove the cropped folder from path?

def readJaffe():
  PATH = "/data/mcr10/jaffe"
  # PATH = "/home/aela/uni/project/jaffe"
  return readAndCrop(PATH , "tiff", args.facedetection, isColoured=False)

def readNottingham():
  PATH = "/home/aela/uni/project/nottingham"
  # PATH = "/data/mcr10/nottingham"
  return readAndCrop(PATH, "gif", args.facedetection, isColoured=False)

def readAberdeen():
  PATH = "/data/mcr10/Aberdeen"
  # PATH = "/home/aela/uni/project/Aberdeen"
  return readAndCrop(PATH, "jpg", args.facedetection, isColoured=True)

def main():
  # deepBeliefKanade()

  # readNottingham()
  # readCroppedYale()
  # readJaffe()
  # readAttData()
  # readAberdeen()
  if args.rbm:
    rbmEmotions()
  elif args.cv:
    deepBeliefKanadeCV()
  elif args.db:
    deepBeliefKanade()

# You can also group the emotions into positive and negative to see
# if you can get better results (probably yes)
if __name__ == '__main__':
  import random
  print "FIXING RANDOMNESS"
  random.seed(6)
  np.random.seed(6)
  main()<|endoftext|>