<commit_before># -*- coding:utf-8 -*-
"""
Model related utilities.

"""

import importlib
import model
import numpy as np

from brian import StateMonitor, SpikeMonitor
from utils import path_to_modline

def set_model_ps(filepath, dicname='PARAMETERS'):
    """Set the model parameters given the parameter set in `filepath`."""
    psmod = importlib.import_module(path_to_modline(filepath))
    model.PARAMETERS = getattr(psmod, dicname)

def intrapop_connections(n_mitral, n_granule, n_subpop, n_mitral_per_subpop):
    """Connection matrix for intra sub-population connections."""
    resmat = np.zeros((n_mitral, n_granule))
    for i_subpop in xrange(n_subpop):
        start = i_subpop*n_mitral_per_subpop
        stop  = start + n_mitral_per_subpop
        resmat[start:stop, i_subpop] = 1.
    return resmat

def interpop_connections(mat_connections, n_mitral, n_subpop, n_mitral_per_subpop, inter_conn_rate, inter_conn_strength, homeostasy=False):
    """
    Adds inter sub-population connections.

    Changes on the connection matrix are done in place.

    """
    if homeostasy:
        init_total = 1.*mat_connections.sum(axis=0)
        tr_init_total = 1.*mat_connections.sum(axis=1)

    res_mat = mat_connections
    n_granule = n_subpop
    subpop_start = zeros((n_subpop)) # which is the first non interconnected mitral from each subpop
    for mtpop in inter_conn_rate:
        assert mtpop >= 0 and mtpop < n_subpop, \
            "Incorrect mitral sub-population number "+str(mtpop)+" for inter-connectivity."
        for grpop in inter_conn_rate[mtpop]:
            assert grpop >= 0 and grpop < n_granule, \
                "Incorrect granule sub-population number "+str(grpop)+" for inter-connectivity."
            conn = inter_conn_rate[mtpop][grpop]
            assert conn >= 0 and conn <= 1, "Connectivity must be in [0, 1]."
            nlinks = int(n_mitral_per_subpop*conn)
            newconn = np.zeros((n_mitral_per_subpop, 1))
            for i in xrange(nlinks):
                try:
                    newconn[i + subpop_start[mtpop]] = inter_conn_strength[mtpop][grpop]
                except:
                    print "Likely, too much connections to have no overlap, rewrite the code !"
                    exit(1)
            subpop_start[mtpop] += nlinks
            start = mtpop*n_mitral_per_subpop
            stop  = start + n_mitral_per_subpop
            res_mat[start:stop, grpop] = newconn[:, 0]

    if not(homeostasy):
        return res_mat, res_mat.transpose()
    else:
        # Connection strengthes are normalized such that each neuron (mitral or granule) receive the same total amount of excitation or inhibition
        # (i.e. the same total of synaptic conductance) as if they were no interglomerular connections
        tr_res_mat = res_mat.transpose().copy()
        tr_res_mat_norm = tr_init_total/tr_res_mat.sum(axis=0)  # here the numerator is 1. because there is only one granule
        tr_res_mat = tr_res_mat*tr_res_mat_norm
        res_mat_norm = init_total/res_mat.sum(axis=0)
        res_mat = res_mat*res_mat_norm
        return res_mat, tr_res_mat

def monit(pop, params, timestep, reclist=True, spikes=False):
    """Returns a dictionnary of monitors for the population."""
    res = {}
    for pname in params:
        res[pname] = StateMonitor(pop, pname, record=reclist, timestep=timestep)
    if spikes:
        res['spikes'] = SpikeMonitor(pop, record=True)
    return res
<commit_msg>Fix small bug: numpy call<commit_after># -*- coding:utf-8 -*-
"""
Model related utilities.

"""

import importlib
import model
import numpy as np

from brian import StateMonitor, SpikeMonitor
from utils import path_to_modline

def set_model_ps(filepath, dicname='PARAMETERS'):
    """Set the model parameters given the parameter set in `filepath`."""
    psmod = importlib.import_module(path_to_modline(filepath))
    model.PARAMETERS = getattr(psmod, dicname)

def intrapop_connections(n_mitral, n_granule, n_subpop, n_mitral_per_subpop):
    """Connection matrix for intra sub-population connections."""
    resmat = np.zeros((n_mitral, n_granule))
    for i_subpop in xrange(n_subpop):
        start = i_subpop*n_mitral_per_subpop
        stop  = start + n_mitral_per_subpop
        resmat[start:stop, i_subpop] = 1.
    return resmat

def interpop_connections(mat_connections, n_mitral, n_subpop, n_mitral_per_subpop, inter_conn_rate, inter_conn_strength, homeostasy=False):
    """
    Adds inter sub-population connections.

    Changes on the connection matrix are done in place.

    """
    if homeostasy:
        init_total = 1.*mat_connections.sum(axis=0)
        tr_init_total = 1.*mat_connections.sum(axis=1)

    res_mat = mat_connections
    n_granule = n_subpop
    subpop_start = np.zeros((n_subpop)) # which is the first non interconnected mitral from each subpop
    for mtpop in inter_conn_rate:
        assert mtpop >= 0 and mtpop < n_subpop, \
            "Incorrect mitral sub-population number "+str(mtpop)+" for inter-connectivity."
        for grpop in inter_conn_rate[mtpop]:
            assert grpop >= 0 and grpop < n_granule, \
                "Incorrect granule sub-population number "+str(grpop)+" for inter-connectivity."
            conn = inter_conn_rate[mtpop][grpop]
            assert conn >= 0 and conn <= 1, "Connectivity must be in [0, 1]."
            nlinks = int(n_mitral_per_subpop*conn)
            newconn = np.zeros((n_mitral_per_subpop, 1))
            for i in xrange(nlinks):
                try:
                    newconn[i + subpop_start[mtpop]] = inter_conn_strength[mtpop][grpop]
                except:
                    print "Likely, too much connections to have no overlap, rewrite the code !"
                    exit(1)
            subpop_start[mtpop] += nlinks
            start = mtpop*n_mitral_per_subpop
            stop  = start + n_mitral_per_subpop
            res_mat[start:stop, grpop] = newconn[:, 0]

    if not(homeostasy):
        return res_mat, res_mat.transpose()
    else:
        # Connection strengthes are normalized such that each neuron (mitral or granule) receive the same total amount of excitation or inhibition
        # (i.e. the same total of synaptic conductance) as if they were no interglomerular connections
        tr_res_mat = res_mat.transpose().copy()
        tr_res_mat_norm = tr_init_total/tr_res_mat.sum(axis=0)  # here the numerator is 1. because there is only one granule
        tr_res_mat = tr_res_mat*tr_res_mat_norm
        res_mat_norm = init_total/res_mat.sum(axis=0)
        res_mat = res_mat*res_mat_norm
        return res_mat, tr_res_mat

def monit(pop, params, timestep, reclist=True, spikes=False):
    """Returns a dictionnary of monitors for the population."""
    res = {}
    for pname in params:
        res[pname] = StateMonitor(pop, pname, record=reclist, timestep=timestep)
    if spikes:
        res['spikes'] = SpikeMonitor(pop, record=True)
    return res
<|endoftext|>


<commit_before>from __future__ import print_function

import numpy
from nose.tools import raises

from .context import skip_if_no_cuda_device, skip_if_no_opencl

import kernel_tuner.core as core
import kernel_tuner.cuda as cuda
import kernel_tuner.opencl as opencl
from kernel_tuner.util import *

block_size_names = ["block_size_x", "block_size_y", "block_size_z"]

def test_get_grid_dimensions1():
    problem_size = (1024, 1024, 1)
    params = {"block_x": 41, "block_y": 37}

    grid_div = ( ["block_x"], ["block_y"], None )

    grid = get_grid_dimensions(problem_size, params,
                    grid_div, block_size_names)

    assert len(grid) == 3
    assert isinstance(grid[0], int)
    assert isinstance(grid[1], int)

    assert grid[0] == 25
    assert grid[1] == 28
    assert grid[2] == 1

    grid = get_grid_dimensions(problem_size, params,
                    (grid_div[0], None, None), block_size_names)

    assert grid[0] == 25
    assert grid[1] == 1024
    assert grid[2] == 1

    grid = get_grid_dimensions(problem_size, params,
                    (None, grid_div[1], None), block_size_names)

    assert grid[0] == 1024
    assert grid[1] == 28
    assert grid[2] == 1

def test_get_grid_dimensions2():
    problem_size = (1024, 1024, 1)
    params = {"block_x": 41, "block_y": 37}

    grid_div_x = ["block_x*8"]
    grid_div_y = ["(block_y+2)/8"]

    grid = get_grid_dimensions(problem_size, params,
                    (grid_div_x, grid_div_y, None), block_size_names)

    assert grid[0] == 4
    assert grid[1] == 256

def test_get_problem_size1():
    problem_size = ("num_blocks_x", "num_blocks_y*3")
    params = {"num_blocks_x": 71, "num_blocks_y": 57}

    answer = get_problem_size(problem_size, params)
    assert answer[0] == 71
    assert answer[1] == 171
    assert answer[2] == 1

def test_get_problem_size2():
    problem_size = "num_blocks_x"
    params = {"num_blocks_x": 71}

    answer = get_problem_size(problem_size, params)
    assert answer[0] == 71
    assert answer[1] == 1
    assert answer[2] == 1

@raises(TypeError)
def test_get_problem_size3():
    problem_size = (3.8, "num_blocks_y*3")
    params = {"num_blocks_y": 57}
    get_problem_size(problem_size, params)

def test_get_grid_dimensions5():
    problem_size = (1024, 1024, 1)
    params = {"block_x": 41, "block_y": 37}

    grid_div_x = ["block_x", "block_y"]
    grid_div_y = ["(block_y+2)/8"]

    grid = get_grid_dimensions(problem_size, params,
                    (grid_div_x, grid_div_y, None), block_size_names)

    assert grid[0] == 1
    assert grid[1] == 256
    assert grid[2] == 1

def test_get_grid_dimensions6():

    problem_size = (numpy.int32(1024), numpy.int64(1024), 1)
    params = {"block_x": 41, "block_y": 37}

    grid_div_x = ["block_x", "block_y"]
    grid_div_y = ["(block_y+2)/8"]

    grid = get_grid_dimensions(problem_size, params,
                    (grid_div_x, grid_div_y, None), block_size_names)

    assert grid[0] == 1
    assert grid[1] == 256
    assert grid[2] == 1

def test_get_thread_block_dimensions():

    params = {"block_size_x": 123, "block_size_y": 257}

    threads = get_thread_block_dimensions(params)
    assert len(threads) == 3
    assert isinstance(threads[0], int)
    assert isinstance(threads[1], int)
    assert isinstance(threads[2], int)

    assert threads[0] == 123
    assert threads[1] == 257
    assert threads[2] == 1

def test_prepare_kernel_string():
    kernel = "this is a weird kernel"
    params = dict()
    params["is"] = 8

    name, output = prepare_kernel_string("this", kernel, params, (3,7), (1,2,3), block_size_names)
    expected = "#define is 8\n" \
               "#define block_size_z 3\n" \
               "#define block_size_y 2\n" \
               "#define block_size_x 1\n" \
               "#define grid_size_y 7\n" \
               "#define grid_size_x 3\n" \
               "this_8 is a weird kernel"
    assert output == expected

def test_replace_param_occurrences():
    kernel = "this is a weird kernel"
    params = dict()
    params["is"] = 8
    params["weird"] = 14

    new_kernel = replace_param_occurrences(kernel, params)
    assert new_kernel == "th8 8 a 14 kernel"

    new_kernel = replace_param_occurrences(kernel, dict())
    assert kernel == new_kernel

    params = dict()
    params["blablabla"] = 8
    new_kernel = replace_param_occurrences(kernel, params)
    assert kernel == new_kernel

def test_check_restrictions():
    params = {"a": 7, "b": 4, "c": 3}
    print(params.values())
    print(params.keys())
    restrictions = [["a==b+c"], ["a==b+c", "b==b", "a-b==c"], ["a==b+c", "b!=b", "a-b==c"]]
    expected = [True, True, False]
    #test the call returns expected
    for r,e in zip(restrictions, expected):
        answer = check_restrictions(r, params.values(), params.keys(), False)
        print(answer)
        assert answer == e

def test_detect_language1():
    lang = None
    kernel_string = "__global__ void vector_add( ... );"
    lang = detect_language(lang, kernel_string)
    assert lang == "CUDA"

def test_detect_language2():
    lang = None
    kernel_string = "__kernel void vector_add( ... );"
    lang = detect_language(lang, kernel_string)
    assert lang == "OpenCL"

def test_detect_language3():
    lang = None
    kernel_string = "blabla"
    lang = detect_language(lang, kernel_string)
    assert lang == "C"

def test_detect_language4():
    lang = "CUDA"
    kernel_string = "blabla"
    try:
        lang = detect_language(lang, kernel_string)
        assert lang == "CUDA"
    except Exception:
        assert False

def test_get_device_interface1():
    skip_if_no_cuda_device()
    lang = "CUDA"
    dev = core.DeviceInterface("", 0, 0, lang=lang)
    assert isinstance(dev, core.DeviceInterface)
    assert isinstance(dev.dev, cuda.CudaFunctions)

def test_get_device_interface2():
    skip_if_no_opencl()
    lang = "OpenCL"
    dev = core.DeviceInterface("", 0, 0, lang=lang)
    assert isinstance(dev, core.DeviceInterface)
    assert isinstance(dev.dev, opencl.OpenCLFunctions)

@raises(Exception)
def test_get_device_interface3():
    lang = "blabla"
    core.DeviceInterface("", 0, 0, lang=lang)

def test_check_argument_list1():
    args = [numpy.int32(5), 'blah', numpy.array([1, 2, 3])]
    try:
        check_argument_list(args)
        print("Expected a TypeError to be raised")
        assert False
    except TypeError as e:
        print(str(e))
        assert "at position 1" in str(e)
    except Exception:
        print("Expected a TypeError to be raised")
        assert False

def test_check_argument_list2():
    args = [numpy.int32(5), numpy.float64(4.6), numpy.array([1, 2, 3])]
    check_argument_list(args)
    #test that no exception is raised
    assert True

def test_get_kernel_string_func():
    #test whether passing a function instead of string works
    def gen_kernel(params):
        return "__global__ void kernel_name() { %s }" % params["block_size_x"]
    params = {"block_size_x": "//do that kernel thing!"}
    expected = "__global__ void kernel_name() { //do that kernel thing! }"
    answer = get_kernel_string(gen_kernel, params)
    assert answer == expected

def test_get_kernel_string_filename_not_found():
    #when the string looks like a filename, but the file does not exist
    #assume the string is not a filename after all
    bogus_filename = "filename_3456789.cu"
    answer = get_kernel_string(bogus_filename)
    assert answer == bogus_filename

def test_looks_like_a_filename1():
    string = "filename.c"
    assert looks_like_a_filename(string)

def test_looks_like_a_filename2():
    string = "__global__ void kernel_name() { //do that kernel thing! }"
    assert not looks_like_a_filename(string)

def test_read_write_file():
    filename = get_temp_filename()

    my_string = "this is the test string"
    try:
        write_file(filename, my_string)
        with open(filename, 'r') as f:
            answer = f.read()
        assert my_string == answer
        answer2 = read_file(filename)
        assert my_string == answer2

    finally:
        delete_temp_file(filename)

<commit_msg>Add tests for new util functions<commit_after>from __future__ import print_function

import numpy
from nose.tools import raises
import warnings

from .context import skip_if_no_cuda_device, skip_if_no_opencl

import kernel_tuner.core as core
import kernel_tuner.cuda as cuda
import kernel_tuner.opencl as opencl
from kernel_tuner.util import *

block_size_names = ["block_size_x", "block_size_y", "block_size_z"]

def test_get_grid_dimensions1():
    problem_size = (1024, 1024, 1)
    params = {"block_x": 41, "block_y": 37}

    grid_div = ( ["block_x"], ["block_y"], None )

    grid = get_grid_dimensions(problem_size, params,
                    grid_div, block_size_names)

    assert len(grid) == 3
    assert isinstance(grid[0], int)
    assert isinstance(grid[1], int)

    assert grid[0] == 25
    assert grid[1] == 28
    assert grid[2] == 1

    grid = get_grid_dimensions(problem_size, params,
                    (grid_div[0], None, None), block_size_names)

    assert grid[0] == 25
    assert grid[1] == 1024
    assert grid[2] == 1

    grid = get_grid_dimensions(problem_size, params,
                    (None, grid_div[1], None), block_size_names)

    assert grid[0] == 1024
    assert grid[1] == 28
    assert grid[2] == 1

def test_get_grid_dimensions2():
    problem_size = (1024, 1024, 1)
    params = {"block_x": 41, "block_y": 37}

    grid_div_x = ["block_x*8"]
    grid_div_y = ["(block_y+2)/8"]

    grid = get_grid_dimensions(problem_size, params,
                    (grid_div_x, grid_div_y, None), block_size_names)

    assert grid[0] == 4
    assert grid[1] == 256

def test_get_problem_size1():
    problem_size = ("num_blocks_x", "num_blocks_y*3")
    params = {"num_blocks_x": 71, "num_blocks_y": 57}

    answer = get_problem_size(problem_size, params)
    assert answer[0] == 71
    assert answer[1] == 171
    assert answer[2] == 1

def test_get_problem_size2():
    problem_size = "num_blocks_x"
    params = {"num_blocks_x": 71}

    answer = get_problem_size(problem_size, params)
    assert answer[0] == 71
    assert answer[1] == 1
    assert answer[2] == 1

@raises(TypeError)
def test_get_problem_size3():
    problem_size = (3.8, "num_blocks_y*3")
    params = {"num_blocks_y": 57}
    get_problem_size(problem_size, params)

def test_get_grid_dimensions5():
    problem_size = (1024, 1024, 1)
    params = {"block_x": 41, "block_y": 37}

    grid_div_x = ["block_x", "block_y"]
    grid_div_y = ["(block_y+2)/8"]

    grid = get_grid_dimensions(problem_size, params,
                    (grid_div_x, grid_div_y, None), block_size_names)

    assert grid[0] == 1
    assert grid[1] == 256
    assert grid[2] == 1

def test_get_grid_dimensions6():

    problem_size = (numpy.int32(1024), numpy.int64(1024), 1)
    params = {"block_x": 41, "block_y": 37}

    grid_div_x = ["block_x", "block_y"]
    grid_div_y = ["(block_y+2)/8"]

    grid = get_grid_dimensions(problem_size, params,
                    (grid_div_x, grid_div_y, None), block_size_names)

    assert grid[0] == 1
    assert grid[1] == 256
    assert grid[2] == 1

def test_get_thread_block_dimensions():

    params = {"block_size_x": 123, "block_size_y": 257}

    threads = get_thread_block_dimensions(params)
    assert len(threads) == 3
    assert isinstance(threads[0], int)
    assert isinstance(threads[1], int)
    assert isinstance(threads[2], int)

    assert threads[0] == 123
    assert threads[1] == 257
    assert threads[2] == 1

def test_prepare_kernel_string():
    kernel = "this is a weird kernel"
    params = dict()
    params["is"] = 8

    name, output = prepare_kernel_string("this", kernel, params, (3,7), (1,2,3), block_size_names)
    expected = "#define is 8\n" \
               "#define block_size_z 3\n" \
               "#define block_size_y 2\n" \
               "#define block_size_x 1\n" \
               "#define grid_size_y 7\n" \
               "#define grid_size_x 3\n" \
               "this_8 is a weird kernel"
    assert output == expected

def test_replace_param_occurrences():
    kernel = "this is a weird kernel"
    params = dict()
    params["is"] = 8
    params["weird"] = 14

    new_kernel = replace_param_occurrences(kernel, params)
    assert new_kernel == "th8 8 a 14 kernel"

    new_kernel = replace_param_occurrences(kernel, dict())
    assert kernel == new_kernel

    params = dict()
    params["blablabla"] = 8
    new_kernel = replace_param_occurrences(kernel, params)
    assert kernel == new_kernel

def test_check_restrictions():
    params = {"a": 7, "b": 4, "c": 3}
    print(params.values())
    print(params.keys())
    restrictions = [["a==b+c"], ["a==b+c", "b==b", "a-b==c"], ["a==b+c", "b!=b", "a-b==c"]]
    expected = [True, True, False]
    #test the call returns expected
    for r,e in zip(restrictions, expected):
        answer = check_restrictions(r, params.values(), params.keys(), False)
        print(answer)
        assert answer == e

def test_detect_language1():
    lang = None
    kernel_string = "__global__ void vector_add( ... );"
    lang = detect_language(lang, kernel_string)
    assert lang == "CUDA"

def test_detect_language2():
    lang = None
    kernel_string = "__kernel void vector_add( ... );"
    lang = detect_language(lang, kernel_string)
    assert lang == "OpenCL"

def test_detect_language3():
    lang = None
    kernel_string = "blabla"
    lang = detect_language(lang, kernel_string)
    assert lang == "C"

def test_detect_language4():
    lang = "CUDA"
    kernel_string = "blabla"
    try:
        lang = detect_language(lang, kernel_string)
        assert lang == "CUDA"
    except Exception:
        assert False

def test_get_device_interface1():
    skip_if_no_cuda_device()
    lang = "CUDA"
    dev = core.DeviceInterface("", 0, 0, lang=lang)
    assert isinstance(dev, core.DeviceInterface)
    assert isinstance(dev.dev, cuda.CudaFunctions)

def test_get_device_interface2():
    skip_if_no_opencl()
    lang = "OpenCL"
    dev = core.DeviceInterface("", 0, 0, lang=lang)
    assert isinstance(dev, core.DeviceInterface)
    assert isinstance(dev.dev, opencl.OpenCLFunctions)

@raises(Exception)
def test_get_device_interface3():
    lang = "blabla"
    core.DeviceInterface("", 0, 0, lang=lang)

def test_check_argument_list1():
    args = [numpy.int32(5), 'blah', numpy.array([1, 2, 3])]
    try:
        check_argument_list(args)
        print("Expected a TypeError to be raised")
        assert False
    except TypeError as e:
        print(str(e))
        assert "at position 1" in str(e)
    except Exception:
        print("Expected a TypeError to be raised")
        assert False

def test_check_argument_list2():
    args = [numpy.int32(5), numpy.float64(4.6), numpy.array([1, 2, 3])]
    check_argument_list(args)
    #test that no exception is raised
    assert True

def test_check_tune_params_list():
    tune_params = ["one_thing", "led_to_another", "and_before_you_know_it",
                   "grid_size_y"]
    try:
        check_tune_params_list(tune_params)
        print("Expected a ValueError to be raised")
        assert False
    except ValueError as e:
        print(str(e))
        assert "at position 1" in str(e)
    except Exception:
        print("Expected a ValueError to be raised")
        assert False

def test_check_tune_params_list2():
    tune_params = ["once_upon_a_time", "in_the_west"]
    try:
        check_tune_params_list(tune_params)
        print("Expected a ValueError to be raised")
        assert False
    except ValueError as e:
        print(str(e))
        assert "at position 1" in str(e)
    except Exception:
        print("Expected a ValueError to be raised")
        assert False

def test_check_tune_params_list3():
    tune_params = ["rock", "paper", "scissors"]
    check_tune_params_list(tune_params)
    # test that no exception is raised
    assert True

def test_check_block_size_params_names_list():
    block_size_names = ["block_size_a", "block_size_b"]
    tune_params = ["hyper", "ultra", "mega", "turbo"]
    with warnings.catch_warnings(record=True) as w:
        # Cause all warnings to always be triggered.
        warnings.simplefilter("always")
        # Trigger a warning.
        check_block_size_params_names_list(block_size_names, tune_params)
        # Verify some things
        assert len(w) == 2
        assert issubclass(w[0].category, UserWarning)
        assert issubclass(w[1].category, UserWarning)
        assert "Block size name block_size_a is not specified in the tunable parameters list!" == str(w[0].message)
        assert "Block size name block_size_b is not specified in the tunable parameters list!" == str(w[1].message)

def test_check_block_size_params_names_list2():
    block_size_names = ["block_size_a", "block_size_b"]
    tune_params = ["block_size_a", "block_size_b", "many_other_things"]
    with warnings.catch_warnings(record=True) as w:
        # Cause all warnings to always be triggered.
        warnings.simplefilter("always")
        # Trigger a warning.
        check_block_size_params_names_list(block_size_names, tune_params)
        # test that no warning is raised
        assert len(w) == 0

def test_check_block_size_params_names_list3():
    block_size_names = None
    tune_params = ["block_size_a", "block_size_b", "many_other_things"]
    with warnings.catch_warnings(record=True) as w:
        # Cause all warnings to always be triggered.
        warnings.simplefilter("always")
        # Trigger a warning.
        check_block_size_params_names_list(block_size_names, tune_params)
        # test that no warning is raised
        assert len(w) == 0

def test_get_kernel_string_func():
    #test whether passing a function instead of string works
    def gen_kernel(params):
        return "__global__ void kernel_name() { %s }" % params["block_size_x"]
    params = {"block_size_x": "//do that kernel thing!"}
    expected = "__global__ void kernel_name() { //do that kernel thing! }"
    answer = get_kernel_string(gen_kernel, params)
    assert answer == expected

def test_get_kernel_string_filename_not_found():
    #when the string looks like a filename, but the file does not exist
    #assume the string is not a filename after all
    bogus_filename = "filename_3456789.cu"
    answer = get_kernel_string(bogus_filename)
    assert answer == bogus_filename

def test_looks_like_a_filename1():
    string = "filename.c"
    assert looks_like_a_filename(string)

def test_looks_like_a_filename2():
    string = "__global__ void kernel_name() { //do that kernel thing! }"
    assert not looks_like_a_filename(string)

def test_read_write_file():
    filename = get_temp_filename()

    my_string = "this is the test string"
    try:
        write_file(filename, my_string)
        with open(filename, 'r') as f:
            answer = f.read()
        assert my_string == answer
        answer2 = read_file(filename)
        assert my_string == answer2

    finally:
        delete_temp_file(filename)

<|endoftext|>


<commit_before>"""Metrics module provide metrics for the Media Collections."""

class MediaCollectionsMetrics(object):

    """Media Collections Metrics.

    Attributes:
        collections (list): list of MediaCollections
        cache_allmedia (list): cache list of all the media.Media from
            collections. None until cache is filled.
    """

    def __init__(self, collections):
        """Constructor.

        Args:
            collections (list): list of MediaCollections
        """
        self.collections = collections
        self.cache_allmedia = None

    def listall(self):
        """List all Media objects from the media collections."""
        if self.cache_allmedia is None:
            self.cache_allmedia = []
            for collection in self.collections:
                self.cache_allmedia.extend(collection.listall())
        # else returns the cache value
        return self.cache_allmedia

    def mediacount(self):
        """Count of media in the collections."""
        return len(self.listall())

    def qicount(self):
        """Amount of quality images in the set of collections."""
        qualityimages = [qi for qi in self.listall() if qi.quality_image]
        return len(qualityimages)

    def vicount(self):
        """Amount of valued images in the set of collections."""
        valuedimages = [vi for vi in self.listall() if vi.valued_image]
        return len(valuedimages)

    def fpcount(self):
        """Amount of featured pictures in the set of collections."""
        featuredpictures = [fp for fp in self.listall() if fp.featured_picture]
        return len(featuredpictures)

    def widthcount(self, width):
        """Amount of pictures with a wider border greater than minimal width.

        Args:
            width (int): minimal width.

        Returns:
            int: amount of picture with a wider border greater than minimal
                width
        """
        widthimages = [img for img in self.listall()
                       if img.width >= width or img.height >= width]
        return len(widthimages)

    def pixelcount(self):
        """Amount of pixels within all media collections."""
        return sum([img.size for img in self.listall()])

    def qualitypixelcount(self):
        """Amount of pixels from quality images in all media collections."""
        return sum([img.size for img in self.listall() if img.quality_image])
<commit_msg>Adding uploadercount metric.<commit_after>"""Metrics module provide metrics for the Media Collections."""

class MediaCollectionsMetrics(object):

    """Media Collections Metrics.

    Attributes:
        collections (list): list of MediaCollections
        cache_allmedia (list): cache list of all the media.Media from
            collections. None until cache is filled.
    """

    def __init__(self, collections):
        """Constructor.

        Args:
            collections (list): list of MediaCollections
        """
        self.collections = collections
        self.cache_allmedia = None
        self.uploaders = None

    def listall(self):
        """List all Media objects from the media collections."""
        if self.cache_allmedia is None:
            self.cache_allmedia = []
            for collection in self.collections:
                self.cache_allmedia.extend(collection.listall())
        # else returns the cache value
        return self.cache_allmedia

    def mediacount(self):
        """Count of media in the collections."""
        return len(self.listall())

    def qicount(self):
        """Amount of quality images in the set of collections."""
        qualityimages = [qi for qi in self.listall() if qi.quality_image]
        return len(qualityimages)

    def vicount(self):
        """Amount of valued images in the set of collections."""
        valuedimages = [vi for vi in self.listall() if vi.valued_image]
        return len(valuedimages)

    def fpcount(self):
        """Amount of featured pictures in the set of collections."""
        featuredpictures = [fp for fp in self.listall() if fp.featured_picture]
        return len(featuredpictures)

    def widthcount(self, width):
        """Amount of pictures with a wider border greater than minimal width.

        Args:
            width (int): minimal width.

        Returns:
            int: amount of picture with a wider border greater than minimal
                width
        """
        widthimages = [img for img in self.listall()
                       if img.width >= width or img.height >= width]
        return len(widthimages)

    def pixelcount(self):
        """Amount of pixels within all media collections."""
        return sum([img.size for img in self.listall()])

    def qualitypixelcount(self):
        """Amount of pixels from quality images in all media collections."""
        return sum([img.size for img in self.listall() if img.quality_image])

    def uploadercount(self):
        """Amount of distinct uploader."""
        if self.uploaders is None:
            self.uploaders = []
            for img in self.listall():
                if img.uploader not in self.uploaders:
                    self.uploaders.append(img.uploader)
        return len(self.uploaders)
<|endoftext|>


<commit_before>import collections
import contextlib
import copy
import json

import numpy
import six

from chainer.backends import cuda
from chainer import configuration
from chainer import serializer as serializer_module
from chainer import variable

def _copy_variable(value):
    if isinstance(value, variable.Variable):
        return copy.copy(value)
    return value

class Reporter(object):

    """Object to which observed values are reported.

    Reporter is used to collect values that users want to watch. The reporter
    object holds a mapping from value names to the actually observed values.
    We call this mapping `observations`.

    When a value is passed to the reporter, an object called `observer` can be
    optionally attached. In this case, the name of the observer is added as the
    prefix of the value name. The observer name should be registered
    beforehand.

    See the following example::

       >>> from chainer import Reporter, report, report_scope
       >>>
       >>> reporter = Reporter()
       >>> observer = object()  # it can be an arbitrary (reference) object
       >>> reporter.add_observer('my_observer:', observer)
       >>> observation = {}
       >>> with reporter.scope(observation):
       ...     reporter.report({'x': 1}, observer)
       ...
       >>> observation
       {'my_observer:x': 1}

    There are also a global API to add values::

       >>> observation = {}
       >>> with report_scope(observation):
       ...     report({'x': 1}, observer)
       ...
       >>> observation
       {'my_observer:x': 1}

    The most important application of Reporter is to report observed values
    from each link or chain in the training and validation procedures.
    :class:`~chainer.training.Trainer` and some extensions prepare their own
    Reporter object with the hierarchy of the target link registered as
    observers. We can use :func:`report` function inside any links and chains
    to report the observed values (e.g., training loss, accuracy, activation
    statistics, etc.).

    Attributes:
        observation: Dictionary of observed values.

    """

    def __init__(self):
        self._observer_names = {}
        self.observation = {}

    def __enter__(self):
        """Makes this reporter object current."""
        _reporters.append(self)

    def __exit__(self, exc_type, exc_value, traceback):
        """Recovers the previous reporter object to the current."""
        _reporters.pop()

    @contextlib.contextmanager
    def scope(self, observation):
        """Creates a scope to report observed values to ``observation``.

        This is a context manager to be passed to ``with`` statements. In this
        scope, the observation dictionary is changed to the given one.

        It also makes this reporter object current.

        Args:
            observation (dict): Observation dictionary. All observations
                reported inside of the ``with`` statement are written to this
                dictionary.

        """
        old = self.observation
        self.observation = observation
        self.__enter__()
        yield
        self.__exit__(None, None, None)
        self.observation = old

    def add_observer(self, name, observer):
        """Registers an observer of values.

        Observer defines a scope of names for observed values. Values observed
        with the observer are registered with names prefixed by the observer
        name.

        Args:
            name (str): Name of the observer.
            observer: The observer object. Note that the reporter distinguishes
                the observers by their object ids (i.e., ``id(owner)``), rather
                than the object equality.

        """
        self._observer_names[id(observer)] = name

    def add_observers(self, prefix, observers):
        """Registers multiple observers at once.

        This is a convenient method to register multiple objects at once.

        Args:
            prefix (str): Prefix of each name of observers.
            observers: Iterator of name and observer pairs.

        """
        for name, observer in observers:
            self._observer_names[id(observer)] = prefix + name

    def report(self, values, observer=None):
        """Reports observed values.

        The values are written with the key, prefixed by the name of the
        observer object if given.

        .. note::
           As of v2.0.0, if a value is of type :class:`~chainer.Variable`, the
           variable is copied without preserving the computational graph and
           the new variable object purged from the graph is stored to the
           observer. This behavior can be changed by setting
           ``chainer.config.keep_graph_on_report`` to ``True``.

        Args:
            values (dict): Dictionary of observed values.
            observer: Observer object. Its object ID is used to retrieve the
                observer name, which is used as the prefix of the registration
                name of the observed value.

        """
        if not configuration.config.keep_graph_on_report:
            values = {k: _copy_variable(v) for k, v in six.iteritems(values)}

        if observer is not None:
            observer_id = id(observer)
            if observer_id not in self._observer_names:
                raise KeyError(
                    'Given observer is not registered to the reporter.')
            observer_name = self._observer_names[observer_id]
            for key, value in six.iteritems(values):
                name = '%s/%s' % (observer_name, key)
                self.observation[name] = value
        else:
            self.observation.update(values)

_reporters = []

def get_current_reporter():
    """Returns the current reporter object."""
    return _reporters[-1]

def report(values, observer=None):
    """Reports observed values with the current reporter object.

    Any reporter object can be set current by the ``with`` statement. This
    function calls the :meth:`Report.report` method of the current reporter.
    If no reporter object is current, this function does nothing.

    .. admonition:: Example

       The most typical example is a use within links and chains. Suppose that
       a link is registered to the current reporter as an observer (for
       example, the target link of the optimizer is automatically registered to
       the reporter of the :class:`~chainer.training.Trainer`). We can report
       some values from the link as follows::

          class MyRegressor(chainer.Chain):
              def __init__(self, predictor):
                  super(MyRegressor, self).__init__(predictor=predictor)

              def __call__(self, x, y):
                  # This chain just computes the mean absolute and squared
                  # errors between the prediction and y.
                  pred = self.predictor(x)
                  abs_error = F.sum(F.abs(pred - y)) / len(x.data)
                  loss = F.mean_squared_error(pred, y)

                  # Report the mean absolute and squared errors.
                  report({'abs_error': abs_error, 'squared_error': loss}, self)

                  return loss

       If the link is named ``'main'`` in the hierarchy (which is the default
       name of the target link in the
       :class:`~chainer.training.StandardUpdater`), these reported values are
       named ``'main/abs_error'`` and ``'main/squared_error'``. If these values
       are reported inside the :class:`~chainer.training.extension.Evaluator`
       extension, ``'validation/'`` is added at the head of the link name, thus
       the item names are changed to ``'validation/main/abs_error'`` and
       ``'validation/main/squared_error'`` (``'validation'`` is the default
       name of the Evaluator extension).

    Args:
        values (dict): Dictionary of observed values.
        observer: Observer object. Its object ID is used to retrieve the
            observer name, which is used as the prefix of the registration name
            of the observed value.

    """
    if _reporters:
        current = _reporters[-1]
        current.report(values, observer)

@contextlib.contextmanager
def report_scope(observation):
    """Returns a report scope with the current reporter.

    This is equivalent to ``get_current_reporter().scope(observation)``,
    except that it does not make the reporter current redundantly.

    """
    current = _reporters[-1]
    old = current.observation
    current.observation = observation
    yield
    current.observation = old

def _get_device(x):
    if numpy.isscalar(x):
        return cuda.DummyDevice
    else:
        return cuda.get_device_from_array(x)

class Summary(object):

    """Online summarization of a sequence of scalars.

    Summary computes the statistics of given scalars online.

    """

    def __init__(self):
        self._x = 0
        self._x2 = 0
        self._n = 0

    def add(self, value):
        """Adds a scalar value.

        Args:
            value: Scalar value to accumulate. It is either a NumPy scalar or
                a zero-dimensional array (on CPU or GPU).

        """
        with _get_device(value):
            self._x += value
            self._x2 += value * value
            self._n += 1

    def compute_mean(self):
        """Computes the mean."""
        x, n = self._x, self._n
        with _get_device(x):
            return x / n

    def make_statistics(self):
        """Computes and returns the mean and standard deviation values.

        Returns:
            tuple: Mean and standard deviation values.

        """
        x, n = self._x, self._n
        xp = cuda.get_array_module(x)
        with _get_device(x):
            mean = x / n
            var = self._x2 / n - mean * mean
            std = xp.sqrt(var)
            return mean, std

    def serialize(self, serializer):
        self._x = serializer('_x', self._x)
        self._x2 = serializer('_x2', self._x2)
        self._n = serializer('_n', self._n)

class DictSummary(object):

    """Online summarization of a sequence of dictionaries.

    ``DictSummary`` computes the statistics of a given set of scalars online.
    It only computes the statistics for scalar values and variables of scalar
    values in the dictionaries.

    """

    def __init__(self):
        self._summaries = collections.defaultdict(Summary)

    def add(self, d):
        """Adds a dictionary of scalars.

        Args:
            d (dict): Dictionary of scalars to accumulate. Only elements of
               scalars, zero-dimensional arrays, and variables of
               zero-dimensional arrays are accumulated.

        """
        summaries = self._summaries
        for k, v in six.iteritems(d):
            if isinstance(v, variable.Variable):
                v = v.data
            if numpy.isscalar(v) or getattr(v, 'ndim', -1) == 0:
                summaries[k].add(v)

    def compute_mean(self):
        """Creates a dictionary of mean values.

        It returns a single dictionary that holds a mean value for each entry
        added to the summary.

        Returns:
            dict: Dictionary of mean values.

        """
        return {name: summary.compute_mean()
                for name, summary in six.iteritems(self._summaries)}

    def make_statistics(self):
        """Creates a dictionary of statistics.

        It returns a single dictionary that holds mean and standard deviation
        values for every entry added to the summary. For an entry of name
        ``'key'``, these values are added to the dictionary by names ``'key'``
        and ``'key.std'``, respectively.

        Returns:
            dict: Dictionary of statistics of all entries.

        """
        stats = {}
        for name, summary in six.iteritems(self._summaries):
            mean, std = summary.make_statistics()
            stats[name] = mean
            stats[name + '.std'] = std

        return stats

    def serialize(self, serializer):
        if isinstance(serializer, serializer_module.Serializer):
            serializer('_names', json.dumps(list(self._summaries.keys())))
            for name, summary in six.iteritems(self._summaries):
                if hasattr(summary, 'serialize'):
                    summary.serialize(serializer['_summaries'][name])
        else:
            names = json.loads(serializer('_names', ''))
            for name in names:
                if hasattr(self._summaries[name], 'serialize'):
                    self._summaries[name].serialize(
                        serializer['_summaries'][name])
<commit_msg>assume all items are Summary<commit_after>import collections
import contextlib
import copy
import json

import numpy
import six

from chainer.backends import cuda
from chainer import configuration
from chainer import serializer as serializer_module
from chainer import variable

def _copy_variable(value):
    if isinstance(value, variable.Variable):
        return copy.copy(value)
    return value

class Reporter(object):

    """Object to which observed values are reported.

    Reporter is used to collect values that users want to watch. The reporter
    object holds a mapping from value names to the actually observed values.
    We call this mapping `observations`.

    When a value is passed to the reporter, an object called `observer` can be
    optionally attached. In this case, the name of the observer is added as the
    prefix of the value name. The observer name should be registered
    beforehand.

    See the following example::

       >>> from chainer import Reporter, report, report_scope
       >>>
       >>> reporter = Reporter()
       >>> observer = object()  # it can be an arbitrary (reference) object
       >>> reporter.add_observer('my_observer:', observer)
       >>> observation = {}
       >>> with reporter.scope(observation):
       ...     reporter.report({'x': 1}, observer)
       ...
       >>> observation
       {'my_observer:x': 1}

    There are also a global API to add values::

       >>> observation = {}
       >>> with report_scope(observation):
       ...     report({'x': 1}, observer)
       ...
       >>> observation
       {'my_observer:x': 1}

    The most important application of Reporter is to report observed values
    from each link or chain in the training and validation procedures.
    :class:`~chainer.training.Trainer` and some extensions prepare their own
    Reporter object with the hierarchy of the target link registered as
    observers. We can use :func:`report` function inside any links and chains
    to report the observed values (e.g., training loss, accuracy, activation
    statistics, etc.).

    Attributes:
        observation: Dictionary of observed values.

    """

    def __init__(self):
        self._observer_names = {}
        self.observation = {}

    def __enter__(self):
        """Makes this reporter object current."""
        _reporters.append(self)

    def __exit__(self, exc_type, exc_value, traceback):
        """Recovers the previous reporter object to the current."""
        _reporters.pop()

    @contextlib.contextmanager
    def scope(self, observation):
        """Creates a scope to report observed values to ``observation``.

        This is a context manager to be passed to ``with`` statements. In this
        scope, the observation dictionary is changed to the given one.

        It also makes this reporter object current.

        Args:
            observation (dict): Observation dictionary. All observations
                reported inside of the ``with`` statement are written to this
                dictionary.

        """
        old = self.observation
        self.observation = observation
        self.__enter__()
        yield
        self.__exit__(None, None, None)
        self.observation = old

    def add_observer(self, name, observer):
        """Registers an observer of values.

        Observer defines a scope of names for observed values. Values observed
        with the observer are registered with names prefixed by the observer
        name.

        Args:
            name (str): Name of the observer.
            observer: The observer object. Note that the reporter distinguishes
                the observers by their object ids (i.e., ``id(owner)``), rather
                than the object equality.

        """
        self._observer_names[id(observer)] = name

    def add_observers(self, prefix, observers):
        """Registers multiple observers at once.

        This is a convenient method to register multiple objects at once.

        Args:
            prefix (str): Prefix of each name of observers.
            observers: Iterator of name and observer pairs.

        """
        for name, observer in observers:
            self._observer_names[id(observer)] = prefix + name

    def report(self, values, observer=None):
        """Reports observed values.

        The values are written with the key, prefixed by the name of the
        observer object if given.

        .. note::
           As of v2.0.0, if a value is of type :class:`~chainer.Variable`, the
           variable is copied without preserving the computational graph and
           the new variable object purged from the graph is stored to the
           observer. This behavior can be changed by setting
           ``chainer.config.keep_graph_on_report`` to ``True``.

        Args:
            values (dict): Dictionary of observed values.
            observer: Observer object. Its object ID is used to retrieve the
                observer name, which is used as the prefix of the registration
                name of the observed value.

        """
        if not configuration.config.keep_graph_on_report:
            values = {k: _copy_variable(v) for k, v in six.iteritems(values)}

        if observer is not None:
            observer_id = id(observer)
            if observer_id not in self._observer_names:
                raise KeyError(
                    'Given observer is not registered to the reporter.')
            observer_name = self._observer_names[observer_id]
            for key, value in six.iteritems(values):
                name = '%s/%s' % (observer_name, key)
                self.observation[name] = value
        else:
            self.observation.update(values)

_reporters = []

def get_current_reporter():
    """Returns the current reporter object."""
    return _reporters[-1]

def report(values, observer=None):
    """Reports observed values with the current reporter object.

    Any reporter object can be set current by the ``with`` statement. This
    function calls the :meth:`Report.report` method of the current reporter.
    If no reporter object is current, this function does nothing.

    .. admonition:: Example

       The most typical example is a use within links and chains. Suppose that
       a link is registered to the current reporter as an observer (for
       example, the target link of the optimizer is automatically registered to
       the reporter of the :class:`~chainer.training.Trainer`). We can report
       some values from the link as follows::

          class MyRegressor(chainer.Chain):
              def __init__(self, predictor):
                  super(MyRegressor, self).__init__(predictor=predictor)

              def __call__(self, x, y):
                  # This chain just computes the mean absolute and squared
                  # errors between the prediction and y.
                  pred = self.predictor(x)
                  abs_error = F.sum(F.abs(pred - y)) / len(x.data)
                  loss = F.mean_squared_error(pred, y)

                  # Report the mean absolute and squared errors.
                  report({'abs_error': abs_error, 'squared_error': loss}, self)

                  return loss

       If the link is named ``'main'`` in the hierarchy (which is the default
       name of the target link in the
       :class:`~chainer.training.StandardUpdater`), these reported values are
       named ``'main/abs_error'`` and ``'main/squared_error'``. If these values
       are reported inside the :class:`~chainer.training.extension.Evaluator`
       extension, ``'validation/'`` is added at the head of the link name, thus
       the item names are changed to ``'validation/main/abs_error'`` and
       ``'validation/main/squared_error'`` (``'validation'`` is the default
       name of the Evaluator extension).

    Args:
        values (dict): Dictionary of observed values.
        observer: Observer object. Its object ID is used to retrieve the
            observer name, which is used as the prefix of the registration name
            of the observed value.

    """
    if _reporters:
        current = _reporters[-1]
        current.report(values, observer)

@contextlib.contextmanager
def report_scope(observation):
    """Returns a report scope with the current reporter.

    This is equivalent to ``get_current_reporter().scope(observation)``,
    except that it does not make the reporter current redundantly.

    """
    current = _reporters[-1]
    old = current.observation
    current.observation = observation
    yield
    current.observation = old

def _get_device(x):
    if numpy.isscalar(x):
        return cuda.DummyDevice
    else:
        return cuda.get_device_from_array(x)

class Summary(object):

    """Online summarization of a sequence of scalars.

    Summary computes the statistics of given scalars online.

    """

    def __init__(self):
        self._x = 0
        self._x2 = 0
        self._n = 0

    def add(self, value):
        """Adds a scalar value.

        Args:
            value: Scalar value to accumulate. It is either a NumPy scalar or
                a zero-dimensional array (on CPU or GPU).

        """
        with _get_device(value):
            self._x += value
            self._x2 += value * value
            self._n += 1

    def compute_mean(self):
        """Computes the mean."""
        x, n = self._x, self._n
        with _get_device(x):
            return x / n

    def make_statistics(self):
        """Computes and returns the mean and standard deviation values.

        Returns:
            tuple: Mean and standard deviation values.

        """
        x, n = self._x, self._n
        xp = cuda.get_array_module(x)
        with _get_device(x):
            mean = x / n
            var = self._x2 / n - mean * mean
            std = xp.sqrt(var)
            return mean, std

    def serialize(self, serializer):
        self._x = serializer('_x', self._x)
        self._x2 = serializer('_x2', self._x2)
        self._n = serializer('_n', self._n)

class DictSummary(object):

    """Online summarization of a sequence of dictionaries.

    ``DictSummary`` computes the statistics of a given set of scalars online.
    It only computes the statistics for scalar values and variables of scalar
    values in the dictionaries.

    """

    def __init__(self):
        self._summaries = collections.defaultdict(Summary)

    def add(self, d):
        """Adds a dictionary of scalars.

        Args:
            d (dict): Dictionary of scalars to accumulate. Only elements of
               scalars, zero-dimensional arrays, and variables of
               zero-dimensional arrays are accumulated.

        """
        summaries = self._summaries
        for k, v in six.iteritems(d):
            if isinstance(v, variable.Variable):
                v = v.data
            if numpy.isscalar(v) or getattr(v, 'ndim', -1) == 0:
                summaries[k].add(v)

    def compute_mean(self):
        """Creates a dictionary of mean values.

        It returns a single dictionary that holds a mean value for each entry
        added to the summary.

        Returns:
            dict: Dictionary of mean values.

        """
        return {name: summary.compute_mean()
                for name, summary in six.iteritems(self._summaries)}

    def make_statistics(self):
        """Creates a dictionary of statistics.

        It returns a single dictionary that holds mean and standard deviation
        values for every entry added to the summary. For an entry of name
        ``'key'``, these values are added to the dictionary by names ``'key'``
        and ``'key.std'``, respectively.

        Returns:
            dict: Dictionary of statistics of all entries.

        """
        stats = {}
        for name, summary in six.iteritems(self._summaries):
            mean, std = summary.make_statistics()
            stats[name] = mean
            stats[name + '.std'] = std

        return stats

    def serialize(self, serializer):
        if isinstance(serializer, serializer_module.Serializer):
            serializer('_names', json.dumps(list(self._summaries.keys())))
            for name, summary in six.iteritems(self._summaries):
                summary.serialize(serializer['_summaries'][name])
        else:
            names = json.loads(serializer('_names', ''))
            for name in names:
                self._summaries[name].serialize(
                    serializer['_summaries'][name])
<|endoftext|>


<commit_before>from os import path
import requests
from urlparse import urlsplit
from urllib import quote_plus

from django.contrib.auth.models import User
from django.core.files.base import ContentFile
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models.signals import pre_save
from django.db.models.query import QuerySet
from django.dispatch import receiver
from django.template.defaultfilters import slugify
from django.utils.translation import ugettext_lazy as _

from met.metadataparser.utils import compare_filecontents
from met.metadataparser.xmlparser import MetadataParser, DESCRIPTOR_TYPES_DISPLAY

def update_obj(mobj, obj, attrs=None):
    for_attrs = attrs or getattr(mobj, 'all_attrs', [])
    for attrb in attrs or for_attrs:
        if (getattr(mobj, attrb, None) and
            getattr(obj, attrb, None) and
            getattr(mobj, attrb) != getattr(obj, attrb)):
            setattr(obj, attrb,  getattr(mobj, attrb))

class Base(models.Model):
    file_url = models.URLField(verbose_name='Metadata url',
                               blank=True, null=True,
                               help_text=_(u'Url to fetch metadata file'))
    file = models.FileField(upload_to='metadata', blank=True, null=True,
                            verbose_name=_(u'metadata xml file'),
                            help_text=_("if url is set, metadata url will be "
                                        "fetched and replace file value"))
    file_id = models.CharField(blank=True, null=True, max_length=100,
                               verbose_name=_(u'File ID'))

    editor_users = models.ManyToManyField(User, null=True, blank=True,
                                          verbose_name=_('editor users'))

    class Meta:
        abstract = True

    class XmlError(Exception):
        pass

    def __unicode__(self):
        return self.url or u"Metadata %s" % self.id

    def load_file(self):
        """Only load file and parse it, don't create/update any objects"""
        if not self.file:
            return None
        metadata = MetadataParser(filename=self.file.path)
        return metadata

    def fetch_metadata_file(self):
        req = requests.get(self.file_url)
        if req.ok:
            req.raise_for_status()
        parsed_url = urlsplit(self.file_url)
        if self.file:
            self.file.seek(0)
            original_file_content = self.file.read()
            if compare_filecontents(original_file_content, req.content):
                return

        filename = path.basename(parsed_url.path)
        self.file.save(filename, ContentFile(req.content), save=False)

    def process_metadata(self):
        raise NotImplemented()

    def can_edit(self, user):
        return (user.is_superuser or user in self.editor_users.all())

class XmlDescriptionError(Exception):
    pass

class Federation(Base):

    name = models.CharField(blank=False, null=False, max_length=100,
                            unique=True, verbose_name=_(u'Name'))
    url = models.URLField(verbose_name='Federation url',
                          blank=True, null=True)
    logo = models.ImageField(upload_to='federation_logo', blank=True,
                             null=True, verbose_name=_(u'Federation logo'))
    is_interfederation = models.BooleanField(default=False, db_index=True,
                                             verbose_name=_(u'Federation of federations'))
    slug = models.SlugField(max_length=200, unique=True)

    @property
    def _metadata(self):

        if not hasattr(self, '_metadata_cache'):
            self._metadata_cache = self.load_file()
        return self._metadata_cache

    def __unicode__(self):
        return self.name

    def get_entity_metadata(self, entityid):
        return self._metadata.get_entity(entityid)

    def get_entity(self, entityid):
        return self.entity_set.get(entityid=entityid)

    def process_metadata(self):
        metadata = self.load_file()
        if (self.file_id and metadata.file_id and
                metadata.file_id == self.file_id):
            return
        else:
            self.file_id = metadata.file_id

        if not metadata:
            return
        if not metadata.is_federation:
            raise XmlDescriptionError("XML Haven't federation form")

        update_obj(metadata.get_federation(), self)

    def process_metadata_entities(self):
        for m_id in self._metadata.get_entities():
            try:
                entity = self.get_entity(entityid=m_id)
            except Entity.DoesNotExist:
                try:
                    entity = Entity.objects.get(entityid=m_id)
                    self.entity_set.add(entity)
                except Entity.DoesNotExist:
                    entity = self.entity_set.create(entityid=m_id)
            entity.process_metadata(self._metadata.get_entity(m_id))

    def get_absolute_url(self):
        return reverse('federation_view', args=[self.slug])

class EntityQuerySet(QuerySet):
    def iterator(self):
        cached_federations = {}
        for entity in super(EntityQuerySet, self).iterator():
            if not entity.file:
                federations = entity.federations.all()
                if federations:
                    federation = federations[0]
                else:
                    raise ValueError("Can't find entity metadata")

                for federation in federations:
                    if not federation.id in cached_federations:
                        cached_federations[federation.id] = federation

                    cached_federation = cached_federations[federation.id]
                    try:
                        entity.load_metadata(federation=cached_federation)
                    except ValueError:
                        # Allow entity in federation but not in federation file
                        continue
                    else:
                        break

            yield entity

class EntityManager(models.Manager):
    def get_query_set(self):
        return EntityQuerySet(self.model, using=self._db)

class EntityType(models.Model):
    name = models.CharField(blank=False, max_length=20, unique=True,
                            verbose_name=_(u'Name'), db_index=True)
    xmlname = models.CharField(blank=False, max_length=20, unique=True,
                            verbose_name=_(u'Name in XML'), db_index=True)

    def __unicode__(self):
        return self.name

class Entity(Base):

    READABLE_PROTOCOLS = {
        'urn:oasis:names:tc:SAML:1.1:protocol': 'SAML 1.1',
        'urn:oasis:names:tc:SAML:2.0:protocol': 'SAML 2.0',
        'urn:mace:shibboleth:1.0': 'Shiboleth 1.0',
    }

    entityid = models.CharField(blank=False, max_length=200, unique=True,
                                verbose_name=_(u'EntityID'), db_index=True)
    federations = models.ManyToManyField(Federation,
                                         verbose_name=_(u'Federations'))

    types = models.ManyToManyField(EntityType, verbose_name=_(u'Type'))
    objects = models.Manager()
    longlist = EntityManager()

    @property
    def organization(self):
        return self._get_property('organization')

    @property
    def name(self):
        return self._get_property('displayname')

    @property
    def description(self):
        return self._get_property('description')

    @property
    def xml_types(self):
        return self._get_property('entity_types')

    @property
    def protocols(self):
        return self._get_property('protocols')

    def display_protocols(self):
        protocols = []
        for proto in self._get_property('protocols'):
            protocols.append(self.READABLE_PROTOCOLS.get(proto, proto))
        return protocols

    @property
    def logos(self):
        return list(self._get_property('logos')) + list(self.entitylogo_set.all())

    class Meta:
        verbose_name = _(u'Entity')
        verbose_name_plural = _(u'Entities')

    def __unicode__(self):
        return self.entityid

    def load_metadata(self, federation=None, entity_data=None):
        if not hasattr(self, '_entity_cached'):
            if self.file:
                self._entity_cached = self.load_file().get_entity(self.entityid)
            elif federation:
                self._entity_cached = federation.get_entity_metadata(self.entityid)
            elif entity_data:
                self._entity_cached = entity_data
            else:
                for federation in self.federations.all():
                    try:
                        entity_cached = federation.get_entity_metadata(self.entityid)
                        if entity_cached and hasattr(self, '_entity_cached'):
                            self._entity_cached.update(entity_cached)
                        else:
                            self._entity_cached = entity_cached
                    except ValueError:
                        continue
            if not hasattr(self, '_entity_cached'):
                raise ValueError("Can't find entity metadata")

    def _get_property(self, prop):
        try:
            self.load_metadata()
        except ValueError:
            return None
        if hasattr(self, '_entity_cached'):
            return self._entity_cached.get(prop, None)
        else:
            raise ValueError("Not metadata loaded")

    def process_metadata(self, entity_data=None):
        if not entity_data:
            self.load_metadata()

        if self.entityid != entity_data.get('entityid'):
            raise ValueError("EntityID is not the same")

        self._entity_cached = entity_data
        if self.xml_types:
            for etype in self.xml_types:
                try:
                    entity_type = EntityType.objects.get(xmlname=etype)
                except EntityType.DoesNotExist:
                    entity_type = EntityType.objects.create(xmlname=etype,
                                              name=DESCRIPTOR_TYPES_DISPLAY[etype])
                if entity_type not in self.types.all():
                    self.types.add(entity_type)

    def can_edit(self, user):
        if super(Entity, self).can_edit(user):
            return True
        for federation in self.federations.all():
            if federation.can_edit(user):
                return True

    def get_absolute_url(self):
        return reverse('entity_view', args=[quote_plus(self.entityid)])

class EntityLogo(models.Model):
    logo = models.ImageField(upload_to='entity_logo', blank=True,
                             null=True, verbose_name=_(u'Entity logo'))
    alt = models.CharField(max_length=100, blank=True, null=True,
                           verbose_name=(u'Alternative text'))

    entity = models.ForeignKey(Entity, blank=False,
                        verbose_name=_('Entity'))

    def __unicode__(self):
        return self.alt or u"logo %i" % self.id

@receiver(pre_save, sender=Federation, dispatch_uid='federation_pre_save')
def federation_pre_save(sender, instance, **kwargs):
    if instance.file_url:
        instance.fetch_metadata_file()
    if instance.name:
        instance.slug = slugify(unicode(instance))[:200]

@receiver(pre_save, sender=Entity, dispatch_uid='entity_pre_save')
def entity_pre_save(sender, instance, **kwargs):
    if instance.file_url:
        instance.fetch_metadata_file()
        instance.process_metadata()
<commit_msg>Change interfederation label<commit_after>from os import path
import requests
from urlparse import urlsplit
from urllib import quote_plus

from django.contrib.auth.models import User
from django.core.files.base import ContentFile
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models.signals import pre_save
from django.db.models.query import QuerySet
from django.dispatch import receiver
from django.template.defaultfilters import slugify
from django.utils.translation import ugettext_lazy as _

from met.metadataparser.utils import compare_filecontents
from met.metadataparser.xmlparser import MetadataParser, DESCRIPTOR_TYPES_DISPLAY

def update_obj(mobj, obj, attrs=None):
    for_attrs = attrs or getattr(mobj, 'all_attrs', [])
    for attrb in attrs or for_attrs:
        if (getattr(mobj, attrb, None) and
            getattr(obj, attrb, None) and
            getattr(mobj, attrb) != getattr(obj, attrb)):
            setattr(obj, attrb,  getattr(mobj, attrb))

class Base(models.Model):
    file_url = models.URLField(verbose_name='Metadata url',
                               blank=True, null=True,
                               help_text=_(u'Url to fetch metadata file'))
    file = models.FileField(upload_to='metadata', blank=True, null=True,
                            verbose_name=_(u'metadata xml file'),
                            help_text=_("if url is set, metadata url will be "
                                        "fetched and replace file value"))
    file_id = models.CharField(blank=True, null=True, max_length=100,
                               verbose_name=_(u'File ID'))

    editor_users = models.ManyToManyField(User, null=True, blank=True,
                                          verbose_name=_('editor users'))

    class Meta:
        abstract = True

    class XmlError(Exception):
        pass

    def __unicode__(self):
        return self.url or u"Metadata %s" % self.id

    def load_file(self):
        """Only load file and parse it, don't create/update any objects"""
        if not self.file:
            return None
        metadata = MetadataParser(filename=self.file.path)
        return metadata

    def fetch_metadata_file(self):
        req = requests.get(self.file_url)
        if req.ok:
            req.raise_for_status()
        parsed_url = urlsplit(self.file_url)
        if self.file:
            self.file.seek(0)
            original_file_content = self.file.read()
            if compare_filecontents(original_file_content, req.content):
                return

        filename = path.basename(parsed_url.path)
        self.file.save(filename, ContentFile(req.content), save=False)

    def process_metadata(self):
        raise NotImplemented()

    def can_edit(self, user):
        return (user.is_superuser or user in self.editor_users.all())

class XmlDescriptionError(Exception):
    pass

class Federation(Base):

    name = models.CharField(blank=False, null=False, max_length=100,
                            unique=True, verbose_name=_(u'Name'))
    url = models.URLField(verbose_name='Federation url',
                          blank=True, null=True)
    logo = models.ImageField(upload_to='federation_logo', blank=True,
                             null=True, verbose_name=_(u'Federation logo'))
    is_interfederation = models.BooleanField(default=False, db_index=True,
                                         verbose_name=_(u'Is interfederation'))
    slug = models.SlugField(max_length=200, unique=True)

    @property
    def _metadata(self):

        if not hasattr(self, '_metadata_cache'):
            self._metadata_cache = self.load_file()
        return self._metadata_cache

    def __unicode__(self):
        return self.name

    def get_entity_metadata(self, entityid):
        return self._metadata.get_entity(entityid)

    def get_entity(self, entityid):
        return self.entity_set.get(entityid=entityid)

    def process_metadata(self):
        metadata = self.load_file()
        if (self.file_id and metadata.file_id and
                metadata.file_id == self.file_id):
            return
        else:
            self.file_id = metadata.file_id

        if not metadata:
            return
        if not metadata.is_federation:
            raise XmlDescriptionError("XML Haven't federation form")

        update_obj(metadata.get_federation(), self)

    def process_metadata_entities(self):
        for m_id in self._metadata.get_entities():
            try:
                entity = self.get_entity(entityid=m_id)
            except Entity.DoesNotExist:
                try:
                    entity = Entity.objects.get(entityid=m_id)
                    self.entity_set.add(entity)
                except Entity.DoesNotExist:
                    entity = self.entity_set.create(entityid=m_id)
            entity.process_metadata(self._metadata.get_entity(m_id))

    def get_absolute_url(self):
        return reverse('federation_view', args=[self.slug])

class EntityQuerySet(QuerySet):
    def iterator(self):
        cached_federations = {}
        for entity in super(EntityQuerySet, self).iterator():
            if not entity.file:
                federations = entity.federations.all()
                if federations:
                    federation = federations[0]
                else:
                    raise ValueError("Can't find entity metadata")

                for federation in federations:
                    if not federation.id in cached_federations:
                        cached_federations[federation.id] = federation

                    cached_federation = cached_federations[federation.id]
                    try:
                        entity.load_metadata(federation=cached_federation)
                    except ValueError:
                        # Allow entity in federation but not in federation file
                        continue
                    else:
                        break

            yield entity

class EntityManager(models.Manager):
    def get_query_set(self):
        return EntityQuerySet(self.model, using=self._db)

class EntityType(models.Model):
    name = models.CharField(blank=False, max_length=20, unique=True,
                            verbose_name=_(u'Name'), db_index=True)
    xmlname = models.CharField(blank=False, max_length=20, unique=True,
                            verbose_name=_(u'Name in XML'), db_index=True)

    def __unicode__(self):
        return self.name

class Entity(Base):

    READABLE_PROTOCOLS = {
        'urn:oasis:names:tc:SAML:1.1:protocol': 'SAML 1.1',
        'urn:oasis:names:tc:SAML:2.0:protocol': 'SAML 2.0',
        'urn:mace:shibboleth:1.0': 'Shiboleth 1.0',
    }

    entityid = models.CharField(blank=False, max_length=200, unique=True,
                                verbose_name=_(u'EntityID'), db_index=True)
    federations = models.ManyToManyField(Federation,
                                         verbose_name=_(u'Federations'))

    types = models.ManyToManyField(EntityType, verbose_name=_(u'Type'))
    objects = models.Manager()
    longlist = EntityManager()

    @property
    def organization(self):
        return self._get_property('organization')

    @property
    def name(self):
        return self._get_property('displayname')

    @property
    def description(self):
        return self._get_property('description')

    @property
    def xml_types(self):
        return self._get_property('entity_types')

    @property
    def protocols(self):
        return self._get_property('protocols')

    def display_protocols(self):
        protocols = []
        for proto in self._get_property('protocols'):
            protocols.append(self.READABLE_PROTOCOLS.get(proto, proto))
        return protocols

    @property
    def logos(self):
        return list(self._get_property('logos')) + list(self.entitylogo_set.all())

    class Meta:
        verbose_name = _(u'Entity')
        verbose_name_plural = _(u'Entities')

    def __unicode__(self):
        return self.entityid

    def load_metadata(self, federation=None, entity_data=None):
        if not hasattr(self, '_entity_cached'):
            if self.file:
                self._entity_cached = self.load_file().get_entity(self.entityid)
            elif federation:
                self._entity_cached = federation.get_entity_metadata(self.entityid)
            elif entity_data:
                self._entity_cached = entity_data
            else:
                for federation in self.federations.all():
                    try:
                        entity_cached = federation.get_entity_metadata(self.entityid)
                        if entity_cached and hasattr(self, '_entity_cached'):
                            self._entity_cached.update(entity_cached)
                        else:
                            self._entity_cached = entity_cached
                    except ValueError:
                        continue
            if not hasattr(self, '_entity_cached'):
                raise ValueError("Can't find entity metadata")

    def _get_property(self, prop):
        try:
            self.load_metadata()
        except ValueError:
            return None
        if hasattr(self, '_entity_cached'):
            return self._entity_cached.get(prop, None)
        else:
            raise ValueError("Not metadata loaded")

    def process_metadata(self, entity_data=None):
        if not entity_data:
            self.load_metadata()

        if self.entityid != entity_data.get('entityid'):
            raise ValueError("EntityID is not the same")

        self._entity_cached = entity_data
        if self.xml_types:
            for etype in self.xml_types:
                try:
                    entity_type = EntityType.objects.get(xmlname=etype)
                except EntityType.DoesNotExist:
                    entity_type = EntityType.objects.create(xmlname=etype,
                                              name=DESCRIPTOR_TYPES_DISPLAY[etype])
                if entity_type not in self.types.all():
                    self.types.add(entity_type)

    def can_edit(self, user):
        if super(Entity, self).can_edit(user):
            return True
        for federation in self.federations.all():
            if federation.can_edit(user):
                return True

    def get_absolute_url(self):
        return reverse('entity_view', args=[quote_plus(self.entityid)])

class EntityLogo(models.Model):
    logo = models.ImageField(upload_to='entity_logo', blank=True,
                             null=True, verbose_name=_(u'Entity logo'))
    alt = models.CharField(max_length=100, blank=True, null=True,
                           verbose_name=(u'Alternative text'))

    entity = models.ForeignKey(Entity, blank=False,
                        verbose_name=_('Entity'))

    def __unicode__(self):
        return self.alt or u"logo %i" % self.id

@receiver(pre_save, sender=Federation, dispatch_uid='federation_pre_save')
def federation_pre_save(sender, instance, **kwargs):
    if instance.file_url:
        instance.fetch_metadata_file()
    if instance.name:
        instance.slug = slugify(unicode(instance))[:200]

@receiver(pre_save, sender=Entity, dispatch_uid='entity_pre_save')
def entity_pre_save(sender, instance, **kwargs):
    if instance.file_url:
        instance.fetch_metadata_file()
        instance.process_metadata()
<|endoftext|>