<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2014-2015 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import

import os
import time
import re

from bigmler.tests.world import world, res_filename
from subprocess import check_call, CalledProcessError
from bigmler.checkpoint import file_number_of_lines
from bigmler.utils import SYSTEM_ENCODING, PYTHON3, open_mode
from bigml.api import check_resource
from bigmler.tests.common_steps import check_debug

def python3_contents(filename, prior_contents):
    """Check for a file that has alternative contents for Python3 and return
       its contents

    """
    directory = os.path.dirname(filename)
    basename = os.path.basename(filename)
    basename_name, basename_ext = basename.split(".")
    filename = os.path.join(directory, "%s_py3.%s" % ( \
        basename_name, basename_ext))
    try:
        with open(filename, open_mode("r")) as file_handler:
            return file_handler.read()
    except IOError:
        return prior_contents

#@step(r'I create a reify output for the resource in "(.*)" for "(.*)')
def i_create_output(step, output=None, language=None, resource_type='source',
                    add_fields=False):
    if output is None and language is None:
        assert False
    world.directory = os.path.dirname(output)
    world.folders.append(world.directory)
    resource_id = getattr(world, resource_type)['resource']
    try:
        command = (u"bigmler reify --id " + resource_id +
                   u" --store --output " + output)
        if add_fields:
            command += u' --add-fields'
        command = check_debug(command)
        if not PYTHON3:
            command.encode(SYSTEM_ENCODING)
        retcode = check_call(command, shell=True)
        if retcode < 0:
            assert False
        else:
            world.output = world.directory
            assert True
    except (OSError, CalledProcessError, IOError) as exc:
        assert False, str(exc)

#@step(r'the "(.*)" file is like "(.*)"')
def i_check_output_file(step, output=None, check_file=None):
    if check_file is None or output is None:
        assert False
    check_file = res_filename(check_file)
    try:
        output_file = os.path.join(world.directory, "reify.py")
        with open(check_file, open_mode("r")) as check_file_handler:
            check_contents = check_file_handler.read().strip("\n")
        # remove unicode mark for strings if Python3
        if PYTHON3:
            check_contents = check_contents.replace( \
                " u'", " '").replace("{u'", "{'")
        with open(output_file, open_mode("r")) as output_file:
            output_file_contents = output_file.read()
        #strip comments at the beginning of the file
        output_file_contents = re.sub(r'""".*"""', '', output_file_contents,
                                      flags=re.S).strip("\n")
        if check_contents == output_file_contents:
            assert True
        else:
            if PYTHON3:
                # look for an alternative in PYTHON3
                check_contents = python3_contents(check_file, check_contents)
            if check_contents == output_file_contents:
                assert True
            else:
                assert False, ("File contents:\n%s\nExpected contents:\n%s" %
                               (output_file_contents, check_contents))
    except Exception, exc:
        assert False, str(exc)

#@step(r'I create a BigML source with data "(.*)" and params "(.*)"')
def create_source(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename), args)
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])

#@step(r'I create a BigML dataset from a source with data "(.*)" and params "(.*)"')
def create_dataset(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source, args)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])

#@step(r'I create a BigML model from a dataset with data "(.*)" and params "(.*)"')
def create_model(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.model = world.api.create_model(world.dataset, args)
    world.api.ok(world.model)
    world.models.append(world.model['resource'])

#@step(r'I create a BigML prediction for (.*) from a model with data "(.*)" and params "(.*)"')
def create_prediction(filename, input_data= None, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.model = world.api.create_model(world.dataset)
    world.api.ok(world.model)
    world.models.append(world.model['resource'])
    world.prediction = world.api.create_prediction(world.model, input_data, args)
    world.api.ok(world.prediction)
    world.predictions.append(world.prediction['resource'])

#@step(r'I create a BigML cluster from a dataset with data "(.*)" and params "(.*)"')
def create_cluster(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.cluster = world.api.create_cluster(world.dataset, args)
    world.api.ok(world.cluster)
    world.clusters.append(world.cluster['resource'])

#@step(r'I create a BigML anomaly from a dataset with data "(.*)" and params "(.*)"')
def create_anomaly(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.anomaly = world.api.create_anomaly(world.dataset, args)
    world.api.ok(world.anomaly)
    world.anomalies.append(world.anomaly['resource'])

#@step(r'I create a BigML centroid for (.*) from a cluster with data "(.*)" and params "(.*)"')
def create_centroid(filename, input_data=None, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.cluster = world.api.create_cluster(world.dataset)
    world.api.ok(world.cluster)
    world.clusters.append(world.cluster['resource'])
    world.centroid = world.api.create_centroid(world.cluster, input_data, args)
    world.api.ok(world.centroid)
    world.centroids.append(world.centroid['resource'])

#@step(r'I create a BigML anomaly score for (.*) from an anomaly detector with data "(.*)" and params "(.*)"')
def create_anomaly_score(filename, input_data=None, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.anomaly = world.api.create_anomaly(world.dataset)
    world.api.ok(world.anomaly)
    world.anomalies.append(world.anomaly['resource'])
    world.anomaly_score = world.api.create_anomaly_score(world.anomaly, input_data, args)
    world.api.ok(world.anomaly_score)
    world.anomaly_scores.append(world.anomaly_score['resource'])

#@step(r'I create a BigML batch prediction from a model with data "(.*)" and params "(.*)"')
def create_batch_prediction(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.model = world.api.create_model(world.dataset)
    world.api.ok(world.model)
    world.models.append(world.model['resource'])
    world.batch_prediction = world.api.create_batch_prediction(world.model, world.dataset, args)
    world.api.ok(world.batch_prediction)
    world.batch_predictions.append(world.batch_prediction['resource'])

#@step(r'I create a BigML batch centroid from a cluster with data "(.*)" and params "(.*)"')
def create_batch_centroid(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.cluster = world.api.create_cluster(world.dataset)
    world.api.ok(world.cluster)
    world.clusters.append(world.cluster['resource'])
    world.batch_centroid = world.api.create_batch_centroid(world.cluster, world.dataset, args)
    world.api.ok(world.batch_centroid)
    world.batch_centroids.append(world.batch_centroid['resource'])

#@step(r'I create a BigML batch anomaly score from an anomaly detector with data "(.*)" and params "(.*)"')
def create_batch_anomaly_score(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.anomaly = world.api.create_anomaly(world.dataset)
    world.api.ok(world.anomaly)
    world.anomalies.append(world.anomaly['resource'])
    world.batch_anomaly_score = world.api.create_batch_anomaly_score(world.anomaly, world.dataset, args)
    world.api.ok(world.batch_anomaly_score)
    world.batch_anomaly_scores.append(world.batch_anomaly_score['resource'])

#@step(r'I create a BigML evaluation with data "(.*)" and params "(.*)"')
def create_evaluation(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.model = world.api.create_model(world.dataset)
    world.api.ok(world.model)
    world.models.append(world.model['resource'])
    world.evaluation = world.api.create_evaluation(world.model, world.dataset, args)
    world.api.ok(world.evaluation)
    world.evaluations.append(world.evaluation['resource'])

#@step(r'I create a BigML ensemble from a dataset with data "(.*)" and params "(.*)"')
def create_ensemble(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.ensemble = world.api.create_ensemble(world.dataset, args)
    world.api.ok(world.ensemble)
    world.ensembles.append(world.ensemble['resource'])
<commit_msg>Bug fixing in reify tests for Python3<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2014-2015 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import

import os
import time
import re

from bigmler.tests.world import world, res_filename
from subprocess import check_call, CalledProcessError
from bigmler.checkpoint import file_number_of_lines
from bigmler.utils import SYSTEM_ENCODING, PYTHON3, open_mode
from bigml.api import check_resource
from bigmler.tests.common_steps import check_debug

def python3_contents(filename, prior_contents):
    """Check for a file that has alternative contents for Python3 and return
       its contents

    """
    directory = os.path.dirname(filename)
    basename = os.path.basename(filename)
    basename_name, basename_ext = basename.split(".")
    filename = os.path.join(directory, "%s_py3.%s" % ( \
        basename_name, basename_ext))
    try:
        with open(filename, open_mode("r")) as file_handler:
            return file_handler.read().replace("\r\n", "\n")
    except IOError:
        return prior_contents

#@step(r'I create a reify output for the resource in "(.*)" for "(.*)')
def i_create_output(step, output=None, language=None, resource_type='source',
                    add_fields=False):
    if output is None and language is None:
        assert False
    world.directory = os.path.dirname(output)
    world.folders.append(world.directory)
    resource_id = getattr(world, resource_type)['resource']
    try:
        command = (u"bigmler reify --id " + resource_id +
                   u" --store --output " + output)
        if add_fields:
            command += u' --add-fields'
        command = check_debug(command)
        if not PYTHON3:
            command.encode(SYSTEM_ENCODING)
        retcode = check_call(command, shell=True)
        if retcode < 0:
            assert False
        else:
            world.output = world.directory
            assert True
    except (OSError, CalledProcessError, IOError) as exc:
        assert False, str(exc)

#@step(r'the "(.*)" file is like "(.*)"')
def i_check_output_file(step, output=None, check_file=None):
    if check_file is None or output is None:
        assert False
    check_file = res_filename(check_file)
    try:
        output_file = os.path.join(world.directory, "reify.py")
        with open(check_file, open_mode("r")) as check_file_handler:
            check_contents = check_file_handler.read().strip("\n")
        # remove unicode mark for strings if Python3
        if PYTHON3:
            check_contents = check_contents.replace( \
                " u'", " '").replace("{u'", "{'")
        with open(output_file, open_mode("r")) as output_file:
            output_file_contents = output_file.read()
        #strip comments at the beginning of the file
        output_file_contents = re.sub(r'""".*"""', '', output_file_contents,
                                      flags=re.S).strip("\n")
        if check_contents == output_file_contents:
            assert True
        else:
            if PYTHON3:
                # look for an alternative in PYTHON3
                check_contents = python3_contents(check_file, check_contents)
            if check_contents == output_file_contents:
                assert True
            else:
                assert False, ("File contents:\n%s\nExpected contents:\n%s" %
                               (output_file_contents, check_contents))
    except Exception, exc:
        assert False, str(exc)

#@step(r'I create a BigML source with data "(.*)" and params "(.*)"')
def create_source(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename), args)
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])

#@step(r'I create a BigML dataset from a source with data "(.*)" and params "(.*)"')
def create_dataset(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source, args)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])

#@step(r'I create a BigML model from a dataset with data "(.*)" and params "(.*)"')
def create_model(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.model = world.api.create_model(world.dataset, args)
    world.api.ok(world.model)
    world.models.append(world.model['resource'])

#@step(r'I create a BigML prediction for (.*) from a model with data "(.*)" and params "(.*)"')
def create_prediction(filename, input_data= None, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.model = world.api.create_model(world.dataset)
    world.api.ok(world.model)
    world.models.append(world.model['resource'])
    world.prediction = world.api.create_prediction(world.model, input_data, args)
    world.api.ok(world.prediction)
    world.predictions.append(world.prediction['resource'])

#@step(r'I create a BigML cluster from a dataset with data "(.*)" and params "(.*)"')
def create_cluster(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.cluster = world.api.create_cluster(world.dataset, args)
    world.api.ok(world.cluster)
    world.clusters.append(world.cluster['resource'])

#@step(r'I create a BigML anomaly from a dataset with data "(.*)" and params "(.*)"')
def create_anomaly(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.anomaly = world.api.create_anomaly(world.dataset, args)
    world.api.ok(world.anomaly)
    world.anomalies.append(world.anomaly['resource'])

#@step(r'I create a BigML centroid for (.*) from a cluster with data "(.*)" and params "(.*)"')
def create_centroid(filename, input_data=None, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.cluster = world.api.create_cluster(world.dataset)
    world.api.ok(world.cluster)
    world.clusters.append(world.cluster['resource'])
    world.centroid = world.api.create_centroid(world.cluster, input_data, args)
    world.api.ok(world.centroid)
    world.centroids.append(world.centroid['resource'])

#@step(r'I create a BigML anomaly score for (.*) from an anomaly detector with data "(.*)" and params "(.*)"')
def create_anomaly_score(filename, input_data=None, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.anomaly = world.api.create_anomaly(world.dataset)
    world.api.ok(world.anomaly)
    world.anomalies.append(world.anomaly['resource'])
    world.anomaly_score = world.api.create_anomaly_score(world.anomaly, input_data, args)
    world.api.ok(world.anomaly_score)
    world.anomaly_scores.append(world.anomaly_score['resource'])

#@step(r'I create a BigML batch prediction from a model with data "(.*)" and params "(.*)"')
def create_batch_prediction(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.model = world.api.create_model(world.dataset)
    world.api.ok(world.model)
    world.models.append(world.model['resource'])
    world.batch_prediction = world.api.create_batch_prediction(world.model, world.dataset, args)
    world.api.ok(world.batch_prediction)
    world.batch_predictions.append(world.batch_prediction['resource'])

#@step(r'I create a BigML batch centroid from a cluster with data "(.*)" and params "(.*)"')
def create_batch_centroid(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.cluster = world.api.create_cluster(world.dataset)
    world.api.ok(world.cluster)
    world.clusters.append(world.cluster['resource'])
    world.batch_centroid = world.api.create_batch_centroid(world.cluster, world.dataset, args)
    world.api.ok(world.batch_centroid)
    world.batch_centroids.append(world.batch_centroid['resource'])

#@step(r'I create a BigML batch anomaly score from an anomaly detector with data "(.*)" and params "(.*)"')
def create_batch_anomaly_score(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.anomaly = world.api.create_anomaly(world.dataset)
    world.api.ok(world.anomaly)
    world.anomalies.append(world.anomaly['resource'])
    world.batch_anomaly_score = world.api.create_batch_anomaly_score(world.anomaly, world.dataset, args)
    world.api.ok(world.batch_anomaly_score)
    world.batch_anomaly_scores.append(world.batch_anomaly_score['resource'])

#@step(r'I create a BigML evaluation with data "(.*)" and params "(.*)"')
def create_evaluation(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.model = world.api.create_model(world.dataset)
    world.api.ok(world.model)
    world.models.append(world.model['resource'])
    world.evaluation = world.api.create_evaluation(world.model, world.dataset, args)
    world.api.ok(world.evaluation)
    world.evaluations.append(world.evaluation['resource'])

#@step(r'I create a BigML ensemble from a dataset with data "(.*)" and params "(.*)"')
def create_ensemble(filename, output=None, args=None):
    source = world.api.create_source(res_filename(filename))
    world.source = source
    world.directory = os.path.dirname(output)
    world.output = output
    world.api.ok(world.source)
    world.sources.append(source['resource'])
    world.dataset = world.api.create_dataset(source)
    world.api.ok(world.dataset)
    world.datasets.append(world.dataset['resource'])
    world.ensemble = world.api.create_ensemble(world.dataset, args)
    world.api.ok(world.ensemble)
    world.ensembles.append(world.ensemble['resource'])
<|endoftext|>


<commit_before>from django import template
from django.template.loader import render_to_string

register = template.Library()

@register.simple_tag
def devilry_student_shortgrade(feedback):
    """
    Takes a :class:`devilry.apps.models.StaticFeedback` and renders it as a
    short oneliner that includes the grade and information about if the grade is
    passed or failed.

    Handles the grades ``Passed`` and ``Failed`` as synonyms for
    ``is_passing_grade``, so we only render a translation of ``Passed`` or
    ``Failed``, instead of both ``is_passing_grade`` and ``Grade``. This avoids
    ugly strings like: ``Passed (Passed)``.
    """
    return render_to_string('devilry_student/devilry_student_shortgrade_tag.django.html', {
        'feedback': feedback,
        'ispassedfailed': feedback.grade == 'Passed' or feedback.grade == 'Failed'
    })<commit_msg>devilry_student...devilry_student_feedback: Removed unused template variable.<commit_after>from django import template
from django.template.loader import render_to_string

register = template.Library()

@register.simple_tag
def devilry_student_shortgrade(feedback):
    """
    Takes a :class:`devilry.apps.models.StaticFeedback` and renders it as a
    short oneliner that includes the grade and information about if the grade is
    passed or failed.

    Handles the grades ``Passed`` and ``Failed`` as synonyms for
    ``is_passing_grade``, so we only render a translation of ``Passed`` or
    ``Failed``, instead of both ``is_passing_grade`` and ``Grade``. This avoids
    ugly strings like: ``Passed (Passed)``.
    """
    return render_to_string('devilry_student/devilry_student_shortgrade_tag.django.html', {
        'feedback': feedback
    })<|endoftext|>


<commit_before>#!/usr/bin/python

from mtools.mtoolbox.logline import LogLine
import webbrowser
import argparse
import sys
import time

import SimpleHTTPServer
import SocketServer

PORT = 8888

if __name__ == '__main__':
    # create parser object
    parser = argparse.ArgumentParser(description='mongod/mongos log file to json converter. \
        Extracts information from each line of the log file and outputs a json document per line. \
        To import into mongodb, use: mlog2json logfile | mongoimport -d DATABASE -c COLLECTION')

    # only create default argument if not using stdin
    if sys.stdin.isatty():
        parser.add_argument('logfile', action='store', help='logfile to convert')

    args = vars(parser.parse_args())

    # open logfile
    if sys.stdin.isatty():
        logfile = open(args['logfile'], 'r')
    else:
        logfile = sys.stdin

    outf = open('mviz_events.json', 'w')
    outf.write('[')
    for line in logfile:
        logline = LogLine(line)
        # group regular connections together
        if logline.datetime and logline.duration:
            if logline.thread and logline.thread.startswith("conn"):
                logline._thread = "conn####"
            # write log line out as json
            outf.write(logline.to_json(['line_str', 'datetime', 'operation', 'thread', 'namespace', 'nscanned', 'nreturned', 'duration']) + ",\n")
    outf.write('{}]')
    outf.close()

    Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
    httpd = SocketServer.TCPServer(("", PORT), Handler)

    print "serving visualization at port", PORT
    webbrowser.open("http://localhost:%i/"%PORT)

    httpd.serve_forever()

<commit_msg>removed mviz.<commit_after><|endoftext|>


<commit_before>import os
import tempfile

from django.conf import settings
from django.contrib.sessions.backends.base import SessionBase, CreateError
from django.core.exceptions import SuspiciousOperation, ImproperlyConfigured

class SessionStore(SessionBase):
    """
    Implements a file based session store.
    """
    def __init__(self, session_key=None):
        self.storage_path = getattr(settings, "SESSION_FILE_PATH", None)
        if not self.storage_path:
            self.storage_path = tempfile.gettempdir()

        # Make sure the storage path is valid.
        if not os.path.isdir(self.storage_path):
            raise ImproperlyConfigured(
                "The session storage path %r doesn't exist. Please set your"
                " SESSION_FILE_PATH setting to an existing directory in which"
                " Django can store session data." % self.storage_path)

        self.file_prefix = settings.SESSION_COOKIE_NAME
        super(SessionStore, self).__init__(session_key)

    def _key_to_file(self, session_key=None):
        """
        Get the file associated with this session key.
        """
        if session_key is None:
            session_key = self.session_key

        # Make sure we're not vulnerable to directory traversal. Session keys
        # should always be md5s, so they should never contain directory
        # components.
        if os.path.sep in session_key:
            raise SuspiciousOperation(
                "Invalid characters (directory components) in session key")

        return os.path.join(self.storage_path, self.file_prefix + session_key)

    def load(self):
        session_data = {}
        try:
            session_file = open(self._key_to_file(), "rb")
            try:
                try:
                    session_data = self.decode(session_file.read())
                except (EOFError, SuspiciousOperation):
                    self.create()
            finally:
                session_file.close()
        except IOError:
            pass
        return session_data

    def create(self):
        while True:
            self._session_key = self._get_new_session_key()
            try:
                self.save(must_create=True)
            except CreateError:
                continue
            self.modified = True
            self._session_cache = {}
            return

    def save(self, must_create=False):
        flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, 'O_BINARY', 0)
        if must_create:
            flags |= os.O_EXCL
        # Because this may trigger a load from storage, we must do it before
        # truncating the file to save.
        session_data = self._get_session(no_load=must_create)
        try:
            fd = os.open(self._key_to_file(self.session_key), flags)
            try:
                os.write(fd, self.encode(session_data))
            finally:
                os.close(fd)
        except OSError, e:
            if must_create and e.errno == errno.EEXIST:
                raise CreateError
            raise
        except (IOError, EOFError):
            pass

    def exists(self, session_key):
        if os.path.exists(self._key_to_file(session_key)):
            return True
        return False

    def delete(self, session_key=None):
        if session_key is None:
            if self._session_key is None:
                return
            session_key = self._session_key
        try:
            os.unlink(self._key_to_file(session_key))
        except OSError:
            pass

    def clean(self):
        pass
<commit_msg>Fixed #8457 -- Fixed a missing import.<commit_after>import errno
import os
import tempfile

from django.conf import settings
from django.contrib.sessions.backends.base import SessionBase, CreateError
from django.core.exceptions import SuspiciousOperation, ImproperlyConfigured

class SessionStore(SessionBase):
    """
    Implements a file based session store.
    """
    def __init__(self, session_key=None):
        self.storage_path = getattr(settings, "SESSION_FILE_PATH", None)
        if not self.storage_path:
            self.storage_path = tempfile.gettempdir()

        # Make sure the storage path is valid.
        if not os.path.isdir(self.storage_path):
            raise ImproperlyConfigured(
                "The session storage path %r doesn't exist. Please set your"
                " SESSION_FILE_PATH setting to an existing directory in which"
                " Django can store session data." % self.storage_path)

        self.file_prefix = settings.SESSION_COOKIE_NAME
        super(SessionStore, self).__init__(session_key)

    def _key_to_file(self, session_key=None):
        """
        Get the file associated with this session key.
        """
        if session_key is None:
            session_key = self.session_key

        # Make sure we're not vulnerable to directory traversal. Session keys
        # should always be md5s, so they should never contain directory
        # components.
        if os.path.sep in session_key:
            raise SuspiciousOperation(
                "Invalid characters (directory components) in session key")

        return os.path.join(self.storage_path, self.file_prefix + session_key)

    def load(self):
        session_data = {}
        try:
            session_file = open(self._key_to_file(), "rb")
            try:
                try:
                    session_data = self.decode(session_file.read())
                except (EOFError, SuspiciousOperation):
                    self.create()
            finally:
                session_file.close()
        except IOError:
            pass
        return session_data

    def create(self):
        while True:
            self._session_key = self._get_new_session_key()
            try:
                self.save(must_create=True)
            except CreateError:
                continue
            self.modified = True
            self._session_cache = {}
            return

    def save(self, must_create=False):
        flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, 'O_BINARY', 0)
        if must_create:
            flags |= os.O_EXCL
        # Because this may trigger a load from storage, we must do it before
        # truncating the file to save.
        session_data = self._get_session(no_load=must_create)
        try:
            fd = os.open(self._key_to_file(self.session_key), flags)
            try:
                os.write(fd, self.encode(session_data))
            finally:
                os.close(fd)
        except OSError, e:
            if must_create and e.errno == errno.EEXIST:
                raise CreateError
            raise
        except (IOError, EOFError):
            pass

    def exists(self, session_key):
        if os.path.exists(self._key_to_file(session_key)):
            return True
        return False

    def delete(self, session_key=None):
        if session_key is None:
            if self._session_key is None:
                return
            session_key = self._session_key
        try:
            os.unlink(self._key_to_file(session_key))
        except OSError:
            pass

    def clean(self):
        pass
<|endoftext|>


<commit_before>'''
Emulate a USB device via GadgetFS (Linux only)

This module utilizes the GagdetFS module in Linux,
which allows a user-space applications to implement USB devices on supported
platforms by reading and writing into files.

Due to the blocking nature of the endpoint files, and since there is no
implementation for poll/select for the endpoints, we create a separate thread
for each endpoint when a USB host is connected to the device.

.. note::

    Before kernel v4.8, there was a bug in the sync i/o mechanism of the
    GadgetFS kernel module, and a patched module is required in order to use
    this feature with Umap2.

    Even in v4.8 GadgetFS requires valid descriptors to be set in order to use
    the module, and then caches them for further use. This means that you
    cannot perform fuzzing of the enumeration with this module unless you use
    the patched version of the kernel module (code available in this repo
    under gadget/inode.c, but might require some tinkering, depending on your
    kernel version).
'''
import platform
import time
import struct
import select
import os
from binascii import hexlify
import threading

from six.moves.queue import Queue, Empty

from umap2.core.usb import Request, DescriptorType
from umap2.core.usb_device import USBDeviceRequest
from umap2.core.usb_endpoint import USBEndpoint
from umap2.phy.iphy import PhyInterface

GFS_CMD_INIT_DEVICE = 0
GFS_CMD_INIT_EP = 1

GFS_EV_NOP = 0
GFS_EV_CONNECT = 1
GFS_EV_DISCONNECT = 2
GFS_EV_SETUP = 3
GFS_EV_SUSPEND = 4

GFS_EVENT_SIZE = 12

GFS_EVENT_TYPE_OFFSET = 8

def filter_descriptors(data, keep_dt):
    '''
    Keep only descriptors with given descriptor type

    :param data: descriptors buffer
    :param keep_dt: type of descriptor to keep
    :return: buffer only with keep_dt descriptors
    '''
    i = 0
    filtered = ''
    while i < len(data) - 2:
        dlen, dtype = struct.unpack('BB', data[i:i + 2])
        if dtype == keep_dt:
            filtered += data[i:i + dlen]
        i += dlen
    return filtered

def set_highspeed_endpoints(dev):
    '''
    If we are in high-speed mode, and this is a bulk endpoint,
    use the high-speed max packet size.

    :param dev: USBDevice that will be modified
    '''
    for conf in dev.configurations:
        for iface in conf.interfaces:
            for ep in iface.endpoints:
                if ep.transfer_type == USBEndpoint.transfer_type_bulk:
                    ep.max_packet_size = ep._get_max_packet_size('highspeed')

class GadgetFsPhy(PhyInterface):
    '''
    Physical layer based on GadgetFS
    '''

    control_filenames = [
        'net2280',
        'gfs_udc',
        'pxa2xx_udc',
        'goku_udc',
        'sh_udc',
        'omap_udc',
        'musb-hdrc',
        'at91_udc',
        'lh740x_udc',
        'atmel_usba_udc',
    ]

    def __init__(self, app, gadgetfs_dir='/dev/gadget'):
        super(GadgetFsPhy, self).__init__(app, 'GadgetFsPhy')
        if platform.system() != 'Linux':
            raise Exception('GadgetFsPhy is only supported on Linux')
        self.gadgetfs_dir = gadgetfs_dir
        self.control_fd = None
        self.control_filename = self._get_control_filename()
        self.configured = False
        # we need to use separate thread for each endpoint ...
        self.ep_threads = {}
        self.in_ep_threads = []

    def _get_control_filename(self):
        '''
        Get the control filename, depending on the USB driver.
        If there's more than one, the first match (alphabetically)
        will be returned.

        :return: control (EP0) filename
        :raises: exception if no control file found
        '''
        for filename in os.listdir(self.gadgetfs_dir):
            if filename in GadgetFsPhy.control_filenames:
                full_path = os.path.join(self.gadgetfs_dir, filename)
                self.info('Found a control file: %s' % (full_path))
                return full_path
        raise Exception(
            'No control file found in %s. Is the gadgetfs driver loaded?' % (self.gadgetfs_dir)
        )

    def _is_high_speed(self):
        '''
        .. todo: implement
        '''
        return True

    def connect(self, device):
        super(GadgetFsPhy, self).connect(device)
        self.control_fd = os.open(self.control_filename, os.O_RDWR | os.O_NONBLOCK)
        self.debug('Opened control file: %s' % (self.control_filename))
        set_highspeed_endpoints(self.connected_device)
        buff = struct.pack('I', GFS_CMD_INIT_DEVICE)
        for conf in self.connected_device.configurations:
            buff += conf.get_descriptor(usb_type='fullspeed', valid=True)
            if self._is_high_speed():
                buff += conf.get_descriptor(usb_type='highspeed', valid=True)  # might need to specify USB version here...
        buff += self.connected_device.get_descriptor(valid=True)
        self.verbose('About to write %#x configuration bytes to control file (%d)' % (len(buff), self.control_fd))
        os.write(self.control_fd, buff)
        self.verbose('Write completed')

    def disconnect(self):
        # signal threads to stop the loop
        for _, t in self.ep_threads.items():
            t.stop_evt.set()
        # close all file descriptors
        fds = [t.ep.fd for (_, t) in self.ep_threads.items()]
        for fd in fds:
            os.close(fd)
            self.verbose('Closed fd: %d' % (fd))
        if self.control_fd:
            os.close(self.control_fd)
        self.control_fd = None
        # now, wait for all threads to complete
        for ep_address, t in self.ep_threads.items():
            self.verbose('closing thread for endpoint %#x' % (ep_address))
            t.join()
        self.ep_threads = {}
        self.in_ep_threads = []
        return super(GadgetFsPhy, self).disconnect()

    def run(self):
        '''
        run loop for handling control (endpoint 0) events
        '''
        self.debug('Started run loop')
        self.stop = False
        while not self.stop:
            ready_eps, _, _ = select.select([self.control_fd], [], [], 0.001)
            if ready_eps:
                self._handle_ep0()
            if self.app.should_stop_phy():
                self.stop = True
            for ept in self.in_ep_threads:
                if not ept.handling_write():
                    self.connected_device.handle_buffer_available(ept.ep.number)
        self.debug('Done with run loop')

    def send_on_endpoint(self, ep_num, data):
        self.debug('send_on_endpoint %d(%d): %s' % (ep_num, len(data), hexlify(data)))
        address = ep_num | 0x80
        if ep_num == 0:
            self.send_on_ep0(data)
        elif address in self.ep_threads:
            self.ep_threads[address].send(data)
        else:
            raise Exception('No IN endpoint %#x (address %#x)' % (ep_num, address))

    def send_on_ep0(self, data):
        if data:
            os.write(self.control_fd, data)
            self.debug('Done writing %d bytes to control endpoint (0)' % (len(data)))
        else:
            self.stall_ep0()

    def stall_ep0(self):
        self.debug('Stalling EP0')
        try:
            if self.req_direction == Request.direction_host_to_device:
                os.write(self.control_fd, b'')
            else:
                os.read(self.control_fd, 0)
        except OSError as oe:
            # 51: Level two halted (e.g. stalled)
            if oe.errno == 51:
                pass

    def _handle_ep0(self):
        # read event
        while True:
            try:
                events = os.read(self.control_fd, GFS_EVENT_SIZE * 5)
                break
            except OSError as ose:
                if ose.errno == 11:  # resource temporarily busy
                    time.sleep(0.01)
                    continue
                raise
        self.info('length of events buffer: %#x' % (len(events)))
        for i in range(0, len(events), GFS_EVENT_SIZE):
            event = events[i:i + GFS_EVENT_SIZE]
            if len(event) < GFS_EVENT_SIZE:
                msg = 'Did not read full event (%d/%d)' % (len(event), GFS_EVENT_SIZE)
                self.error(msg)
                raise Exception(msg)
            event_type = struct.unpack('<I', event[GFS_EVENT_TYPE_OFFSET:GFS_EVENT_TYPE_OFFSET + 4])[0]
            if event_type == GFS_EV_NOP:
                self._handle_ep0_nop(event)
            elif event_type == GFS_EV_CONNECT:
                self._handle_ep0_connect(event)
            elif event_type == GFS_EV_DISCONNECT:
                self._handle_ep0_disconnect(event)
            elif event_type == GFS_EV_SETUP:
                self._handle_ep0_setup(event)
            elif event_type == GFS_EV_SUSPEND:
                self._handle_ep0_suspend(event)
            else:
                self.warning('Got unknown event type for EP0 %#x' % (event_type))

    def _handle_ep0_nop(self, event):
        self.debug('EP0 event type NOP(%#x)' % (GFS_EV_NOP))

    def _handle_ep0_connect(self, event):
        self.debug('EP0 event type CONNECT(%#x)' % (GFS_EV_CONNECT))

    def _handle_ep0_disconnect(self, event):
        self.debug('EP0 event type DISCONNECT(%#x)' % (GFS_EV_DISCONNECT))

    def _handle_ep0_setup(self, event):
        self.debug('EP0 event type SETUP(%#x)' % (GFS_EV_SETUP))
        self.app.signal_setup_packet_received()
        # read setup data (offset in event)
        setup_data = event[:GFS_EVENT_TYPE_OFFSET]
        req = USBDeviceRequest(setup_data)
        self.req_direction = req.get_direction()
        if self.req_direction == Request.direction_host_to_device and req.length > 0:
            # Turns out, that at this point we cannot
            #
            self.info('expecting additional data on control ep - %#x bytes' % (req.length))
            self.info(req)
            data = os.read(self.control_fd, req.length)
            if len(data) != req.length:
                self.error('EP0 data have wrong length')
            else:
                req.data = data
        self.connected_device.handle_request(setup_data)
        self.configured = self.connected_device.configuration is not None

    def _setup_endpoints(self):
        conf = self.connected_device.configuration
        for iface in conf.interfaces:
            for ep in iface.endpoints:
                self._setup_endpoint(ep)

    def _setup_endpoint(self, ep):
        if self._ep_already_opened(ep):
            self._update_ep(ep)
        else:
            self._open_endpoint_fd(ep)
            buff = struct.pack('I', GFS_CMD_INIT_EP)
            descs = ep.get_descriptor(usb_type='fullspeed', valid=True)
            if self._is_high_speed():
                descs += ep.get_descriptor(usb_type='highspeed', valid=True)
            descs = filter_descriptors(descs, DescriptorType.endpoint)
            buff += descs
            os.write(ep.fd, buff)
            if ep.direction == USBEndpoint.direction_out:
                t = OutEpThread(self, ep)
            else:
                t = InEpThread(self, ep)
                self.in_ep_threads.append(t)
            self.ep_threads[ep.address] = t
            t.start()

    def _ep_already_opened(self, ep):
        return ep.address in self.ep_threads

    def _update_ep(self, ep):
        self.ep_threads[ep.address].ep = ep

    def _open_endpoint_fd(self, ep):
        '''
        :param ep: USBEndpoint object

        :raises Exception: if no endpoint file found, or failed to open

        .. todo:: detect transfer-type specific endpoint files
        '''
        num = ep.number
        s_dir = 'out' if ep.direction == USBEndpoint.direction_out else 'in'
        filename = 'ep%d%s' % (num, s_dir)
        path = os.path.join(self.gadgetfs_dir, filename)
        fd = os.open(path, os.O_RDWR | os.O_NONBLOCK)
        self.debug('Opened endpoint %d' % (num))
        self.debug('ep: %d dir: %s file: %s fd: %d' % (num, s_dir, filename, fd))
        ep.fd = fd

    def _handle_ep0_suspend(self, event):
        self.debug('EP0 event type SUSPEND(%#x)' % (GFS_EV_SUSPEND))

    def ack_status_stage(self):
        os.read(self.control_fd, 0)
        self._setup_endpoints()

class EndpointThread(threading.Thread):
    '''Thread for endpoint I/O'''

    def __init__(self, phy, ep):
        super(EndpointThread, self).__init__()
        self.phy = phy
        self.ep = ep
        self.stop_evt = threading.Event()

    def run(self):
        self.phy.debug('Starting thread for EP %#x' % (self.ep.address))
        first = True
        while not self.stop_evt.isSet():
            try:
                self.io_op()
            except OSError as err:
                # bad fd, transport socket closed
                if err.errno in [9, 108] and first:
                    first = False
                    continue
                self.phy.error('Error in EP%d handling thread: %s' % (self.ep.number, err))

class InEpThread(EndpointThread):

    def __init__(self, phy, ep):
        super(InEpThread, self).__init__(phy, ep)
        self.queue = Queue()

    def send(self, data):
        self.queue.put(data)

    def handling_write(self):
        return not self.queue.empty()

    def io_op(self):
        '''
         Fetch data from send queue and write to endpoint
        '''
        try:
            data = self.queue.get(True, 0.1)
            os.write(self.ep.fd, data)
        except Empty:
            pass

class OutEpThread(EndpointThread):

    def __init__(self, phy, ep):
        super(OutEpThread, self).__init__(phy, ep)
        self.read_size = self.ep._get_max_packet_size('highspeed')

    def io_op(self):
        '''
        read data from endpoint fd and let the endpoint handle it
        '''
        self.phy.debug('About to read from EP%d' % (self.ep.number))
        buff = os.read(self.ep.fd, self.read_size)
        self.phy.debug('Done reading from EP%d' % (self.ep.number))
        self.phy.connected_device.handle_data_available(self.ep.number, buff)

<commit_msg>[GadgetFsPhy] relax logging<commit_after>'''
Emulate a USB device via GadgetFS (Linux only)

This module utilizes the GagdetFS module in Linux,
which allows a user-space applications to implement USB devices on supported
platforms by reading and writing into files.

Due to the blocking nature of the endpoint files, and since there is no
implementation for poll/select for the endpoints, we create a separate thread
for each endpoint when a USB host is connected to the device.

.. note::

    Before kernel v4.8, there was a bug in the sync i/o mechanism of the
    GadgetFS kernel module, and a patched module is required in order to use
    this feature with Umap2.

    Even in v4.8 GadgetFS requires valid descriptors to be set in order to use
    the module, and then caches them for further use. This means that you
    cannot perform fuzzing of the enumeration with this module unless you use
    the patched version of the kernel module (code available in this repo
    under gadget/inode.c, but might require some tinkering, depending on your
    kernel version).
'''
import platform
import time
import struct
import select
import os
from binascii import hexlify
import threading

from six.moves.queue import Queue, Empty

from umap2.core.usb import Request, DescriptorType
from umap2.core.usb_device import USBDeviceRequest
from umap2.core.usb_endpoint import USBEndpoint
from umap2.phy.iphy import PhyInterface

GFS_CMD_INIT_DEVICE = 0
GFS_CMD_INIT_EP = 1

GFS_EV_NOP = 0
GFS_EV_CONNECT = 1
GFS_EV_DISCONNECT = 2
GFS_EV_SETUP = 3
GFS_EV_SUSPEND = 4

GFS_EVENT_SIZE = 12

GFS_EVENT_TYPE_OFFSET = 8

def filter_descriptors(data, keep_dt):
    '''
    Keep only descriptors with given descriptor type

    :param data: descriptors buffer
    :param keep_dt: type of descriptor to keep
    :return: buffer only with keep_dt descriptors
    '''
    i = 0
    filtered = ''
    while i < len(data) - 2:
        dlen, dtype = struct.unpack('BB', data[i:i + 2])
        if dtype == keep_dt:
            filtered += data[i:i + dlen]
        i += dlen
    return filtered

def set_highspeed_endpoints(dev):
    '''
    If we are in high-speed mode, and this is a bulk endpoint,
    use the high-speed max packet size.

    :param dev: USBDevice that will be modified
    '''
    for conf in dev.configurations:
        for iface in conf.interfaces:
            for ep in iface.endpoints:
                if ep.transfer_type == USBEndpoint.transfer_type_bulk:
                    ep.max_packet_size = ep._get_max_packet_size('highspeed')

class GadgetFsPhy(PhyInterface):
    '''
    Physical layer based on GadgetFS
    '''

    control_filenames = [
        'net2280',
        'gfs_udc',
        'pxa2xx_udc',
        'goku_udc',
        'sh_udc',
        'omap_udc',
        'musb-hdrc',
        'at91_udc',
        'lh740x_udc',
        'atmel_usba_udc',
    ]

    def __init__(self, app, gadgetfs_dir='/dev/gadget'):
        super(GadgetFsPhy, self).__init__(app, 'GadgetFsPhy')
        if platform.system() != 'Linux':
            raise Exception('GadgetFsPhy is only supported on Linux')
        self.gadgetfs_dir = gadgetfs_dir
        self.control_fd = None
        self.control_filename = self._get_control_filename()
        self.configured = False
        # we need to use separate thread for each endpoint ...
        self.ep_threads = {}
        self.in_ep_threads = []

    def _get_control_filename(self):
        '''
        Get the control filename, depending on the USB driver.
        If there's more than one, the first match (alphabetically)
        will be returned.

        :return: control (EP0) filename
        :raises: exception if no control file found
        '''
        for filename in os.listdir(self.gadgetfs_dir):
            if filename in GadgetFsPhy.control_filenames:
                full_path = os.path.join(self.gadgetfs_dir, filename)
                self.info('Found a control file: %s' % (full_path))
                return full_path
        raise Exception(
            'No control file found in %s. Is the gadgetfs driver loaded?' % (self.gadgetfs_dir)
        )

    def _is_high_speed(self):
        '''
        .. todo: implement
        '''
        return True

    def connect(self, device):
        super(GadgetFsPhy, self).connect(device)
        self.control_fd = os.open(self.control_filename, os.O_RDWR | os.O_NONBLOCK)
        self.debug('Opened control file: %s' % (self.control_filename))
        set_highspeed_endpoints(self.connected_device)
        buff = struct.pack('I', GFS_CMD_INIT_DEVICE)
        for conf in self.connected_device.configurations:
            buff += conf.get_descriptor(usb_type='fullspeed', valid=True)
            if self._is_high_speed():
                buff += conf.get_descriptor(usb_type='highspeed', valid=True)  # might need to specify USB version here...
        buff += self.connected_device.get_descriptor(valid=True)
        self.verbose('About to write %#x configuration bytes to control file (%d)' % (len(buff), self.control_fd))
        os.write(self.control_fd, buff)
        self.verbose('Write completed')

    def disconnect(self):
        # signal threads to stop the loop
        for _, t in self.ep_threads.items():
            t.stop_evt.set()
        # close all file descriptors
        fds = [t.ep.fd for (_, t) in self.ep_threads.items()]
        for fd in fds:
            os.close(fd)
            self.verbose('Closed fd: %d' % (fd))
        if self.control_fd:
            os.close(self.control_fd)
        self.control_fd = None
        # now, wait for all threads to complete
        for ep_address, t in self.ep_threads.items():
            self.verbose('closing thread for endpoint %#x' % (ep_address))
            t.join()
        self.ep_threads = {}
        self.in_ep_threads = []
        return super(GadgetFsPhy, self).disconnect()

    def run(self):
        '''
        run loop for handling control (endpoint 0) events
        '''
        self.debug('Started run loop')
        self.stop = False
        while not self.stop:
            ready_eps, _, _ = select.select([self.control_fd], [], [], 0.001)
            if ready_eps:
                self._handle_ep0()
            if self.app.should_stop_phy():
                self.stop = True
            for ept in self.in_ep_threads:
                if not ept.handling_write():
                    self.connected_device.handle_buffer_available(ept.ep.number)
        self.debug('Done with run loop')

    def send_on_endpoint(self, ep_num, data):
        self.debug('send_on_endpoint %d(%d): %s' % (ep_num, len(data), hexlify(data)))
        address = ep_num | 0x80
        if ep_num == 0:
            self.send_on_ep0(data)
        elif address in self.ep_threads:
            self.ep_threads[address].send(data)
        else:
            raise Exception('No IN endpoint %#x (address %#x)' % (ep_num, address))

    def send_on_ep0(self, data):
        if data:
            os.write(self.control_fd, data)
            self.debug('Done writing %d bytes to control endpoint (0)' % (len(data)))
        else:
            self.stall_ep0()

    def stall_ep0(self):
        self.debug('Stalling EP0')
        try:
            if self.req_direction == Request.direction_host_to_device:
                os.write(self.control_fd, b'')
            else:
                os.read(self.control_fd, 0)
        except OSError as oe:
            # 51: Level two halted (e.g. stalled)
            if oe.errno == 51:
                pass

    def _handle_ep0(self):
        # read event
        while True:
            try:
                events = os.read(self.control_fd, GFS_EVENT_SIZE * 5)
                break
            except OSError as ose:
                if ose.errno == 11:  # resource temporarily busy
                    time.sleep(0.01)
                    continue
                raise
        for i in range(0, len(events), GFS_EVENT_SIZE):
            event = events[i:i + GFS_EVENT_SIZE]
            if len(event) < GFS_EVENT_SIZE:
                msg = 'Did not read full event (%d/%d)' % (len(event), GFS_EVENT_SIZE)
                self.error(msg)
                raise Exception(msg)
            event_type = struct.unpack('<I', event[GFS_EVENT_TYPE_OFFSET:GFS_EVENT_TYPE_OFFSET + 4])[0]
            if event_type == GFS_EV_NOP:
                self._handle_ep0_nop(event)
            elif event_type == GFS_EV_CONNECT:
                self._handle_ep0_connect(event)
            elif event_type == GFS_EV_DISCONNECT:
                self._handle_ep0_disconnect(event)
            elif event_type == GFS_EV_SETUP:
                self._handle_ep0_setup(event)
            elif event_type == GFS_EV_SUSPEND:
                self._handle_ep0_suspend(event)
            else:
                self.warning('Got unknown event type for EP0 %#x' % (event_type))

    def _handle_ep0_nop(self, event):
        self.debug('EP0 event type NOP(%#x)' % (GFS_EV_NOP))

    def _handle_ep0_connect(self, event):
        self.debug('EP0 event type CONNECT(%#x)' % (GFS_EV_CONNECT))

    def _handle_ep0_disconnect(self, event):
        self.debug('EP0 event type DISCONNECT(%#x)' % (GFS_EV_DISCONNECT))

    def _handle_ep0_setup(self, event):
        self.debug('EP0 event type SETUP(%#x)' % (GFS_EV_SETUP))
        self.app.signal_setup_packet_received()
        # read setup data (offset in event)
        setup_data = event[:GFS_EVENT_TYPE_OFFSET]
        req = USBDeviceRequest(setup_data)
        self.req_direction = req.get_direction()
        if self.req_direction == Request.direction_host_to_device and req.length > 0:
            # Turns out, that at this point we cannot
            #
            self.debug(req)
            self.debug('expecting additional data on control ep - %#x bytes' % (req.length))
            data = os.read(self.control_fd, req.length)
            if len(data) != req.length:
                self.error('EP0 data have wrong length')
            else:
                req.data = data
        self.connected_device.handle_request(setup_data)
        self.configured = self.connected_device.configuration is not None

    def _setup_endpoints(self):
        conf = self.connected_device.configuration
        for iface in conf.interfaces:
            for ep in iface.endpoints:
                self._setup_endpoint(ep)

    def _setup_endpoint(self, ep):
        if self._ep_already_opened(ep):
            self._update_ep(ep)
        else:
            self._open_endpoint_fd(ep)
            buff = struct.pack('I', GFS_CMD_INIT_EP)
            descs = ep.get_descriptor(usb_type='fullspeed', valid=True)
            if self._is_high_speed():
                descs += ep.get_descriptor(usb_type='highspeed', valid=True)
            descs = filter_descriptors(descs, DescriptorType.endpoint)
            buff += descs
            os.write(ep.fd, buff)
            if ep.direction == USBEndpoint.direction_out:
                t = OutEpThread(self, ep)
            else:
                t = InEpThread(self, ep)
                self.in_ep_threads.append(t)
            self.ep_threads[ep.address] = t
            t.start()

    def _ep_already_opened(self, ep):
        return ep.address in self.ep_threads

    def _update_ep(self, ep):
        self.ep_threads[ep.address].ep = ep

    def _open_endpoint_fd(self, ep):
        '''
        :param ep: USBEndpoint object

        :raises Exception: if no endpoint file found, or failed to open

        .. todo:: detect transfer-type specific endpoint files
        '''
        num = ep.number
        s_dir = 'out' if ep.direction == USBEndpoint.direction_out else 'in'
        filename = 'ep%d%s' % (num, s_dir)
        path = os.path.join(self.gadgetfs_dir, filename)
        fd = os.open(path, os.O_RDWR | os.O_NONBLOCK)
        self.debug('Opened endpoint %d' % (num))
        self.debug('ep: %d dir: %s file: %s fd: %d' % (num, s_dir, filename, fd))
        ep.fd = fd

    def _handle_ep0_suspend(self, event):
        self.debug('EP0 event type SUSPEND(%#x)' % (GFS_EV_SUSPEND))

    def ack_status_stage(self):
        os.read(self.control_fd, 0)
        self._setup_endpoints()

class EndpointThread(threading.Thread):
    '''Thread for endpoint I/O'''

    def __init__(self, phy, ep):
        super(EndpointThread, self).__init__()
        self.phy = phy
        self.ep = ep
        self.stop_evt = threading.Event()

    def run(self):
        self.phy.debug('Starting thread for EP %#x' % (self.ep.address))
        first = True
        while not self.stop_evt.isSet():
            try:
                self.io_op()
            except OSError as err:
                # bad fd, transport socket closed
                if err.errno in [9, 108] and first:
                    first = False
                    continue
                self.phy.error('Error in EP%d handling thread: %s' % (self.ep.number, err))

class InEpThread(EndpointThread):

    def __init__(self, phy, ep):
        super(InEpThread, self).__init__(phy, ep)
        self.queue = Queue()

    def send(self, data):
        self.queue.put(data)

    def handling_write(self):
        return not self.queue.empty()

    def io_op(self):
        '''
         Fetch data from send queue and write to endpoint
        '''
        try:
            data = self.queue.get(True, 0.1)
            os.write(self.ep.fd, data)
        except Empty:
            pass

class OutEpThread(EndpointThread):

    def __init__(self, phy, ep):
        super(OutEpThread, self).__init__(phy, ep)
        self.read_size = self.ep._get_max_packet_size('highspeed')

    def io_op(self):
        '''
        read data from endpoint fd and let the endpoint handle it
        '''
        self.phy.debug('About to read from EP%d' % (self.ep.number))
        buff = os.read(self.ep.fd, self.read_size)
        self.phy.debug('Done reading from EP%d' % (self.ep.number))
        self.phy.connected_device.handle_data_available(self.ep.number, buff)

<|endoftext|>


<commit_before>#coding=utf8
from uliweb import functions, url_for
import logging
from forms import QueryForm

log = logging.getLogger(__name__)
class Default(object):pass

class MultiView(object):
    """
    Multi View Class

    Support: list, add, delete, edit, detail, etc
    """

    def _process_fields_convert_map(self, parameters):
        """
        process fields_convert_map, ListView doesn't support list type but dict

        fields_convert_map should be define as list or dict
        for list, it can be:
            [name, name, ...]
            [(name, func), (name, func), ...] if func is str, it'll be the property name of class
        for dict, it can be:
            {'name':func, ...}
        :param model: model object
        :param parameters:
        :param prefix: it'll used to combine prefix+_convert_xxx to get convert function
            from class
        :return:
        """
        if 'fields_convert_map' in parameters:
            _f = parameters.get('fields_convert_map') or []
            parameters['fields_convert_map'] = self._get_fields_convert_map(_f)

    def _get_fields_convert_map(self, fields):
        """
        process fields_convert_map, ListView doesn't support list type but dict

        fields_convert_map should be define as list or dict
        for list, it can be:
            [name, name, ...]
            [(name, func), (name, func), ...] if func is str, it'll be the property name of class
        for dict, it can be:
            {'name':func, ...}
        :param model: model object
        :param parameters:
        :param prefix: it'll used to combine prefix+_convert_xxx to get convert function
            from class
        :return:
        """
        _f = fields
        t = {}
        if isinstance(_f, list):
            for k in _f:
                if isinstance(k, str):
                    t[k] = getattr(self, '_convert_{}'.format(k))
                elif isinstance(k, (tuple, list)):
                    name = k[0]
                    func = k[1]
                    if isinstance(func, str):
                        t[name] = getattr(self, '_convert_{}'.format(func))
                    elif callable(func):
                        t[name] = func
                    else:
                        raise ValueError("Fields convert function should be str or callable, but %r found" % type(func))
                else:
                    raise ValueError("Fields convert element should be str or tuple or list, but %r found" % type(k))
        elif isinstance(_f, dict):
            for k, v in _f.items():
                if isinstance(v, str):
                    t[k] = getattr(self, '_convert_{}'.format(v))
                elif callable(v):
                    t[k] = v
                else:
                    raise ValueError("Fields convert function should be str or callable, but %r found" % type(func))
        return t

    def _list_view(self, model, **kwargs):
        """
        :param model:
        :param fields_convert_map: it's different from ListView
        :param kwargs:
        :return:
        """
        view =  functions.ListView(model, **kwargs)
        return view

    def _query_view(self, model, **kwargs):
        """
        :param model:
        :return: (query, condition)

        Default use QueryForm
        """
        QueryForm = functions.get_form('QueryForm')

        if 'form_cls' not in kwargs:
            kwargs['form_cls'] = QueryForm
        query = functions.QueryView(model, **kwargs)
        return query

    def _list(self, model, queryview=None, queryform=None, **kwargs):
        from uliweb import request, json, CONTENT_TYPE_JSON
        from sqlalchemy import and_
        from uliweb.utils.generic import get_sort_field

        if queryview:
            queryview.run()
            condition = queryview.get_condition()
        else:
            condition = None

        if 'condition' in kwargs:
            condition = and_(condition, kwargs['condition'])
            kwargs['condition'] = condition
        else:
            kwargs['condition'] = condition

        #process order
        if 'order_by' not in kwargs:
            order_by = get_sort_field(model)
            if order_by is not None:
                kwargs['order_by'] = order_by

        self._process_fields_convert_map(kwargs)
        downloads = {}
        downloads['filename'] = kwargs.pop('download_filename', 'download.xlsx')
        downloads['action'] = kwargs.pop('download_action', 'download')
        downloads['fields_convert_map'] = kwargs.pop('download_fields_convert_map',
                                                  kwargs.get('fields_convert_map'))
        downloads['domain'] = kwargs.pop('download_domain', '')
        downloads['timeout'] = 0
        downloads.update(kwargs.pop('download_kwargs', {}))
        self._process_fields_convert_map(downloads)

        #get list view
        view = self._list_view(model=model, **kwargs)

        if 'data' in request.values and request.is_xhr:
            return json(view.json(), content_type=CONTENT_TYPE_JSON)
        elif 'download' in request.GET:
            return view.download(**downloads)
        else:
            result = view.run()
            if queryview:
                result.update({'query_form':queryform or queryview.form})
            else:
                result.update({'query_form':''})
            result.update({'table':view})
            return result

    def _view(self, model, obj, **kwargs):
        self._process_fields_convert_map(kwargs)

        view = functions.DetailView(model, obj=obj, **kwargs)
        return view.run()

    def _add(self, model, json_result=False, **kwargs):
        self._process_fields_convert_map(kwargs)

        view = functions.AddView(model, **kwargs)
        return view.run(json_result=json_result)

    def _edit(self, model, obj, json_result=False, **kwargs):
        self._process_fields_convert_map(kwargs)

        view = functions.EditView(model, obj=obj, **kwargs)
        return view.run(json_result=json_result)

    def _delete(self, model, obj, json_result=False, **kwargs):
        view = functions.DeleteView(model, obj=obj, **kwargs)
        return view.run(json_result=json_result)

    def _select_list(self, queryview=None, download_filename=None, **kwargs):
        from uliweb import request, json

        if queryview:
            queryview.run()
            condition = queryview.get_condition()
        else:
            condition = None

        if 'condition' in kwargs:
            condition = and_(condition, kwargs['condition'])
            kwargs['condition'] = condition
        else:
            kwargs['condition'] = condition

        view = functions.SelectListView(**kwargs)
        if 'data' in request.values:
            return json(view.json())
        elif 'download' in request.GET:
            filename = download_filename or 'download.xls'
            kwargs.setdefault('action', 'download')
            kwargs.setdefault('timeout', 0)
            return view.download(filename, **kwargs)
        else:
            result = view.run()
            if queryview:
                result.update({'query_form':queryview.form})
            else:
                result.update({'query_form':''})
            result.update({'table':view})
            return result

    def _search(self, model, condition=None, search_field='name',
                value_field='id', label_field=None, pagination=True):
        """
        Default search function
        :param search_field: Used for search field, default is 'name'
        :param value_field: Used for id field, default is id
        :param label_field: Used for label field, default is None, then it'll use unicode() function
        """
        from uliweb import json, request

        name = request.GET.get('term', '')
        M = functions.get_model(model)

        def _v(label_field):
            if label_field:
                return lambda x: getattr(x, label_field)
            else:
                return lambda x: unicode(x)

        v_field = request.values.get('label', 'title')
        page = int(request.values.get('page') or 1)
        limit = int(request.values.get('limit') or 10)
        v_func = _v(label_field)
        if name:
            if condition is None:
                condition = M.c[search_field].like('%' + name + '%')
            if pagination:
                query = M.filter(condition)
                total = query.count()
                rows = [{'id': getattr(obj, value_field), v_field: v_func(obj)}
                            for obj in query.limit(limit).offset((page-1)*limit)]
                result = {'total':total, 'rows':rows}
            else:
                result = [{'id': getattr(obj, value_field), v_field: v_func(obj)}
                      for obj in M.filter(condition)]
        else:
            result = []
        return json(result)

def get_model_columns(model, fields=None, meta='Table'):
    """
    Get fields info according model class, the fields just like ListView fields definition
    :param fields: A list
    :param meta: if no fields, it'll use meta
    """
    from copy import deepcopy

    fields = fields or []
    model = functions.get_model(model)

    if not fields:
        if hasattr(model, meta):
            fields = getattr(model, meta).fields
        else:
            fields = [x for x, y in model._fields_list]

    fields_list = []
    for x in fields:
        if isinstance(x, (str, unicode)):
            f = get_grid_column(model, x)
        elif isinstance(x, dict):
            name = x['name']
            f = deepcopy(x)
            if 'title' not in x:
                f.update(get_grid_column(model, name))
        else:
            raise ValueError("Field should be string or dict type, but {!r} found".format(x))
        fields_list.append(f)
    return fields_list

def get_grid_column(model, name):
    field = getattr(model, name, None)
    d = {'name':name, 'title':name}
    if field:
        d = {'name': name, 'title': field.verbose_name or field.name}
    return d
<commit_msg>Add download convert method support, so you can define `_convert_download_{fieldname}` for default convert function for download<commit_after>#coding=utf8
from uliweb import functions, url_for
import logging
from forms import QueryForm

log = logging.getLogger(__name__)
class Default(object):pass

class MultiView(object):
    """
    Multi View Class

    Support: list, add, delete, edit, detail, etc
    """

    def _process_fields_convert_map(self, parameters, download=False):
        """
        process fields_convert_map, ListView doesn't support list type but dict

        fields_convert_map should be define as list or dict
        for list, it can be:
            [name, name, ...]
            [(name, func), (name, func), ...] if func is str, it'll be the property name of class
        for dict, it can be:
            {'name':func, ...}
        :param model: model object
        :param parameters:
        :param prefix: it'll used to combine prefix+_convert_xxx to get convert function
            from class
        :return:
        """
        if 'fields_convert_map' in parameters:
            _f = parameters.get('fields_convert_map') or []
            parameters['fields_convert_map'] = self._get_fields_convert_map(_f, download)

    def _get_fields_convert_map(self, fields, download=False):
        """
        process fields_convert_map, ListView doesn't support list type but dict

        fields_convert_map should be define as list or dict
        for list, it can be:
            [name, name, ...]
            [(name, func), (name, func), ...] if func is str, it'll be the property name of class
        for dict, it can be:
            {'name':func, ...}
        :param model: model object
        :param parameters:
        :param prefix: it'll used to combine prefix+_convert_xxx to get convert function
            from class
        :return:
        """
        _f = fields
        t = {}

        # add support for download field
        def _get(name):
            _name = '_convert_download_{}'.format(name)
            if download and hasattr(self, _name):
                return getattr(self, _name)
            return getattr(self, '_convert_{}'.format(name))

        if isinstance(_f, list):
            for k in _f:
                if isinstance(k, str):
                    t[k] = _get(k)
                elif isinstance(k, (tuple, list)):
                    name = k[0]
                    func = k[1]
                    if isinstance(func, str):
                        t[name] = _get(func)
                    elif callable(func):
                        t[name] = func
                    else:
                        raise ValueError("Fields convert function should be str or callable, but %r found" % type(func))
                else:
                    raise ValueError("Fields convert element should be str or tuple or list, but %r found" % type(k))
        elif isinstance(_f, dict):
            for k, v in _f.items():
                if isinstance(v, str):
                    t[k] = _get(v)
                elif callable(v):
                    t[k] = v
                else:
                    raise ValueError("Fields convert function should be str or callable, but %r found" % type(func))
        return t

    def _list_view(self, model, **kwargs):
        """
        :param model:
        :param fields_convert_map: it's different from ListView
        :param kwargs:
        :return:
        """
        view =  functions.ListView(model, **kwargs)
        return view

    def _query_view(self, model, **kwargs):
        """
        :param model:
        :return: (query, condition)

        Default use QueryForm
        """
        QueryForm = functions.get_form('QueryForm')

        if 'form_cls' not in kwargs:
            kwargs['form_cls'] = QueryForm
        query = functions.QueryView(model, **kwargs)
        return query

    def _list(self, model, queryview=None, queryform=None, **kwargs):
        from uliweb import request, json, CONTENT_TYPE_JSON
        from sqlalchemy import and_
        from uliweb.utils.generic import get_sort_field
        import copy

        if queryview:
            queryview.run()
            condition = queryview.get_condition()
            post_query_condition = kwargs.get('post_query_condition')
            if post_query_condition:
                condition = post_query_condition(condition)
        else:
            condition = None

        if 'condition' in kwargs:
            condition = and_(condition, kwargs['condition'])

        post_condition = kwargs.get('post_condition')
        if post_condition:
            condition = post_condition(condition)
        kwargs['condition'] = condition

        #process order
        order_by = get_sort_field(model)
        if order_by is not None:
            kwargs['order_by'] = order_by

        _fields = copy.copy(kwargs.get('fields_convert_map', []))

        self._process_fields_convert_map(kwargs)
        downloads = {}
        downloads['filename'] = kwargs.pop('download_filename', 'download.xlsx')
        downloads['action'] = kwargs.pop('download_action', 'download')
        downloads['fields_convert_map'] = kwargs.pop('download_fields_convert_map',
                                                  _fields)
        downloads['domain'] = kwargs.pop('download_domain', '')
        downloads['timeout'] = 0
        downloads.update(kwargs.pop('download_kwargs', {}))
        self._process_fields_convert_map(downloads, download=True)

        #get list view
        view = self._list_view(model=model, **kwargs)

        if 'data' in request.values and request.is_xhr:
            return json(view.json(), content_type=CONTENT_TYPE_JSON)
        elif 'download' in request.GET:
            return view.download(**downloads)
        else:
            result = view.run()
            if queryview:
                result.update({'query_form':queryform or queryview.form})
            else:
                result.update({'query_form':''})
            result.update({'table':view})
            return result

    def _view(self, model, obj, **kwargs):
        self._process_fields_convert_map(kwargs)

        view = functions.DetailView(model, obj=obj, **kwargs)
        return view.run()

    def _add(self, model, json_result=False, **kwargs):
        self._process_fields_convert_map(kwargs)

        view = functions.AddView(model, **kwargs)
        return view.run(json_result=json_result)

    def _edit(self, model, obj, json_result=False, **kwargs):
        self._process_fields_convert_map(kwargs)

        view = functions.EditView(model, obj=obj, **kwargs)
        return view.run(json_result=json_result)

    def _delete(self, model, obj, json_result=False, **kwargs):
        view = functions.DeleteView(model, obj=obj, **kwargs)
        return view.run(json_result=json_result)

    def _select_list(self, queryview=None, download_filename=None, **kwargs):
        from uliweb import request, json

        if queryview:
            queryview.run()
            condition = queryview.get_condition()
        else:
            condition = None

        if 'condition' in kwargs:
            condition = and_(condition, kwargs['condition'])
            kwargs['condition'] = condition
        else:
            kwargs['condition'] = condition

        view = functions.SelectListView(**kwargs)
        if 'data' in request.values:
            return json(view.json())
        elif 'download' in request.GET:
            filename = download_filename or 'download.xls'
            kwargs.setdefault('action', 'download')
            kwargs.setdefault('timeout', 0)
            return view.download(filename, **kwargs)
        else:
            result = view.run()
            if queryview:
                result.update({'query_form':queryview.form})
            else:
                result.update({'query_form':''})
            result.update({'table':view})
            return result

    def _search(self, model, condition=None, search_field='name',
                value_field='id', label_field=None, pagination=True):
        """
        Default search function
        :param search_field: Used for search field, default is 'name'
        :param value_field: Used for id field, default is id
        :param label_field: Used for label field, default is None, then it'll use unicode() function
        """
        from uliweb import json, request

        name = request.GET.get('term', '')
        M = functions.get_model(model)

        def _v(label_field):
            if label_field:
                return lambda x: getattr(x, label_field)
            else:
                return lambda x: unicode(x)

        v_field = request.values.get('label', 'title')
        page = int(request.values.get('page') or 1)
        limit = int(request.values.get('limit') or 10)
        v_func = _v(label_field)
        if name:
            if condition is None:
                condition = M.c[search_field].like('%' + name + '%')
            if pagination:
                query = M.filter(condition)
                total = query.count()
                rows = [{'id': getattr(obj, value_field), v_field: v_func(obj)}
                            for obj in query.limit(limit).offset((page-1)*limit)]
                result = {'total':total, 'rows':rows}
            else:
                result = [{'id': getattr(obj, value_field), v_field: v_func(obj)}
                      for obj in M.filter(condition)]
        else:
            result = []
        return json(result)

def get_model_columns(model, fields=None, meta='Table'):
    """
    Get fields info according model class, the fields just like ListView fields definition
    :param fields: A list
    :param meta: if no fields, it'll use meta
    """
    from copy import deepcopy

    fields = fields or []
    model = functions.get_model(model)

    if not fields:
        if hasattr(model, meta):
            fields = getattr(model, meta).fields
        else:
            fields = [x for x, y in model._fields_list]

    fields_list = []
    for x in fields:
        if isinstance(x, (str, unicode)):
            f = get_grid_column(model, x)
        elif isinstance(x, dict):
            name = x['name']
            f = deepcopy(x)
            if 'title' not in x:
                f.update(get_grid_column(model, name))
        else:
            raise ValueError("Field should be string or dict type, but {!r} found".format(x))
        fields_list.append(f)
    return fields_list

def get_grid_column(model, name):
    field = getattr(model, name, None)
    d = {'name':name, 'title':name}
    if field:
        d = {'name': name, 'title': field.verbose_name or field.name}
    return d
<|endoftext|>


<commit_before>import textwrap

from .helpers import TestCase
from jedi import Script

class TestCallSignatures(TestCase):
    def _run(self, source, expected_name, expected_index=0, line=None, column=None):
        signatures = Script(source, line, column).call_signatures()

        assert len(signatures) <= 1

        if not signatures:
            assert expected_name is None
        else:
            assert signatures[0].call_name == expected_name
            assert signatures[0].index == expected_index

    def test_call_signatures(self):
        def run(source, name, index=0, column=None, line=1):
            self._run(source, name, index, line, column)

        # simple
        s1 = "abs(a, str("
        run(s1, 'abs', 0, 4)
        run(s1, 'abs', 1, 6)
        run(s1, 'abs', 1, 7)
        run(s1, 'abs', 1, 8)
        run(s1, 'str', 0, 11)

        s2 = "abs(), "
        run(s2, 'abs', 0, 4)
        run(s2, None, column=5)
        run(s2, None)

        s3 = "abs()."
        run(s3, None, column=5)
        run(s3, None)

        # more complicated
        s4 = 'abs(zip(), , set,'
        run(s4, None, column=3)
        run(s4, 'abs', 0, 4)
        run(s4, 'zip', 0, 8)
        run(s4, 'abs', 0, 9)
        #run(s4, 'abs', 1, 10)

        s5 = "abs(1,\nif 2:\n def a():"
        run(s5, 'abs', 0, 4)
        run(s5, 'abs', 1, 6)

        s6 = "str().center("
        run(s6, 'center', 0)
        run(s6, 'str', 0, 4)

        s7 = "str().upper().center("
        s8 = "str(int[zip("
        run(s7, 'center', 0)
        run(s8, 'zip', 0)
        run(s8, 'str', 0, 8)

        run("import time; abc = time; abc.sleep(", 'sleep', 0)

        # jedi-vim #9
        run("with open(", 'open', 0)

        # jedi-vim #11
        run("for sorted(", 'sorted', 0)
        run("for s in sorted(", 'sorted', 0)

        # jedi #57
        s = "def func(alpha, beta): pass\n" \
            "func(alpha='101',"
        run(s, 'func', 0, column=13, line=2)

    def test_function_definition_complex(self):
        s = """
                def abc(a,b):
                    pass

                def a(self):
                    abc(

                if 1:
                    pass
            """
        self._run(s, 'abc', 0, line=6, column=24)
        s = """
                import re
                def huhu(it):
                    re.compile(
                    return it * 2
            """
        self._run(s, 'compile', 0, line=4, column=31)

        # jedi-vim #70
        s = """def foo("""
        assert Script(s).call_signatures() == []

        # jedi-vim #116
        s = """import functools; test = getattr(functools, 'partial'); test("""
        self._run(s, 'partial', 0)

    def test_call_signature_on_module(self):
        """github issue #240"""
        s = 'import datetime; datetime('
        # just don't throw an exception (if numpy doesn't exist, just ignore it)
        assert Script(s).call_signatures() == []

    def test_function_definition_empty_paren_pre_space(self):
        s = textwrap.dedent("""\
        def f(a, b):
            pass
        f( )""")
        self._run(s, 'f', 0, line=3, column=3)
<commit_msg>test for multiple call_signatures<commit_after>import textwrap

from .helpers import TestCase
from jedi import Script

class TestCallSignatures(TestCase):
    def _run(self, source, expected_name, expected_index=0, line=None, column=None):
        signatures = Script(source, line, column).call_signatures()

        assert len(signatures) <= 1

        if not signatures:
            assert expected_name is None
        else:
            assert signatures[0].call_name == expected_name
            assert signatures[0].index == expected_index

    def test_call_signatures(self):
        def run(source, name, index=0, column=None, line=1):
            self._run(source, name, index, line, column)

        # simple
        s1 = "abs(a, str("
        run(s1, 'abs', 0, 4)
        run(s1, 'abs', 1, 6)
        run(s1, 'abs', 1, 7)
        run(s1, 'abs', 1, 8)
        run(s1, 'str', 0, 11)

        s2 = "abs(), "
        run(s2, 'abs', 0, 4)
        run(s2, None, column=5)
        run(s2, None)

        s3 = "abs()."
        run(s3, None, column=5)
        run(s3, None)

        # more complicated
        s4 = 'abs(zip(), , set,'
        run(s4, None, column=3)
        run(s4, 'abs', 0, 4)
        run(s4, 'zip', 0, 8)
        run(s4, 'abs', 0, 9)
        #run(s4, 'abs', 1, 10)

        s5 = "abs(1,\nif 2:\n def a():"
        run(s5, 'abs', 0, 4)
        run(s5, 'abs', 1, 6)

        s6 = "str().center("
        run(s6, 'center', 0)
        run(s6, 'str', 0, 4)

        s7 = "str().upper().center("
        s8 = "str(int[zip("
        run(s7, 'center', 0)
        run(s8, 'zip', 0)
        run(s8, 'str', 0, 8)

        run("import time; abc = time; abc.sleep(", 'sleep', 0)

        # jedi-vim #9
        run("with open(", 'open', 0)

        # jedi-vim #11
        run("for sorted(", 'sorted', 0)
        run("for s in sorted(", 'sorted', 0)

        # jedi #57
        s = "def func(alpha, beta): pass\n" \
            "func(alpha='101',"
        run(s, 'func', 0, column=13, line=2)

    def test_function_definition_complex(self):
        s = """
                def abc(a,b):
                    pass

                def a(self):
                    abc(

                if 1:
                    pass
            """
        self._run(s, 'abc', 0, line=6, column=24)
        s = """
                import re
                def huhu(it):
                    re.compile(
                    return it * 2
            """
        self._run(s, 'compile', 0, line=4, column=31)

        # jedi-vim #70
        s = """def foo("""
        assert Script(s).call_signatures() == []

        # jedi-vim #116
        s = """import functools; test = getattr(functools, 'partial'); test("""
        self._run(s, 'partial', 0)

    def test_call_signature_on_module(self):
        """github issue #240"""
        s = 'import datetime; datetime('
        # just don't throw an exception (if numpy doesn't exist, just ignore it)
        assert Script(s).call_signatures() == []

    def test_function_definition_empty_paren_pre_space(self):
        s = textwrap.dedent("""\
        def f(a, b):
            pass
        f( )""")
        self._run(s, 'f', 0, line=3, column=3)

    def test_multiple_signatures(self):
        s = textwrap.dedent("""\
        if x:
            def f(a, b):
                pass
        else:
            def f(a, b):
                pass
        f(""")
        assert len(Script(s).call_signatures()) == 2
<|endoftext|>