<commit_before># This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Set of common functions for all handlers."""

import types

from bson import tz_util
from datetime import (
    date,
    datetime,
    time,
    timedelta,
)
from pymongo import (
    ASCENDING,
    DESCENDING,
)

from models import (
    AGGREGATE_KEY,
    CREATED_KEY,
    DATE_RANGE_KEY,
    FIELD_KEY,
    LIMIT_KEY,
    NOT_FIELD_KEY,
    SKIP_KEY,
    SORT_KEY,
    SORT_ORDER_KEY,
)
from utils import get_log

# Default value to calculate a date range in case the provided value is
# out of range.
DEFAULT_DATE_RANGE = 15

LOG = get_log()

def get_all_query_values(query_args_func, valid_keys):
    """Handy function to get all query args in a batch.

    :param query_args_func: A function used to return a list of the query
    arguments.
    :type query_args_func: function
    :param valid_keys: A list containing the valid keys that should be
    retrieved.
    :type valid_keys: list
    """
    spec = get_query_spec(query_args_func, valid_keys)
    sort = get_query_sort(query_args_func)
    fields = get_query_fields(query_args_func)
    skip, limit = get_skip_and_limit(query_args_func)
    unique = get_aggregate_value(query_args_func)

    return (spec, sort, fields, skip, limit, unique)

def get_aggregate_value(query_args_func):
    """Get teh value of the aggregate key.

    :param query_args_func: A function used to return a list of the query
    arguments.
    :type query_args_func: function
    :return The aggregate value as string.
    """
    aggregate = query_args_func(AGGREGATE_KEY)
    if all([aggregate and isinstance(aggregate, types.ListType)]):
        aggregate = aggregate[-1]
    else:
        aggregate = None
    return aggregate

def get_query_spec(query_args_func, valid_keys):
    """Get values from the query string to build a `spec` data structure.

    A `spec` data structure is a dictionary whose keys are the keys
    accepted by this handler method.

    :param query_args_func: A function used to return a list of the query
    arguments.
    :type query_args_func: function
    :param valid_keys: A list containing the valid keys that should be
    retrieved.
    :type valid_keys: list
    :return A `spec` data structure (dictionary).
    """
    def _get_spec_values():
        """Get the values for the spec data structure.

        Internally used only, with some logic to differentiate between single
        and multiple values. Makes sure also that the list of values if valid,
        meaning that we do not have None or empty values.

        :return A tuple with the key and its value.
        """
        for key in valid_keys:
            val = query_args_func(key) or []
            if val:
                # Go through the values and make sure we have valid ones.
                val = [v for v in val if v]
                len_val = len(val)

                if len_val == 1:
                    val = val[0]
                elif len_val > 1:
                    # More than one value, make sure we look for all of them.
                    val = {'$in': val}

            yield key, val

    spec = {}
    if all([valid_keys and isinstance(valid_keys, types.ListType)]):
        spec = {
            k: v for k, v in [
                (key, val)
                for key, val in _get_spec_values()
                if val
            ]
        }

    return spec

def get_and_add_date_range(spec, query_args_func):
    """Retrieve the `date_range` query from the request.

    Add the retrieved `date_range` value into the provided `spec` data
    structure.

    :param spec: The dictionary where to store the key-value.
    :type spec: dictionary
    :param query_args_func: A function used to return a list of query
    arguments.
    :type query_args_func: function
    :return The passed `spec` updated.
    """
    date_range = query_args_func(DATE_RANGE_KEY)
    if date_range:
        # Today needs to be set at the end of the day!
        today = datetime.combine(
            date.today(), time(23, 59, 59, tzinfo=tz_util.utc)
        )
        previous = calculate_date_range(date_range)

        spec[CREATED_KEY] = {'$gte': previous, '$lt': today}
    return spec

def calculate_date_range(date_range):
    """Calculate the new date subtracting the passed number of days.

    It removes the passed days from today date, calculated at midnight
    UTC.

    :param date_range: The number of days to remove from today.
    :type date_range int, long, str
    :return A new `datetime.date` object that is the result of the
    subtraction of `datetime.date.today()` and
    `datetime.timedelta(days=date_range)`.
    """
    if isinstance(date_range, types.ListType):
        date_range = date_range[-1]

    if isinstance(date_range, types.StringTypes):
        date_range = int(date_range)

    date_range = abs(date_range)
    if date_range > timedelta.max.days:
        date_range = DEFAULT_DATE_RANGE

    # Calcuate with midnight in mind though, so we get the starting of
    # the day for the previous date.
    today = datetime.combine(
        date.today(), time(tzinfo=tz_util.utc)
    )
    delta = timedelta(days=date_range)

    return today - delta

def get_query_fields(query_args_func):
    """Get values from the query string to build a `fields` data structure.

    A `field` data structure can be either a list or a dictionary.

    :param query_args_func: A function used to return a list of query
    arguments.
    :type query_args_func: function
    :return A `fields` data structure (list or dictionary).
    """
    fields = None
    y_fields, n_fields = map(query_args_func, [FIELD_KEY, NOT_FIELD_KEY])

    if y_fields and not n_fields:
        fields = list(set(y_fields))
    elif n_fields:
        fields = dict.fromkeys(list(set(y_fields)), True)
        fields.update(dict.fromkeys(list(set(n_fields)), False))

    return fields

def get_query_sort(query_args_func):
    """Get values from the query string to build a `sort` data structure.

    A `sort` data structure is a list of tuples in a `key-value` fashion.
    The keys are the values passed as the `sort` argument on the query,
    they values are based on the `sort_order` argument and defaults to the
    descending order.

    :param query_args_func: A function used to return a list of query
    arguments.
    :type query_args_func: function
    :return A `sort` data structure, or None.
    """
    sort = None
    sort_fields, sort_order = map(query_args_func, [SORT_KEY, SORT_ORDER_KEY])

    if sort_fields:
        if all([sort_order, isinstance(sort_order, types.ListType)]):
            sort_order = int(sort_order[-1])
        else:
            sort_order = DESCENDING

        # Wrong number for sort order? Force descending.
        if all([sort_order != ASCENDING, sort_order != DESCENDING]):
            LOG.warn(
                "Wrong sort order used (%d), default to %d",
                sort_order, DESCENDING
            )
            sort_order = DESCENDING

        sort = [
            (field, sort_order)
            for field in sort_fields
        ]

    return sort

def get_skip_and_limit(query_args_func):
    """Retrieve the `skip` and `limit` query arguments.

    :param query_args_func: A function used to return a list of query
    arguments.
    :type query_args_func: function
    :return A tuple with the `skip` and `limit` arguments.
    """
    skip, limit = map(query_args_func, [SKIP_KEY, LIMIT_KEY])

    if all([skip, isinstance(skip, types.ListType)]):
        skip = int(skip[-1])
    else:
        skip = 0

    if all([limit, isinstance(limit, types.ListType)]):
        limit = int(limit[-1])
    else:
        limit = 0

    return skip, limit
<commit_msg>common: Add valid keys definition.<commit_after># This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Set of common functions for all handlers."""

import types

from bson import tz_util
from datetime import (
    date,
    datetime,
    time,
    timedelta,
)
from pymongo import (
    ASCENDING,
    DESCENDING,
)

from models import (
    AGGREGATE_KEY,
    ARCHITECTURE_KEY,
    BOARD_KEY,
    CREATED_KEY,
    DATE_RANGE_KEY,
    DEFCONFIG_KEY,
    ERRORS_KEY,
    FIELD_KEY,
    JOB_ID_KEY,
    JOB_KEY,
    KERNEL_KEY,
    LIMIT_KEY,
    NOT_FIELD_KEY,
    PRIVATE_KEY,
    SKIP_KEY,
    SORT_KEY,
    SORT_ORDER_KEY,
    STATUS_KEY,
    TIME_KEY,
    WARNINGS_KEY,
    BOOT_COLLECTION,
    DEFCONFIG_COLLECTION,
    JOB_COLLECTION,
    ADMIN_KEY,
    DELETE_KEY,
    EMAIL_KEY,
    EXPIRES_KEY,
    GET_KEY,
    IP_ADDRESS_KEY,
    IP_RESTRICTED,
    POST_KEY,
    SUPERUSER_KEY,
    USERNAME_KEY,
    PROPERTIES_KEY,
    TOKEN_KEY,
    EXPIRED_KEY,
)
from utils import get_log

# Default value to calculate a date range in case the provided value is
# out of range.
DEFAULT_DATE_RANGE = 15

LOG = get_log()

# All the available collections as key-value. The key is the same used for the
# URL configuration.
COLLECTIONS = {
    'boot': BOOT_COLLECTION,
    'defconfig': DEFCONFIG_COLLECTION,
    'job': JOB_COLLECTION,
}

# Handlers valid keys.
BOOT_VALID_KEYS = {
    'POST': [JOB_KEY, KERNEL_KEY],
    'GET': [
        CREATED_KEY, WARNINGS_KEY, JOB_ID_KEY, BOARD_KEY,
        JOB_KEY, KERNEL_KEY, DEFCONFIG_KEY, TIME_KEY, STATUS_KEY,
    ],
    'DELETE': [
        JOB_KEY, KERNEL_KEY, DEFCONFIG_KEY, BOARD_KEY, JOB_ID_KEY
    ]
}

COUNT_VALID_KEYS = {
    'GET': [
        ARCHITECTURE_KEY,
        BOARD_KEY,
        CREATED_KEY,
        DEFCONFIG_KEY,
        ERRORS_KEY,
        JOB_ID_KEY,
        JOB_KEY,
        KERNEL_KEY,
        PRIVATE_KEY,
        STATUS_KEY,
        TIME_KEY,
        WARNINGS_KEY,
    ],
}

DEFCONFIG_VALID_KEYS = {
    'GET': [
        DEFCONFIG_KEY, WARNINGS_KEY, ERRORS_KEY, ARCHITECTURE_KEY,
        JOB_KEY, KERNEL_KEY, STATUS_KEY, JOB_ID_KEY, CREATED_KEY,
    ],
}

TOKEN_VALID_KEYS = {
    'POST': [
        ADMIN_KEY,
        DELETE_KEY,
        EMAIL_KEY,
        EXPIRES_KEY,
        GET_KEY,
        IP_ADDRESS_KEY,
        IP_RESTRICTED,
        POST_KEY,
        SUPERUSER_KEY,
        USERNAME_KEY,
    ],
    'GET': [
        CREATED_KEY,
        EMAIL_KEY,
        EXPIRED_KEY,
        EXPIRES_KEY,
        IP_ADDRESS_KEY,
        PROPERTIES_KEY,
        TOKEN_KEY,
        USERNAME_KEY,
    ],
}

SUBSCRIPTION_VALID_KEYS = {
    'GET': [JOB_KEY],
    'POST': [JOB_KEY, EMAIL_KEY],
    'DELETE': [EMAIL_KEY],
}

JOB_VALID_KEYS = {
    'POST': [JOB_KEY, KERNEL_KEY],
    'GET': [
        JOB_KEY, KERNEL_KEY, STATUS_KEY, PRIVATE_KEY, CREATED_KEY,
    ],
}

def get_all_query_values(query_args_func, valid_keys):
    """Handy function to get all query args in a batch.

    :param query_args_func: A function used to return a list of the query
    arguments.
    :type query_args_func: function
    :param valid_keys: A list containing the valid keys that should be
    retrieved.
    :type valid_keys: list
    """
    spec = get_query_spec(query_args_func, valid_keys)
    spec = get_and_add_date_range(spec, query_args_func)

    sort = get_query_sort(query_args_func)
    fields = get_query_fields(query_args_func)
    skip, limit = get_skip_and_limit(query_args_func)
    unique = get_aggregate_value(query_args_func)

    return (spec, sort, fields, skip, limit, unique)

def get_aggregate_value(query_args_func):
    """Get teh value of the aggregate key.

    :param query_args_func: A function used to return a list of the query
    arguments.
    :type query_args_func: function
    :return The aggregate value as string.
    """
    aggregate = query_args_func(AGGREGATE_KEY)
    if all([aggregate and isinstance(aggregate, types.ListType)]):
        aggregate = aggregate[-1]
    else:
        aggregate = None
    return aggregate

def get_query_spec(query_args_func, valid_keys):
    """Get values from the query string to build a `spec` data structure.

    A `spec` data structure is a dictionary whose keys are the keys
    accepted by this handler method.

    :param query_args_func: A function used to return a list of the query
    arguments.
    :type query_args_func: function
    :param valid_keys: A list containing the valid keys that should be
    retrieved.
    :type valid_keys: list
    :return A `spec` data structure (dictionary).
    """
    def _get_spec_values():
        """Get the values for the spec data structure.

        Internally used only, with some logic to differentiate between single
        and multiple values. Makes sure also that the list of values if valid,
        meaning that we do not have None or empty values.

        :return A tuple with the key and its value.
        """
        for key in valid_keys:
            val = query_args_func(key) or []
            if val:
                # Go through the values and make sure we have valid ones.
                val = [v for v in val if v]
                len_val = len(val)

                if len_val == 1:
                    val = val[0]
                elif len_val > 1:
                    # More than one value, make sure we look for all of them.
                    val = {'$in': val}

            yield key, val

    spec = {}
    if all([valid_keys and isinstance(valid_keys, types.ListType)]):
        spec = {
            k: v for k, v in [
                (key, val)
                for key, val in _get_spec_values()
                if val
            ]
        }

    return spec

def get_and_add_date_range(spec, query_args_func):
    """Retrieve the `date_range` query from the request.

    Add the retrieved `date_range` value into the provided `spec` data
    structure.

    :param spec: The dictionary where to store the key-value.
    :type spec: dictionary
    :param query_args_func: A function used to return a list of query
    arguments.
    :type query_args_func: function
    :return The passed `spec` updated.
    """
    date_range = query_args_func(DATE_RANGE_KEY)
    if date_range:
        # Today needs to be set at the end of the day!
        today = datetime.combine(
            date.today(), time(23, 59, 59, tzinfo=tz_util.utc)
        )
        previous = calculate_date_range(date_range)

        spec[CREATED_KEY] = {'$gte': previous, '$lt': today}
    return spec

def calculate_date_range(date_range):
    """Calculate the new date subtracting the passed number of days.

    It removes the passed days from today date, calculated at midnight
    UTC.

    :param date_range: The number of days to remove from today.
    :type date_range int, long, str
    :return A new `datetime.date` object that is the result of the
    subtraction of `datetime.date.today()` and
    `datetime.timedelta(days=date_range)`.
    """
    if isinstance(date_range, types.ListType):
        date_range = date_range[-1]

    if isinstance(date_range, types.StringTypes):
        date_range = int(date_range)

    date_range = abs(date_range)
    if date_range > timedelta.max.days:
        date_range = DEFAULT_DATE_RANGE

    # Calcuate with midnight in mind though, so we get the starting of
    # the day for the previous date.
    today = datetime.combine(
        date.today(), time(tzinfo=tz_util.utc)
    )
    delta = timedelta(days=date_range)

    return today - delta

def get_query_fields(query_args_func):
    """Get values from the query string to build a `fields` data structure.

    A `field` data structure can be either a list or a dictionary.

    :param query_args_func: A function used to return a list of query
    arguments.
    :type query_args_func: function
    :return A `fields` data structure (list or dictionary).
    """
    fields = None
    y_fields, n_fields = map(query_args_func, [FIELD_KEY, NOT_FIELD_KEY])

    if y_fields and not n_fields:
        fields = list(set(y_fields))
    elif n_fields:
        fields = dict.fromkeys(list(set(y_fields)), True)
        fields.update(dict.fromkeys(list(set(n_fields)), False))

    return fields

def get_query_sort(query_args_func):
    """Get values from the query string to build a `sort` data structure.

    A `sort` data structure is a list of tuples in a `key-value` fashion.
    The keys are the values passed as the `sort` argument on the query,
    they values are based on the `sort_order` argument and defaults to the
    descending order.

    :param query_args_func: A function used to return a list of query
    arguments.
    :type query_args_func: function
    :return A `sort` data structure, or None.
    """
    sort = None
    sort_fields, sort_order = map(query_args_func, [SORT_KEY, SORT_ORDER_KEY])

    if sort_fields:
        if all([sort_order, isinstance(sort_order, types.ListType)]):
            sort_order = int(sort_order[-1])
        else:
            sort_order = DESCENDING

        # Wrong number for sort order? Force descending.
        if all([sort_order != ASCENDING, sort_order != DESCENDING]):
            LOG.warn(
                "Wrong sort order used (%d), default to %d",
                sort_order, DESCENDING
            )
            sort_order = DESCENDING

        sort = [
            (field, sort_order)
            for field in sort_fields
        ]

    return sort

def get_skip_and_limit(query_args_func):
    """Retrieve the `skip` and `limit` query arguments.

    :param query_args_func: A function used to return a list of query
    arguments.
    :type query_args_func: function
    :return A tuple with the `skip` and `limit` arguments.
    """
    skip, limit = map(query_args_func, [SKIP_KEY, LIMIT_KEY])

    if all([skip, isinstance(skip, types.ListType)]):
        skip = int(skip[-1])
    else:
        skip = 0

    if all([limit, isinstance(limit, types.ListType)]):
        limit = int(limit[-1])
    else:
        limit = 0

    return skip, limit
<|endoftext|>


<commit_before>import psycopg2
import psycopg2.extras

# Database user credentials
DATABASE = "seads"
USER	 = "seadapi"

def query(parsed_url):
	"""
	Handle parsed URL data and query the database as appropriate

	:param parsed_url: Array of url parameters
	:return: Generator of result strings
	"""
	if 'device_id' not in parsed_url.keys():
		raise Exception("Relieved malformed URL data")

	header = ['time', 'I', 'W', 'V', 'T']
	start_time = end_time = data_type = subset = None
	if 'type' in parsed_url.keys():
		data_type = parsed_url['type']
		header = ['time', parsed_url['type']]
	if 'start_time' in parsed_url.keys():
		start_time = parsed_url['start_time']
	if 'end_time' in parsed_url.keys():
		end_time = parsed_url['end_time']
	if 'end_time' in parsed_url.keys():
		subset = parsed_url['subset']

	if start_time or end_time or data_type:
		results = retrieve_within_filters(
			parsed_url['device_id'],
			start_time,
			end_time,
			data_type,
			subset,
		)
	else:
		results = retrieve_historical(parsed_url['device_id'])

	return format_data(header, results)

def retrieve_within_filters(device_id, start_time, end_time, data_type, subset):
	"""
	Return sensor data for a device within a specified timeframe

	:param device_id: The serial number of the device in question
	:param start_time: The start of the time range for which to query for data
	:param end_time: The end of the time range for which to query for data
	:param data_type: The type of data to query for
	:param subset: The size of the subset
	:return: Generator of database row tuples
	"""
	params = [device_id]
	where = None

	if subset:
		data = '''
(SELECT * FROM (
	SELECT *, ((row_number() OVER (ORDER BY "time"))
		% ceil(count(*) OVER () / %s)::int) AS rn
	FROM   data_raw
	) sub
WHERE sub.rn = 0)'''
		params.insert(0, subset)

	else:
		data = 'data_raw'

	if start_time and end_time:
		where = "WHERE serial = %s AND time BETWEEN to_timestamp(%s) AND to_timestamp(%s)"
		params.append(start_time)
		params.append(end_time)
	elif start_time:
		where = "WHERE serial = %s AND time >= to_timestamp(%s)"
		params.append(start_time)
	elif end_time:
		where = "WHERE serial = %s AND time <= to_timestamp(%s)"
		params.append(end_time)
	if data_type:
		if where:
			where += " AND type = %s"
		else:
			where = "WHERE serial = %s AND type = %s"
		params.append(data_type)
		query = "SELECT time, data FROM " + data + " as raw " + where
	else:
		query = write_crosstab(where, data)

	rows = perform_query(query, tuple(params))
	return rows

def retrieve_historical(device_id):
	"""
	Return sensor data for a specific device
	TODO: add a page size limit?

	:param device_id: The serial number of the device in question
	:return: Generator of database row tuples
	"""
	query = write_crosstab("WHERE serial = %s")
	params = (device_id, )
	rows = perform_query(query, params)
	return rows

def write_crosstab(where, data='data_raw'):
	"""
	Write a PostgreSQL crosstab() query to create a pivot table and rearrange the data into a more useful form

	:param where: WHERE clause for SQL query
	:param data: Table or subquery from which to get the data
	:return: Complete SQL query
	"""
	query = "SELECT * FROM crosstab(" +\
				"'SELECT time, type, data from " + data + " as raw " + where + "'," +\
				" 'SELECT unnest(ARRAY[''I'', ''W'', ''V'', ''T''])') " + \
			"AS ct_result(time TIMESTAMP, I SMALLINT, W SMALLINT, V SMALLINT, T SMALLINT);"
	return query

def perform_query(query, params):
	"""
	Initiate a connection to the database and return a cursor to return db rows a dictionaries

	:param query: SQL query string
	:param params: List of SQL query parameters
	:return: Result cursor
	"""
	con = None
	try:
		con = psycopg2.connect("dbname='" + DATABASE +
				"' user='" + USER + "'")
		cursor = con.cursor()
		cursor.execute(query, params)
		return cursor.fetchall()

	except psycopg2.DatabaseError as e:
		print('Database error: %s' % e)

	finally:
		if con:
			con.close()

def format_data(header, data):
	"""
	Process rows of data returned by the db and format them appropriately

	:param header: The first row of the result
	:param data: Result cursor
	:return: Generator of result strings
	"""
	data.insert(0, header)
	return map(lambda x: str(list(map(str, x))) + '\n', data)
<commit_msg>fixed similar typo<commit_after>import psycopg2
import psycopg2.extras

# Database user credentials
DATABASE = "seads"
USER	 = "seadapi"

def query(parsed_url):
	"""
	Handle parsed URL data and query the database as appropriate

	:param parsed_url: Array of url parameters
	:return: Generator of result strings
	"""
	if 'device_id' not in parsed_url.keys():
		raise Exception("Relieved malformed URL data")

	header = ['time', 'I', 'W', 'V', 'T']
	start_time = end_time = data_type = subset = None
	if 'type' in parsed_url.keys():
		data_type = parsed_url['type']
		header = ['time', parsed_url['type']]
	if 'start_time' in parsed_url.keys():
		start_time = parsed_url['start_time']
	if 'end_time' in parsed_url.keys():
		end_time = parsed_url['end_time']
	if 'subset' in parsed_url.keys():
		subset = parsed_url['subset']

	if start_time or end_time or data_type:
		results = retrieve_within_filters(
			parsed_url['device_id'],
			start_time,
			end_time,
			data_type,
			subset,
		)
	else:
		results = retrieve_historical(parsed_url['device_id'])

	return format_data(header, results)

def retrieve_within_filters(device_id, start_time, end_time, data_type, subset):
	"""
	Return sensor data for a device within a specified timeframe

	:param device_id: The serial number of the device in question
	:param start_time: The start of the time range for which to query for data
	:param end_time: The end of the time range for which to query for data
	:param data_type: The type of data to query for
	:param subset: The size of the subset
	:return: Generator of database row tuples
	"""
	params = [device_id]
	where = None

	if subset:
		data = '''
(SELECT * FROM (
	SELECT *, ((row_number() OVER (ORDER BY "time"))
		% ceil(count(*) OVER () / %s)::int) AS rn
	FROM   data_raw
	) sub
WHERE sub.rn = 0)'''
		params.insert(0, subset)

	else:
		data = 'data_raw'

	if start_time and end_time:
		where = "WHERE serial = %s AND time BETWEEN to_timestamp(%s) AND to_timestamp(%s)"
		params.append(start_time)
		params.append(end_time)
	elif start_time:
		where = "WHERE serial = %s AND time >= to_timestamp(%s)"
		params.append(start_time)
	elif end_time:
		where = "WHERE serial = %s AND time <= to_timestamp(%s)"
		params.append(end_time)
	if data_type:
		if where:
			where += " AND type = %s"
		else:
			where = "WHERE serial = %s AND type = %s"
		params.append(data_type)
		query = "SELECT time, data FROM " + data + " as raw " + where
	else:
		query = write_crosstab(where, data)

	rows = perform_query(query, tuple(params))
	return rows

def retrieve_historical(device_id):
	"""
	Return sensor data for a specific device
	TODO: add a page size limit?

	:param device_id: The serial number of the device in question
	:return: Generator of database row tuples
	"""
	query = write_crosstab("WHERE serial = %s")
	params = (device_id, )
	rows = perform_query(query, params)
	return rows

def write_crosstab(where, data='data_raw'):
	"""
	Write a PostgreSQL crosstab() query to create a pivot table and rearrange the data into a more useful form

	:param where: WHERE clause for SQL query
	:param data: Table or subquery from which to get the data
	:return: Complete SQL query
	"""
	query = "SELECT * FROM crosstab(" +\
				"'SELECT time, type, data from " + data + " as raw " + where + "'," +\
				" 'SELECT unnest(ARRAY[''I'', ''W'', ''V'', ''T''])') " + \
			"AS ct_result(time TIMESTAMP, I SMALLINT, W SMALLINT, V SMALLINT, T SMALLINT);"
	return query

def perform_query(query, params):
	"""
	Initiate a connection to the database and return a cursor to return db rows a dictionaries

	:param query: SQL query string
	:param params: List of SQL query parameters
	:return: Result cursor
	"""
	con = None
	try:
		con = psycopg2.connect("dbname='" + DATABASE +
				"' user='" + USER + "'")
		cursor = con.cursor()
		cursor.execute(query, params)
		return cursor.fetchall()

	except psycopg2.DatabaseError as e:
		print('Database error: %s' % e)

	finally:
		if con:
			con.close()

def format_data(header, data):
	"""
	Process rows of data returned by the db and format them appropriately

	:param header: The first row of the result
	:param data: Result cursor
	:return: Generator of result strings
	"""
	data.insert(0, header)
	return map(lambda x: str(list(map(str, x))) + '\n', data)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2013-Today  Carlos Eduardo Vercelino - CLVsol
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    'name': 'Family (Off)',
    'summary': 'Family (Off) Module used by CLVsol Solutions.',
    'version': '4.0.0',
    'author': 'Carlos Eduardo Vercelino - CLVsol',
    'category': 'Generic Modules/Others',
    'license': 'AGPL-3',
    'website': 'https://github.com/CLVsol',
    'depends': [
        'clv_base',
        'clv_global_log',
        'clv_partner_entity',
        'clv_global_tag',
        'clv_address',
    ],
    'data': [
        'security/family_off_security.xml',
        'security/ir.model.access.csv',
        'views/family_off_view.xml',
        'views/family_off_log_view.xml',
        'views/family_category_view.xml',
        'views/family_marker_view.xml',
        'views/res_partner_view.xml',
        'views/global_tag_view.xml',
        'views/family_off_name_view.xml',
        'views/family_view.xml',
        'views/address_view.xml',
        'views/address_off_view.xml',
    ],
    'demo': [],
    'test': [],
    'init_xml': [],
    'test': [],
    'update_xml': [],
    'installable': True,
    'application': False,
    'active': False,
    'css': [],
}
<commit_msg>[IMP] clv_family_off (IMP __manifest__.py)<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2013-Today  Carlos Eduardo Vercelino - CLVsol
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    'name': 'Family (Off)',
    'summary': 'Family (Off) Module used by CLVsol Solutions.',
    'version': '4.0.0',
    'author': 'Carlos Eduardo Vercelino - CLVsol',
    'category': 'Generic Modules/Others',
    'license': 'AGPL-3',
    'website': 'https://github.com/CLVsol',
    'depends': [
        'clv_base',
        'clv_global_log',
        'clv_partner_entity',
        'clv_global_tag',
        'clv_address',
        'clv_address_off',
        'clv_family',
    ],
    'data': [
        'security/family_off_security.xml',
        'security/ir.model.access.csv',
        'views/family_off_view.xml',
        'views/family_off_log_view.xml',
        'views/family_category_view.xml',
        'views/family_marker_view.xml',
        'views/res_partner_view.xml',
        'views/global_tag_view.xml',
        'views/family_off_name_view.xml',
        'views/family_view.xml',
        'views/address_view.xml',
        'views/address_off_view.xml',
    ],
    'demo': [],
    'test': [],
    'init_xml': [],
    'test': [],
    'update_xml': [],
    'installable': True,
    'application': False,
    'active': False,
    'css': [],
}
<|endoftext|>


<commit_before>from osim.env import L2M2019CtrlEnv
from osim.control.osim_loco_reflex_song2019 import OsimReflexCtrl
from joblib import Parallel, delayed

import sys
import numpy as np

trial_name = 'trial_190505_L2M2019CtrlEnv_d0_'

params = np.ones(45)
#params = np.loadtxt('./optim_data/cma/trial_181029_walk_3D_noStand_8_best.txt')
N_POP = 16 # 8 = 4 + floor(3*log(45))
N_PROC = 2
TIMEOUT = 10*60

def f_ind(n_gen, i_worker, params):
    flag_model = '3D'
    flag_ctrl_mode = '3D' # use 2D
    seed = None
    difficulty = 0
    sim_dt = 0.01
    sim_t = 20
    timstep_limit = int(round(sim_t/sim_dt))

    try:
        locoCtrl = OsimReflexCtrl(mode=flag_ctrl_mode, dt=sim_dt)
        env = L2M2019CtrlEnv(locoCtrl=locoCtrl, seed=seed, difficulty=difficulty, visualize=False)
        env.change_model(model=flag_model, difficulty=difficulty, seed=seed)
        observation = env.reset(project=True, seed=seed)
    except Exception as e_msg:
        print("simulation error!!!")
        print(e_msg)
        #import pdb; pdb.set_trace()
        return 0
    env.spec.timestep_limit = timstep_limit+100

    total_reward = 0
    error_sim = 0;
    t = 0
    for i in range(timstep_limit+100):
        t += sim_dt

        observation, reward, done, info = env.step(params, project=True)
        total_reward += reward

        if done:
            break

    print('    par#={} gen#={}: score={} time={}sec #step={}'.format(i_worker, n_gen, total_reward, t, env.footstep['n']))

    return total_reward  # minimization

class CMATrainPar(object):
    def __init__(self, ):
        self.n_gen = 0
        self.best_total_reward = -np.inf

    def f(self, v_params):
        self.n_gen += 1
        v_total_reward = Parallel(n_jobs=N_PROC, timeout=TIMEOUT)\
        (delayed(f_ind)(self.n_gen, i, p) for i, p in enumerate(v_params))

        for total_reward in v_total_reward:
            if self.best_total_reward  < total_reward:
                filename = "./optim_data/cma/" + trial_name + "best_w.txt"
                print("")
                print("----")
                print("update the best score!!!!")
                print("\tprev = %.8f" % self.best_total_reward )
                print("\tcurr = %.8f" % total_reward)
                print("\tsave to [%s]" % filename)
                print("----")
                print("")
                self.best_total_reward  = total_reward
                np.savetxt(filename, params)

        return [-r for r in v_total_reward]

if __name__ == '__main__':
    prob = CMATrainPar()

    from cmaes.solver_cma import CMASolverPar
    solver = CMASolverPar(prob)

    solver.options.set("popsize", N_POP)
    solver.options.set("maxiter", 400)
    solver.options.set("verb_filenameprefix", 'optim_data/cma/' + trial_name)
    solver.set_verbose(True)

    x0 = params
    sigma = .01

    res = solver.solve(x0, sigma)
    print(res)
<commit_msg>[cmaes] print fix<commit_after>from osim.env import L2M2019CtrlEnv
from osim.control.osim_loco_reflex_song2019 import OsimReflexCtrl
from joblib import Parallel, delayed

import sys
import numpy as np

trial_name = 'trial_190505_L2M2019CtrlEnv_d0_'

params = np.ones(45)
#params = np.loadtxt('./optim_data/cma/trial_181029_walk_3D_noStand_8_best.txt')
N_POP = 16 # 8 = 4 + floor(3*log(45))
N_PROC = 2
TIMEOUT = 10*60

def f_ind(n_gen, i_worker, params):
    flag_model = '3D'
    flag_ctrl_mode = '3D' # use 2D
    seed = None
    difficulty = 0
    sim_dt = 0.01
    sim_t = 20
    timstep_limit = int(round(sim_t/sim_dt))

    try:
        locoCtrl = OsimReflexCtrl(mode=flag_ctrl_mode, dt=sim_dt)
        env = L2M2019CtrlEnv(locoCtrl=locoCtrl, seed=seed, difficulty=difficulty, visualize=False)
        env.change_model(model=flag_model, difficulty=difficulty, seed=seed)
        observation = env.reset(project=True, seed=seed)
    except Exception as e_msg:
        print("simulation error!!!")
        print(e_msg)
        #import pdb; pdb.set_trace()
        return 0
    env.spec.timestep_limit = timstep_limit+100

    total_reward = 0
    error_sim = 0;
    t = 0
    for i in range(timstep_limit+100):
        t += sim_dt

        observation, reward, done, info = env.step(params, project=True)
        total_reward += reward

        if done:
            break

    print('    gen#={} sim#={}: score={} time={}sec #step={}'.format(n_gen, i_worker, total_reward, t, env.footstep['n']))

    return total_reward  # minimization

class CMATrainPar(object):
    def __init__(self, ):
        self.n_gen = 0
        self.best_total_reward = -np.inf

    def f(self, v_params):
        self.n_gen += 1
        v_total_reward = Parallel(n_jobs=N_PROC, timeout=TIMEOUT)\
        (delayed(f_ind)(self.n_gen, i, p) for i, p in enumerate(v_params))

        for total_reward in v_total_reward:
            if self.best_total_reward  < total_reward:
                filename = "./optim_data/cma/" + trial_name + "best_w.txt"
                print("")
                print("----")
                print("update the best score!!!!")
                print("\tprev = %.8f" % self.best_total_reward )
                print("\tcurr = %.8f" % total_reward)
                print("\tsave to [%s]" % filename)
                print("----")
                print("")
                self.best_total_reward  = total_reward
                np.savetxt(filename, params)

        return [-r for r in v_total_reward]

if __name__ == '__main__':
    prob = CMATrainPar()

    from cmaes.solver_cma import CMASolverPar
    solver = CMASolverPar(prob)

    solver.options.set("popsize", N_POP)
    solver.options.set("maxiter", 400)
    solver.options.set("verb_filenameprefix", 'optim_data/cma/' + trial_name)
    solver.set_verbose(True)

    x0 = params
    sigma = .01

    res = solver.solve(x0, sigma)
    print(res)
<|endoftext|>


<commit_before># D_puzzle1.py

""" Ground truth: temperature alone causes the color.
"""
from __future__ import print_function
from sense_hat import SenseHat
import time

sense = SenseHat()

print("Temperature    Humidity       Pressure       MagneticField\n\n")

while True:
    # measure variables
    temperature = sense.get_temperature()
    humidity = sense.get_humidity()
    pressure = sense.get_pressure()
    raw = sense.get_compass_raw()
    m_field = raw['x']**2 + raw['y']**2 + raw['z']**2

    t = int(min([max([(temperature - 18.) * 255. / 19., 0]), 255]))

    color = [t,t,t]

    t_out = str(temperature) + 15*" "
    h_out = str(humidity) + 15*" "
    p_out = str(pressure) + 15*" "
    m_out = str(m_field) + 15*" "

    print(t_out + h_out + p_out + m_out, end='\r')

    sense.set_pixels(64 * [color])

    time.sleep(0.1)
<commit_msg>Debugging D scriptswq.<commit_after># D_puzzle1.py

""" Ground truth: temperature alone causes the color.
"""
from __future__ import print_function
from sense_hat import SenseHat
import time

sense = SenseHat()

print("Temperature" + 5*" " + "Humidity" + 5*" " + "Pressure" + 5*" " + " MagneticField\n\n")

while True:
    # measure variables
    temperature = sense.get_temperature()
    humidity = sense.get_humidity()
    pressure = sense.get_pressure()
    raw = sense.get_compass_raw()
    m_field = raw['x']**2 + raw['y']**2 + raw['z']**2

    t = int(min([max([(temperature - 18.) * 255. / 25., 0]), 255]))

    color = [t,0,0]

    t_out = str(temperature) + 15*" "
    h_out = str(humidity) + 5*" "
    p_out = str(pressure) + 5*" "
    m_out = str(m_field) + 5*" "

    print(t_out + h_out + p_out + m_out, end='\r')

    sense.set_pixels(64 * [color])

    time.sleep(0.1)
<|endoftext|>


<commit_before>import numpy as np

class Visdata(object):
      """
      Class to hold all necessary info relating to one set of visibilities.
      Auto-updates amp&phase or real&imag if those values are changed, but
      MUST SET WITH, eg, visobj.amp = (a numpy array of the new values);
      CANNOT USE, eg, visobj.amp[0] = newval, AS THIS DOES NOT CALL THE
      SETTER FUNCTIONS.
      """

      def __init__(self,u,v,real,imag,sigma,ant1=None,ant2=None,PBfwhm=None,filename=None):
            self.u = u
            self.v = v
            self.real = real
            self.imag = imag
            self.sigma = sigma
            self.ant1 = ant1
            self.ant2 = ant2
            self.PBfwhm = PBfwhm
            self.filename = filename

      @property
      def uvdist(self):
            return np.sqrt(self.u**2. + self.v**2.)

      @property
      def real(self):
            return self._real
      @real.setter
      def real(self,val):
            self._real = val
            # Setting amp & phase during __init__ will fail since imag is still unknown
            # Doing so during conjugate() will also fail, but gives a ValueError
            try:
                  self._amp = np.sqrt(self._real**2. + self.imag**2.)
                  self._phase = np.arctan2(self.imag,self._real)
            except (AttributeError,ValueError):
                  self._amp = None
                  self._phase = None

      @property
      def imag(self):
            return self._imag
      @imag.setter
      def imag(self,val):
            self._imag = val
            try:
                  self._amp = np.sqrt(self.real**2. + self._imag**2.)
                  self._phase = np.arctan2(self._imag,self.real)
            except (AttributeError,ValueError):
                  self._amp = None
                  self._phase = None

      @property
      def amp(self):
            return  self._amp
      @amp.setter
      def amp(self,val):
            self._amp = val
            self._real = val * np.cos(self.phase)
            self._imag = val * np.sin(self.phase)

      @property
      def phase(self):
            return self._phase
      @phase.setter
      def phase(self,val):
            self._phase = val
            self._real = self.amp * np.cos(val)
            self._imag = self.amp * np.sin(val)

      def __add__(self,other):
            return Visdata(self.u,self.v,self.real+other.real,self.imag+other.imag,\
                  (self.sigma**-2. + other.sigma**-2.)**-0.5)

      def __sub__(self,other):
            return Visdata(self.u,self.v,self.real-other.real,self.imag-other.imag,\
                  (self.sigma**-2. + other.sigma**-2.)**-0.5)

      def conjugate(self):
            u = np.concatenate((self.u,-self.u))
            v = np.concatenate((self.v,-self.v))
            real = np.concatenate((self.real,self.real))
            imag = np.concatenate((self.imag,-self.imag))
            sigma = np.concatenate((self.sigma,self.sigma))
            ant1 = np.concatenate((self.ant1,self.ant2))
            ant2 = np.concatenate((self.ant2,self.ant1))
            self.u = u
            self.v = v
            self.real = real
            self.imag = imag
            self.sigma = sigma
            self.ant1 = ant1
            self.ant2 = ant2

class ImageData(object):
      """
      Class to hold info relating to images. Simpler than visibility data.
      Each parameter should be a 2D array (even the x&y coordinates). Also allows
      for a mask to be passed, where non-zero points in the mask are considered bad.

      If you don't pass a noise map, one will be created as sqrt(data).
      """

      def __init__(self,x,y,data,sigma=None,mask=None):
            if sigma is None:
                  sigma = np.sqrt(np.abs(np.asarray(data)))

            if mask is None:
                  mask = np.zeros(x.shape)

            self.x = np.asarray(x)
            self.y = np.asarray(y)
            self.data = np.asarray(data)
            self.simga = np.asarray(sigma)
            self.mask = np.asarray(mask)

      def __add__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data+other.data,self.sigma,self.mask)
            else: return ImageData(self.x,self.y,self.data+other,self.sigma,self.mask)

      def __subtract__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data-self.other,self.sigma,self.mask)
            else: return ImageData(self.x,self.y,self.data-other,self.sigma,self.mask)

      def __mul__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data*self.other,self.sigma,self.mask)
            else: return ImageData(self.x,self.y,self.data*other,self.sigma,self.mask)

      def __div__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data/self.other,self.sigma,self.mask)
            else: return ImageData(self.x,self.y,self.data/np.asarray(other),self.sigma,self.mask)

      def __truediv__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data/self.other,self.sigma,self.mask)
            else: return ImageData(self.x,self.y,self.data/np.asarray(other),self.sigma,self.mask)

<commit_msg>added sersic profile<commit_after>import numpy as np

class Visdata(object):
      """
      Class to hold all necessary info relating to one set of visibilities.
      Auto-updates amp&phase or real&imag if those values are changed, but
      MUST SET WITH, eg, visobj.amp = (a numpy array of the new values);
      CANNOT USE, eg, visobj.amp[0] = newval, AS THIS DOES NOT CALL THE
      SETTER FUNCTIONS.
      """

      def __init__(self,u,v,real,imag,sigma,ant1=None,ant2=None,PBfwhm=None,filename=None):
            self.u = u
            self.v = v
            self.real = real
            self.imag = imag
            self.sigma = sigma
            self.ant1 = ant1
            self.ant2 = ant2
            self.PBfwhm = PBfwhm
            self.filename = filename

      @property
      def uvdist(self):
            return np.sqrt(self.u**2. + self.v**2.)

      @property
      def real(self):
            return self._real
      @real.setter
      def real(self,val):
            self._real = val
            # Setting amp & phase during __init__ will fail since imag is still unknown
            # Doing so during conjugate() will also fail, but gives a ValueError
            try:
                  self._amp = np.sqrt(self._real**2. + self.imag**2.)
                  self._phase = np.arctan2(self.imag,self._real)
            except (AttributeError,ValueError):
                  self._amp = None
                  self._phase = None

      @property
      def imag(self):
            return self._imag
      @imag.setter
      def imag(self,val):
            self._imag = val
            try:
                  self._amp = np.sqrt(self.real**2. + self._imag**2.)
                  self._phase = np.arctan2(self._imag,self.real)
            except (AttributeError,ValueError):
                  self._amp = None
                  self._phase = None

      @property
      def amp(self):
            return  self._amp
      @amp.setter
      def amp(self,val):
            self._amp = val
            self._real = val * np.cos(self.phase)
            self._imag = val * np.sin(self.phase)

      @property
      def phase(self):
            return self._phase
      @phase.setter
      def phase(self,val):
            self._phase = val
            self._real = self.amp * np.cos(val)
            self._imag = self.amp * np.sin(val)

      def __add__(self,other):
            return Visdata(self.u,self.v,self.real+other.real,self.imag+other.imag,\
                  (self.sigma**-2. + other.sigma**-2.)**-0.5)

      def __sub__(self,other):
            return Visdata(self.u,self.v,self.real-other.real,self.imag-other.imag,\
                  (self.sigma**-2. + other.sigma**-2.)**-0.5)

      def conjugate(self):
            u = np.concatenate((self.u,-self.u))
            v = np.concatenate((self.v,-self.v))
            real = np.concatenate((self.real,self.real))
            imag = np.concatenate((self.imag,-self.imag))
            sigma = np.concatenate((self.sigma,self.sigma))
            ant1 = np.concatenate((self.ant1,self.ant2))
            ant2 = np.concatenate((self.ant2,self.ant1))
            self.u = u
            self.v = v
            self.real = real
            self.imag = imag
            self.sigma = sigma
            self.ant1 = ant1
            self.ant2 = ant2

class ImageData(object):
      """
      Class to hold info relating to images. Simpler than visibility data.
      Each parameter should be a 2D array (even the x&y coordinates). Also allows
      for a mask to be passed, where non-zero points in the mask are considered bad.

      If you don't pass a noise map, one will be created as sqrt(data).
      """

      def __init__(self,x,y,data,header=None,sigma=None,mask=None,filename=None):
            if noisemap is None:
                  noisemap = np.sqrt(np.abs(np.asarray(data)))

            if mask is None:
                  mask = np.zeros(x.shape)

            self.x = np.asarray(x)
            self.y = np.asarray(y)
            self.header = header
            self.data = np.asarray(data)
            self.noisemap = np.asarray(noisemap)
            self.mask = np.asarray(mask)
            self.filename = filename

      def __add__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data+other.data,self.header,self.sigma,self.mask,self.filename)
            else: return ImageData(self.x,self.y,self.data+other,self.header,self.sigma,self.mask,self.filename)

      def __subtract__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data-self.other,self.header,self.sigma,self.mask,self.filename)
            else: return ImageData(self.x,self.y,self.data-other,self.header,self.sigma,self.mask,self.filename)

      def __mul__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data*self.other,self.header,self.sigma,self.mask,self.filename)
            else: return ImageData(self.x,self.y,self.data*other,self.header,self.sigma,self.mask,self.filename)

      def __div__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data/self.other,self.sigma,self.mask)
            else: return ImageData(self.x,self.y,self.data/np.asarray(other),self.sigma,self.mask)

      def __truediv__(self,other):
            if isinstance(other,ImageData):
                  return ImageData(self.x,self.y,self.data/self.other,self.sigma,self.mask)
            else: return ImageData(self.x,self.y,self.data/np.asarray(other),self.sigma,self.mask)

<|endoftext|>


<commit_before>import redis
from flask import Blueprint, render_template, request, jsonify, current_app, g, url_for
from rq import push_connection, pop_connection, Queue

from .forms import TaskForm
from .. import tasks

bp = Blueprint('main', __name__)

def get_redis_connection():
    redis_connection = getattr(g, '_redis_connection', None)
    if redis_connection is None:
        redis_url = current_app.config['REDIS_URL']
        redis_connection = g._redis_connection = redis.from_url(redis_url)
    return redis_connection

@bp.before_request
def push_rq_connection():
    push_connection(get_redis_connection())

@bp.teardown_request
def pop_rq_connection(exception=None):
    pop_connection()

@bp.route('/status/<job_id>')
def job_status(job_id):
    q = Queue()
    job = q.fetch_job(job_id)
    if job is None:
        response = {'status': 'unknown'}
    else:
        response = {
            'status': job.get_status(),
            'result': job.result,
        }
        if job.is_failed:
            response['message'] = job.exc_info.strip().split('\n')[-1]
    return jsonify(response)

@bp.route('/_run_task', methods=['POST'])
def run_task():
    task = request.form.get('task')
    q = Queue()
    job = q.enqueue(tasks.run, task)
    return jsonify({}), 202, {'Location': url_for('main.job_status', job_id=job.get_id())}

@bp.route('/')
def index():
    form = TaskForm()
    return render_template('index.html', form=form)

@bp.route('/upload', methods=['GET', 'POST'])
def upload():
    if form.validate_on_submit():
        f = form.photo.data
        filename = secure_filename(f.filename)
        f.save(os.path.join(
            app.instance_path, 'photos', filename
        ))
        return redirect(url_for('index'))

    return render_template('upload.html', form=form)
<commit_msg>upload form<commit_after>import redis
from flask import Blueprint, render_template, request, jsonify, current_app, g, url_for
from rq import push_connection, pop_connection, Queue

from .forms import TaskForm
from .. import tasks

from flask_wtf import FlaskForm
from flask_wtf.file import FileField, FileRequired
from werkzeug.utils import secure_filename

bp = Blueprint('main', __name__)

def get_redis_connection():
    redis_connection = getattr(g, '_redis_connection', None)
    if redis_connection is None:
        redis_url = current_app.config['REDIS_URL']
        redis_connection = g._redis_connection = redis.from_url(redis_url)
    return redis_connection

@bp.before_request
def push_rq_connection():
    push_connection(get_redis_connection())

@bp.teardown_request
def pop_rq_connection(exception=None):
    pop_connection()

@bp.route('/status/<job_id>')
def job_status(job_id):
    q = Queue()
    job = q.fetch_job(job_id)
    if job is None:
        response = {'status': 'unknown'}
    else:
        response = {
            'status': job.get_status(),
            'result': job.result,
        }
        if job.is_failed:
            response['message'] = job.exc_info.strip().split('\n')[-1]
    return jsonify(response)

@bp.route('/_run_task', methods=['POST'])
def run_task():
    task = request.form.get('task')
    q = Queue()
    job = q.enqueue(tasks.run, task)
    return jsonify({}), 202, {'Location': url_for('main.job_status', job_id=job.get_id())}

@bp.route('/')
def index():
    form = TaskForm()
    return render_template('index.html', form=form)

@bp.route('/upload', methods=['GET', 'POST'])
def upload():
    if form.validate_on_submit():
        f = form.photo.data
        filename = secure_filename(f.filename)
        f.save(os.path.join(
            app.instance_path, 'photos', filename
        ))
        return redirect(url_for('index'))

    return render_template('upload.html', form=form)
<|endoftext|>


<commit_before>from app.models import JSONModel, ModelList
from app.notify_client.email_branding_client import email_branding_client
from app.notify_client.letter_branding_client import letter_branding_client
from app.notify_client.organisations_api_client import organisations_client

class Branding(JSONModel):
    ALLOWED_PROPERTIES = {"id", "name"}
    __sort_attribute__ = "name"

    def __bool__(self):
        return bool(self.id)

class EmailBranding(Branding):
    ALLOWED_PROPERTIES = Branding.ALLOWED_PROPERTIES | {
        "colour",
        "logo",
        "text",
        "brand_type",
    }

    NHS_ID = "a7dc4e56-660b-4db7-8cff-12c37b12b5ea"

    @classmethod
    def from_id(cls, id):
        if id is None:
            return cls({key: None for key in cls.ALLOWED_PROPERTIES} | {"name": "GOV.UK"})
        return cls(email_branding_client.get_email_branding(id)["email_branding"])

    @property
    def is_nhs(self):
        return self.id == self.NHS_ID

    @property
    def is_govuk(self):
        return self.id is None

class LetterBranding(Branding):
    ALLOWED_PROPERTIES = Branding.ALLOWED_PROPERTIES | {"filename"}

    @classmethod
    def from_id(cls, id):
        if id is None:
            return cls({key: None for key in cls.ALLOWED_PROPERTIES})
        return cls(letter_branding_client.get_letter_branding(id))

    @property
    def is_nhs(self):
        return self.name == "NHS"

class AllBranding(ModelList):
    class NotFound(Exception):
        pass

    @property
    def ids(self):
        return tuple(branding.id for branding in self)

    @property
    def as_id_and_name(self):
        return [(branding.id, branding.name) for branding in self]

    def get_item_by_id(self, id):
        for branding in self:
            if branding.id == id:
                return branding
        raise self.NotFound

    def excluding(self, *ids_to_exclude):
        return tuple(branding for branding in self if branding.id not in ids_to_exclude)

class AllEmailBranding(AllBranding):
    client_method = email_branding_client.get_all_email_branding
    model = EmailBranding

class EmailBrandingPool(AllEmailBranding):
    client_method = organisations_client.get_email_branding_pool

    def __init__(self, id):
        self.items = tuple()
        if id:
            self.items = self.client_method(id)

class AllLetterBranding(AllBranding):
    client_method = letter_branding_client.get_all_letter_branding
    model = LetterBranding
<commit_msg>Make it clearer what the comprehension is doing<commit_after>from app.models import JSONModel, ModelList
from app.notify_client.email_branding_client import email_branding_client
from app.notify_client.letter_branding_client import letter_branding_client
from app.notify_client.organisations_api_client import organisations_client

class Branding(JSONModel):
    ALLOWED_PROPERTIES = {"id", "name"}
    __sort_attribute__ = "name"

    def __bool__(self):
        return bool(self.id)

    @classmethod
    def with_default_values(cls, **kwargs):
        return cls({key: None for key in cls.ALLOWED_PROPERTIES} | kwargs)

class EmailBranding(Branding):
    ALLOWED_PROPERTIES = Branding.ALLOWED_PROPERTIES | {
        "colour",
        "logo",
        "text",
        "brand_type",
    }

    NHS_ID = "a7dc4e56-660b-4db7-8cff-12c37b12b5ea"

    @classmethod
    def from_id(cls, id):
        if id is None:
            return cls.with_default_values(name="GOV.UK")
        return cls(email_branding_client.get_email_branding(id)["email_branding"])

    @property
    def is_nhs(self):
        return self.id == self.NHS_ID

    @property
    def is_govuk(self):
        return self.id is None

class LetterBranding(Branding):
    ALLOWED_PROPERTIES = Branding.ALLOWED_PROPERTIES | {"filename"}

    @classmethod
    def from_id(cls, id):
        if id is None:
            return cls.with_default_values()
        return cls(letter_branding_client.get_letter_branding(id))

    @property
    def is_nhs(self):
        return self.name == "NHS"

class AllBranding(ModelList):
    class NotFound(Exception):
        pass

    @property
    def ids(self):
        return tuple(branding.id for branding in self)

    @property
    def as_id_and_name(self):
        return [(branding.id, branding.name) for branding in self]

    def get_item_by_id(self, id):
        for branding in self:
            if branding.id == id:
                return branding
        raise self.NotFound

    def excluding(self, *ids_to_exclude):
        return tuple(branding for branding in self if branding.id not in ids_to_exclude)

class AllEmailBranding(AllBranding):
    client_method = email_branding_client.get_all_email_branding
    model = EmailBranding

class EmailBrandingPool(AllEmailBranding):
    client_method = organisations_client.get_email_branding_pool

    def __init__(self, id):
        self.items = tuple()
        if id:
            self.items = self.client_method(id)

class AllLetterBranding(AllBranding):
    client_method = letter_branding_client.get_all_letter_branding
    model = LetterBranding
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This module is part of the GeoTag-X project validator tool.
# It contains validators for various components of a tutorial configuration.
#
# Author: Jeremy Othieno (j.othieno@gmail.com)
#
# Copyright (c) 2016-2017 UNITAR/UNOSAT
#
# The MIT License (MIT)
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.
from helper import check_arg_type, is_empty_string, is_configuration_string, find_unexpected_keys

def is_tutorial_configuration(
    tutorial_configuration,
    task_presenter_configuration,
    enable_logging=False,
    validate_task_presenter_configuration=True
):
    """Validates the specified tutorial configuration.

    Args:
        tutorial_configuration (dict): A tutorial configuration to validate.
        task_presenter_configuration (dict): The task presenter configuration complemented by the tutorial configuration.
        enable_logging (bool): If set to True, the function will log the operations it performs.
        validate_task_presenter_configuration (bool): If set to True, the specified task presenter configuration is validated too.

    Returns:
        <bool, str|None>: A pair containing the value True if the specified configuration
            is valid, False otherwise; and an error message in case the configuration is invalid.

    Raises:
        TypeError: If either of the configuration arguments is not a dictionary, or the
            remaining arguments are not booleans.
    """
    check_arg_type(is_tutorial_configuration, "tutorial_configuration", tutorial_configuration, dict)
    check_arg_type(is_tutorial_configuration, "task_presenter_configuration", task_presenter_configuration, dict)
    check_arg_type(is_tutorial_configuration, "enable_logging", enable_logging, bool)
    check_arg_type(is_tutorial_configuration, "validate_task_presenter_configuration", validate_task_presenter_configuration, bool)

    if validate_task_presenter_configuration:
        from task_presenter import is_task_presenter_configuration
        valid, message = is_task_presenter_configuration(task_presenter_configuration)
        if not valid:
            return (False, message)

    missing = [k for k in is_tutorial_configuration.REQUIRED_FIELDS if k not in tutorial_configuration]
    if missing:
        return (False, "The tutorial configuration is missing the following fields: '{}'.".format("', '".join(missing)))

    available_languages = task_presenter_configuration["language"]["available"] if "language" in task_presenter_configuration else None

    from functools import partial
    validators = {
        "enable-random-order": is_tutorial_enable_random_order,
        "default-message": partial(is_tutorial_default_message, languages=available_languages),
        "subjects": partial(__is_tutorial_subjects, languages=available_languages),
    }
    for key, configuration in tutorial_configuration.iteritems():
        validator = validators.get(key)
        if not validator:
            return (False, "The tutorial configuration key '{}' is not recognized.".format(key))

        valid, message = validator(configuration)
        if not valid:
            return (False, message)

    return (True, None)

is_tutorial_configuration.REQUIRED_FIELDS = frozenset([
    "subjects",
])
"""A set of required tutorial configuration fields."""

def is_tutorial_enable_random_order(enable_random_order):
    """Validates the specified 'Enable random order' flag.

    Args:
        enable_random_order (bool): A flag to validate.

    Returns:
        <bool, str|None>: A pair containing the value True if the specified flag
            is valid, False otherwise; and an error message in case the flag is invalid.
    """
    error_message = "The 'enable-random-order' argument must be a boolean."
    return (True, None) if isinstance(enable_random_order, bool) else (False, error_message)

def is_tutorial_default_message(default_message, languages=None):
    """Validates the specified set of default messages.

    Args:
        default_message (dict): A set of default messages.
        languages (list): A list of available languages.

    Returns:
        <bool, str|None>: A pair containing the value True if the specified message set
            is valid, False otherwise; and an error message in case the set is invalid.
    Raises:
        TypeError: If the default_message argument is not a dictionary, or if the
            languages argument is not a list or NoneType.
    """
    check_arg_type(is_tutorial_default_message, "default_message", default_message, dict)
    check_arg_type(is_tutorial_default_message, "languages", languages, (list, type(None)))

    unexpected_keys = find_unexpected_keys(default_message, is_tutorial_configuration.DEFAULT_MESSAGE_FIELDS)
    if unexpected_keys:
        message = "The tutorial's 'default-message' contains the following unrecognized fields: '{}'."
        return (False, message.format("', '".join(unexpected_keys)))

    for key, message in default_message.iteritems():
        if not is_configuration_string(message, languages):
            return (False, "The tutorial's default message field '{}' is invalid. A message must be a non-empty or normalized string.".format(key))

    return (True, None)

is_tutorial_configuration.DEFAULT_MESSAGE_FIELDS = frozenset([
    "on-wrong-answer",
    "on-correct-answer",
])
"""A set of default message fields."""

def __is_tutorial_subjects(tutorial_subjects, languages=None):
    """Validates the specified list of tutorial subjects.

    Args:
        tutorial_subjects (list): A list of tutorial subjects to validate.
        languages (list): A list of available languages.

    Returns:
        <bool, str|None>: A pair containing the value True if the specified list
            is valid, False otherwise; and an error message in case the list is invalid.

    Raises:
        TypeError: If the tutorial_subjects argument is not a list, or the languages
            argument is not a list or NoneType.
    """
    check_arg_type(__is_tutorial_subjects, "tutorial_subjects", tutorial_subjects, list)
    check_arg_type(__is_tutorial_subjects, "languages", languages, (list, type(None)))

    if not tutorial_subjects:
        return (False, "A project tutorial must contain one or more subjects.")

    for s in tutorial_subjects:
        valid, message = is_tutorial_subject(s, languages)
        if not valid:
            return (False, message)

    return (True, None)

def is_tutorial_subject(tutorial_subject, languages=None):
    """Validates the specified tutorial subject.

    Args:
        tutorial_subject (dict): A tutorial subject to validate.
        languages (list): A list of available languages.

    Returns:
        <bool, str|NoneType>: A pair containing the value True if the specified subject
            is valid, False otherwise; and an error message in case the subject is invalid.

    Raises:
        TypeError: If the tutorial_subject argument is not a dictionary, or the languages
            argument is not a list or NoneType.
    """
    check_arg_type(is_tutorial_subject, "tutorial_subject", tutorial_subject, dict)
    check_arg_type(is_tutorial_subject, "languages", languages, (list, type(None)))

    missing = [k for k in is_tutorial_subject.REQUIRED_FIELDS if k not in tutorial_subject]
    if missing:
        message = "A tutorial's subject configuration is missing the following fields: '{}'."
        return (False, message.format("', '".join(missing)))

    def is_source(subject_source):
        message = "A tutorial subject's 'source' field must be a non-empty string."
        return (False, message) if is_empty_string(subject_source) else (True, None)

    def is_page(subject_page):
        message = "A tutorial subject's 'page' field must be a non-empty string."
        return (False, message) if is_empty_string(subject_page) else (True, None)

    def is_assertions(subject_assertions):
        check_arg_type(is_assertions, "subject_assertions", subject_assertions, dict)

        from question import is_question_key
        for key, assertion in subject_assertions.iteritems():
            valid, message = is_question_key(key)
            if not valid:
                return (False, message)

            valid, message = is_tutorial_subject_assertion(assertion, languages)
            if not valid:
                return (False, message)

        return (True, None)

    validators = {
        "source": is_source,
        "page": is_page,
        "assertions": is_assertions,
    }
    for key, field in tutorial_subject.iteritems():
        validator = validators.get(key)
        if not validator:
            return (False, "The tutorial subject field '{}' is not recognized.".format(key))

        valid, message = validator(field)
        if not valid:
            return (False, message)

    return (True, None)

is_tutorial_subject.REQUIRED_FIELDS = frozenset([
    "source",
    "page",
    "assertions",
])
"""A collection of required tutorial subject fields."""

def is_tutorial_subject_assertion(tutorial_subject_assertion, languages=None):
    """Validates the specified tutorial subject assertion.

    Args:
        tutorial_subject_assertion (dict): A subject assertion to validate.
        languages (list): A list of available languages.

    Returns:
        <bool, str|NoneType>: A pair containing the value True if the specified assertion
            is valid, False otherwise; and an error message in case validation failed.

    Raises:
        TypeError: If the tutorial_subject_assertion argument is not a dictionary, or
            the languages argument is not a list or NoneType.
    """
    check_arg_type(is_tutorial_subject_assertion, "tutorial_subject_assertion", tutorial_subject_assertion, dict)
    check_arg_type(is_tutorial_subject_assertion, "languages", languages, (list, type(None)))

    missing = [k for k in is_tutorial_subject_assertion.REQUIRED_FIELDS if k not in tutorial_subject_assertion]
    if missing:
        message = "A tutorial subject assertion is missing the following fields: '{}'."
        return (False, message.format("', '".join(missing)))

    def is_expects(assertion_expects):
        message = "A tutorial subject assertion's 'expects' field must be a non-empty string."
        return (False, message) if is_empty_string(assertion_expects) else (True, None)

    def is_messages(assertion_messages):
        check_arg_type(is_messages, "assertion_messages", assertion_messages, dict)
        if any(not is_configuration_string(m, languages) for m in assertion_messages.itervalues()):
            return (False, "A tutorial subject assertion message must be a non-empty or normalized string.")
        return (True, None)

    def is_autocomplete(assertion_autocomplete):
        message = "A tutorial subject assertion's 'autocomplete' field must contain a boolean value."
        return (True, None) if isinstance(assertion_autocomplete, bool) else (False, message)

    validators = {
        "expects": is_expects,
        "messages": is_messages,
        "autocomplete": is_autocomplete,
    }
    for key, field in tutorial_subject_assertion.iteritems():
        validator = validators.get(key)
        if not validator:
            return (False, "The tutorial subject assertion field '{}' is not recognized.".format(key))
        else:
            valid, message = validator(field)
            if not valid:
                return (False, message)

    return (True, None)

is_tutorial_subject_assertion.REQUIRED_FIELDS = frozenset([
    "expects",
])
"""A collection of required tutorial subject assertion fields."""
<commit_msg>Refactor the is_tutorial_configuration function<commit_after># -*- coding: utf-8 -*-
#
# This module is part of the GeoTag-X project validator tool.
# It contains validators for various components of a tutorial configuration.
#
# Author: Jeremy Othieno (j.othieno@gmail.com)
#
# Copyright (c) 2016-2017 UNITAR/UNOSAT
#
# The MIT License (MIT)
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.
from helper import check_arg_type, is_empty_string, is_configuration_string, find_unexpected_keys

def is_tutorial_configuration(
    tutorial_configuration,
    task_presenter_configuration,
    enable_logging=False,
    validate_task_presenter_configuration=True
):
    """Validates the specified tutorial configuration.

    Args:
        tutorial_configuration (dict): A tutorial configuration to validate.
        task_presenter_configuration (dict): The task presenter configuration complemented by the tutorial configuration.
        enable_logging (bool): If set to True, the function will log the operations it performs.
        validate_task_presenter_configuration (bool): If set to True, the specified task presenter configuration is validated too.

    Returns:
        <bool, str|None>: A pair containing the value True if the specified configuration
            is valid, False otherwise; and an error message in case the configuration is invalid.

    Raises:
        TypeError: If either of the configuration arguments is not a dictionary, or the
            remaining arguments are not booleans.
    """
    check_arg_type(is_tutorial_configuration, "tutorial_configuration", tutorial_configuration, dict)
    check_arg_type(is_tutorial_configuration, "task_presenter_configuration", task_presenter_configuration, dict)
    check_arg_type(is_tutorial_configuration, "enable_logging", enable_logging, bool)
    check_arg_type(is_tutorial_configuration, "validate_task_presenter_configuration", validate_task_presenter_configuration, bool)

    if validate_task_presenter_configuration:
        from task_presenter import is_task_presenter_configuration
        valid, message = is_task_presenter_configuration(task_presenter_configuration)
        if not valid:
            return (False, message)

    missing = [k for k in is_tutorial_configuration.REQUIRED_FIELDS if k not in tutorial_configuration]
    if missing:
        return (False, "The tutorial configuration is missing the following fields: '{}'.".format("', '".join(missing)))

    available_languages = task_presenter_configuration["language"]["available"] if "language" in task_presenter_configuration else None

    def is_subjects(subjects):
        check_arg_type(is_subjects, "subjects", subjects, list)
        if not subjects:
            return (False, "A project tutorial must contain at least one subject.")
        for subject in subjects:
            valid, message = is_tutorial_subject(subject, available_languages)
            if not valid:
                return (False, message)
        return (True, None)

    from functools import partial
    validators = {
        "enable-random-order": is_tutorial_enable_random_order,
        "default-message": partial(is_tutorial_default_message, languages=available_languages),
        "subjects": is_subjects,
    }
    for key, configuration in tutorial_configuration.iteritems():
        validator = validators.get(key)
        if not validator:
            return (False, "The tutorial configuration key '{}' is not recognized.".format(key))

        valid, message = validator(configuration)
        if not valid:
            return (False, message)

    return (True, None)

is_tutorial_configuration.REQUIRED_FIELDS = frozenset([
    "subjects",
])
"""A set of required tutorial configuration fields."""

def is_tutorial_enable_random_order(enable_random_order):
    """Validates the specified 'Enable random order' flag.

    Args:
        enable_random_order (bool): A flag to validate.

    Returns:
        <bool, str|None>: A pair containing the value True if the specified flag
            is valid, False otherwise; and an error message in case the flag is invalid.
    """
    error_message = "The 'enable-random-order' argument must be a boolean."
    return (True, None) if isinstance(enable_random_order, bool) else (False, error_message)

def is_tutorial_default_message(default_message, languages=None):
    """Validates the specified set of default messages.

    Args:
        default_message (dict): A set of default messages.
        languages (list): A list of available languages.

    Returns:
        <bool, str|None>: A pair containing the value True if the specified message set
            is valid, False otherwise; and an error message in case the set is invalid.
    Raises:
        TypeError: If the default_message argument is not a dictionary, or if the
            languages argument is not a list or NoneType.
    """
    check_arg_type(is_tutorial_default_message, "default_message", default_message, dict)
    check_arg_type(is_tutorial_default_message, "languages", languages, (list, type(None)))

    unexpected_keys = find_unexpected_keys(default_message, is_tutorial_configuration.DEFAULT_MESSAGE_FIELDS)
    if unexpected_keys:
        message = "The tutorial's 'default-message' contains the following unrecognized fields: '{}'."
        return (False, message.format("', '".join(unexpected_keys)))

    for key, message in default_message.iteritems():
        if not is_configuration_string(message, languages):
            return (False, "The tutorial's default message field '{}' is invalid. A message must be a non-empty or normalized string.".format(key))

    return (True, None)

is_tutorial_configuration.DEFAULT_MESSAGE_FIELDS = frozenset([
    "on-wrong-answer",
    "on-correct-answer",
])
"""A set of default message fields."""

def is_tutorial_subject(tutorial_subject, languages=None):
    """Validates the specified tutorial subject.

    Args:
        tutorial_subject (dict): A tutorial subject to validate.
        languages (list): A list of available languages.

    Returns:
        <bool, str|NoneType>: A pair containing the value True if the specified subject
            is valid, False otherwise; and an error message in case the subject is invalid.

    Raises:
        TypeError: If the tutorial_subject argument is not a dictionary, or the languages
            argument is not a list or NoneType.
    """
    check_arg_type(is_tutorial_subject, "tutorial_subject", tutorial_subject, dict)
    check_arg_type(is_tutorial_subject, "languages", languages, (list, type(None)))

    missing = [k for k in is_tutorial_subject.REQUIRED_FIELDS if k not in tutorial_subject]
    if missing:
        message = "A tutorial's subject configuration is missing the following fields: '{}'."
        return (False, message.format("', '".join(missing)))

    def is_source(subject_source):
        message = "A tutorial subject's 'source' field must be a non-empty string."
        return (False, message) if is_empty_string(subject_source) else (True, None)

    def is_page(subject_page):
        message = "A tutorial subject's 'page' field must be a non-empty string."
        return (False, message) if is_empty_string(subject_page) else (True, None)

    def is_assertions(subject_assertions):
        check_arg_type(is_assertions, "subject_assertions", subject_assertions, dict)

        from question import is_question_key
        for key, assertion in subject_assertions.iteritems():
            valid, message = is_question_key(key)
            if not valid:
                return (False, message)

            valid, message = is_tutorial_subject_assertion(assertion, languages)
            if not valid:
                return (False, message)

        return (True, None)

    validators = {
        "source": is_source,
        "page": is_page,
        "assertions": is_assertions,
    }
    for key, field in tutorial_subject.iteritems():
        validator = validators.get(key)
        if not validator:
            return (False, "The tutorial subject field '{}' is not recognized.".format(key))

        valid, message = validator(field)
        if not valid:
            return (False, message)

    return (True, None)

is_tutorial_subject.REQUIRED_FIELDS = frozenset([
    "source",
    "page",
    "assertions",
])
"""A collection of required tutorial subject fields."""

def is_tutorial_subject_assertion(tutorial_subject_assertion, languages=None):
    """Validates the specified tutorial subject assertion.

    Args:
        tutorial_subject_assertion (dict): A subject assertion to validate.
        languages (list): A list of available languages.

    Returns:
        <bool, str|NoneType>: A pair containing the value True if the specified assertion
            is valid, False otherwise; and an error message in case validation failed.

    Raises:
        TypeError: If the tutorial_subject_assertion argument is not a dictionary, or
            the languages argument is not a list or NoneType.
    """
    check_arg_type(is_tutorial_subject_assertion, "tutorial_subject_assertion", tutorial_subject_assertion, dict)
    check_arg_type(is_tutorial_subject_assertion, "languages", languages, (list, type(None)))

    missing = [k for k in is_tutorial_subject_assertion.REQUIRED_FIELDS if k not in tutorial_subject_assertion]
    if missing:
        message = "A tutorial subject assertion is missing the following fields: '{}'."
        return (False, message.format("', '".join(missing)))

    def is_expects(assertion_expects):
        message = "A tutorial subject assertion's 'expects' field must be a non-empty string."
        return (False, message) if is_empty_string(assertion_expects) else (True, None)

    def is_messages(assertion_messages):
        check_arg_type(is_messages, "assertion_messages", assertion_messages, dict)
        if any(not is_configuration_string(m, languages) for m in assertion_messages.itervalues()):
            return (False, "A tutorial subject assertion message must be a non-empty or normalized string.")
        return (True, None)

    def is_autocomplete(assertion_autocomplete):
        message = "A tutorial subject assertion's 'autocomplete' field must contain a boolean value."
        return (True, None) if isinstance(assertion_autocomplete, bool) else (False, message)

    validators = {
        "expects": is_expects,
        "messages": is_messages,
        "autocomplete": is_autocomplete,
    }
    for key, field in tutorial_subject_assertion.iteritems():
        validator = validators.get(key)
        if not validator:
            return (False, "The tutorial subject assertion field '{}' is not recognized.".format(key))
        else:
            valid, message = validator(field)
            if not valid:
                return (False, message)

    return (True, None)

is_tutorial_subject_assertion.REQUIRED_FIELDS = frozenset([
    "expects",
])
"""A collection of required tutorial subject assertion fields."""
<|endoftext|>


<commit_before>from flask.ext.wtf import Form
from wtforms.fields import (
    StringField,
    IntegerField,
    SubmitField
)
from wtforms.validators import InputRequired, Length

class ResourceForm(Form):
    autocomplete = StringField('Enter the address')

    name = StringField('Name', validators=[
        InputRequired(),
        Length(1, 64)
    ])
    description = StringField('Description', validators=[
        InputRequired(),
    ])
    website = StringField('Website')

    street_number = IntegerField('Street Number', validators=[
        InputRequired()
    ])
    route = StringField('Street Address', validators=[
        InputRequired()
    ])
    locality = StringField('City', validators=[
        InputRequired()
    ])
    administrative_area_level_1 = StringField('State', validators=[
        InputRequired()
    ])
    postal_code = StringField('ZIP Code', validators=[
        InputRequired(),
        Length(5)
    ])

    submit = SubmitField('Add Resource')

class ReviewForm(Form):
    rating = IntegerField('Rating', validators=[
        InputRequired()
        # Length(1,1)
    ])
    content = StringField('Content', validators=[
        InputRequired()
    ])
    submit = SubmitField('Add Review')
<commit_msg>ReviewForm has number range and button name is generalized for writing and editing review<commit_after>from flask.ext.wtf import Form
from wtforms.fields import (
    StringField,
    IntegerField,
    SubmitField
)
from wtforms.validators import InputRequired, Length, NumberRange

class ResourceForm(Form):
    autocomplete = StringField('Enter the address')

    name = StringField('Name', validators=[
        InputRequired(),
        Length(1, 64)
    ])
    description = StringField('Description', validators=[
        InputRequired(),
    ])
    website = StringField('Website')

    street_number = IntegerField('Street Number', validators=[
        InputRequired()
    ])
    route = StringField('Street Address', validators=[
        InputRequired()
    ])
    locality = StringField('City', validators=[
        InputRequired()
    ])
    administrative_area_level_1 = StringField('State', validators=[
        InputRequired()
    ])
    postal_code = StringField('ZIP Code', validators=[
        InputRequired(),
        Length(5)
    ])

    submit = SubmitField('Add Resource')

class ReviewForm(Form):
    rating = IntegerField('Rating', validators=[
        InputRequired(),
        NumberRange(1, 5)
    ])
    content = StringField('Content', validators=[
        InputRequired()
    ])
    submit = SubmitField('Finish Review')
<|endoftext|>


<commit_before>import sys
import os
import re
import socket
import zlib
import imp
import subprocess as ssubprocess
import sshuttle.helpers as helpers
from sshuttle.helpers import debug2

def readfile(name):
    tokens = name.split(".")
    f = None

    token = tokens[0]
    token_name = [token]
    token_str = ".".join(token_name)

    try:
        f, pathname, description = imp.find_module(token_str)

        for token in tokens[1:]:
            module = imp.load_module(token_str, f, pathname, description)
            if f is not None:
                f.close()

            token_name.append(token)
            token_str = ".".join(token_name)

            f, pathname, description = imp.find_module(
                token, module.__path__)

        if f is not None:
            contents = f.read()
        else:
            contents = ""

    finally:
        if f is not None:
            f.close()

    return contents.encode("UTF8")

def empackage(z, name, data=None):
    if not data:
        data = readfile(name)
    content = z.compress(data)
    content += z.flush(zlib.Z_SYNC_FLUSH)

    return b'%s\n%d\n%s' % (name.encode("ASCII"), len(content), content)

def connect(ssh_cmd, rhostport, python, stderr, options):
    portl = []

    if (rhostport or '').count(':') > 1:
        if rhostport.count(']') or rhostport.count('['):
            result = rhostport.split(']')
            rhost = result[0].strip('[')
            if len(result) > 1:
                result[1] = result[1].strip(':')
                if result[1] is not '':
                    portl = ['-p', str(int(result[1]))]
        # can't disambiguate IPv6 colons and a port number. pass the hostname
        # through.
        else:
            rhost = rhostport
    else:  # IPv4
        l = (rhostport or '').split(':', 1)
        rhost = l[0]
        if len(l) > 1:
            portl = ['-p', str(int(l[1]))]

    if rhost == '-':
        rhost = None

    z = zlib.compressobj(1)
    content = readfile('sshuttle.assembler')
    optdata = ''.join("%s=%r\n" % (k, v) for (k, v) in list(options.items()))
    optdata = optdata.encode("UTF8")
    content2 = (empackage(z, 'sshuttle') +
                empackage(z, 'sshuttle.cmdline_options', optdata) +
                empackage(z, 'sshuttle.helpers') +
                empackage(z, 'sshuttle.ssnet') +
                empackage(z, 'sshuttle.hostwatch') +
                empackage(z, 'sshuttle.server') +
                b"\n")

    pyscript = r"""
                import sys;
                verbosity=%d;
                stdin=getattr(sys.stdin,"buffer",sys.stdin);
                exec(compile(stdin.read(%d), "assembler.py", "exec"))
                """ % (helpers.verbose or 0, len(content))
    pyscript = re.sub(r'\s+', ' ', pyscript.strip())

    if not rhost:
        # ignore the --python argument when running locally; we already know
        # which python version works.
        argv = [sys.argv[1], '-c', pyscript]
    else:
        if ssh_cmd:
            sshl = ssh_cmd.split(' ')
        else:
            sshl = ['ssh']
        if python:
            pycmd = "'%s' -c '%s'" % (python, pyscript)
        else:
            pycmd = ("exec /bin/sh -c \'P=python3.5; $P -V 2>/dev/null || P=python; "
                     "exec \"$P\" -c \\'%s\\'\'") % pyscript
        argv = (sshl +
                portl +
                [rhost, '--', pycmd])
    (s1, s2) = socket.socketpair()

    def setup():
        # runs in the child process
        s2.close()
    s1a, s1b = os.dup(s1.fileno()), os.dup(s1.fileno())
    s1.close()
    debug2('executing: %r\n' % argv)
    p = ssubprocess.Popen(argv, stdin=s1a, stdout=s1b, preexec_fn=setup,
                          close_fds=True, stderr=stderr)
    os.close(s1a)
    os.close(s1b)
    s2.sendall(content)
    s2.sendall(content2)
    return p, s2
<commit_msg>Fix shell quoting<commit_after>import sys
import os
import re
import socket
import zlib
import imp
import subprocess as ssubprocess
import sshuttle.helpers as helpers
from sshuttle.helpers import debug2

try:
    # Python >= 3.5
    from shlex import quote
except ImportError:
    # Python 2.x
    from pipes import quote

def readfile(name):
    tokens = name.split(".")
    f = None

    token = tokens[0]
    token_name = [token]
    token_str = ".".join(token_name)

    try:
        f, pathname, description = imp.find_module(token_str)

        for token in tokens[1:]:
            module = imp.load_module(token_str, f, pathname, description)
            if f is not None:
                f.close()

            token_name.append(token)
            token_str = ".".join(token_name)

            f, pathname, description = imp.find_module(
                token, module.__path__)

        if f is not None:
            contents = f.read()
        else:
            contents = ""

    finally:
        if f is not None:
            f.close()

    return contents.encode("UTF8")

def empackage(z, name, data=None):
    if not data:
        data = readfile(name)
    content = z.compress(data)
    content += z.flush(zlib.Z_SYNC_FLUSH)

    return b'%s\n%d\n%s' % (name.encode("ASCII"), len(content), content)

def connect(ssh_cmd, rhostport, python, stderr, options):
    portl = []

    if (rhostport or '').count(':') > 1:
        if rhostport.count(']') or rhostport.count('['):
            result = rhostport.split(']')
            rhost = result[0].strip('[')
            if len(result) > 1:
                result[1] = result[1].strip(':')
                if result[1] is not '':
                    portl = ['-p', str(int(result[1]))]
        # can't disambiguate IPv6 colons and a port number. pass the hostname
        # through.
        else:
            rhost = rhostport
    else:  # IPv4
        l = (rhostport or '').split(':', 1)
        rhost = l[0]
        if len(l) > 1:
            portl = ['-p', str(int(l[1]))]

    if rhost == '-':
        rhost = None

    z = zlib.compressobj(1)
    content = readfile('sshuttle.assembler')
    optdata = ''.join("%s=%r\n" % (k, v) for (k, v) in list(options.items()))
    optdata = optdata.encode("UTF8")
    content2 = (empackage(z, 'sshuttle') +
                empackage(z, 'sshuttle.cmdline_options', optdata) +
                empackage(z, 'sshuttle.helpers') +
                empackage(z, 'sshuttle.ssnet') +
                empackage(z, 'sshuttle.hostwatch') +
                empackage(z, 'sshuttle.server') +
                b"\n")

    pyscript = r"""
                import sys;
                verbosity=%d;
                stdin=getattr(sys.stdin,"buffer",sys.stdin);
                exec(compile(stdin.read(%d), "assembler.py", "exec"))
                """ % (helpers.verbose or 0, len(content))
    pyscript = re.sub(r'\s+', ' ', pyscript.strip())

    if not rhost:
        # ignore the --python argument when running locally; we already know
        # which python version works.
        argv = [sys.argv[1], '-c', pyscript]
    else:
        if ssh_cmd:
            sshl = ssh_cmd.split(' ')
        else:
            sshl = ['ssh']
        if python:
            pycmd = "'%s' -c '%s'" % (python, pyscript)
        else:
            pycmd = ("P=python3.5; $P -V 2>/dev/null || P=python; "
                     "exec \"$P\" -c %s") % quote(pyscript)
            pycmd = ("exec /bin/sh -c %s" % quote(pycmd))
        argv = (sshl +
                portl +
                [rhost, '--', pycmd])
    (s1, s2) = socket.socketpair()

    def setup():
        # runs in the child process
        s2.close()
    s1a, s1b = os.dup(s1.fileno()), os.dup(s1.fileno())
    s1.close()
    debug2('executing: %r\n' % argv)
    p = ssubprocess.Popen(argv, stdin=s1a, stdout=s1b, preexec_fn=setup,
                          close_fds=True, stderr=stderr)
    os.close(s1a)
    os.close(s1b)
    s2.sendall(content)
    s2.sendall(content2)
    return p, s2
<|endoftext|>