<commit_before>__author__ = 'jonathan'

import logging
import time
import uuid

import pandas as pd
from sqlalchemy.sql.expression import BinaryExpression
from sqlalchemy.util._collections import KeyedTuple

from lib.rome.core.dataformat import get_decoder
from lib.rome.core.lazy import LazyValue
from lib.rome.core.utils import get_objects, is_novabase

import math

file_logger_enabled = False
try:
    file_logger = logging.getLogger('rome_file_logger')
    hdlr = logging.FileHandler('/opt/logs/rome.log')
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    file_logger.addHandler(hdlr)
    file_logger.setLevel(logging.INFO)
    file_logger_enabled = True
except:
    pass

def all_selectable_are_functions(models):
    return all(x._is_function for x in [y for y in models if not y.is_hidden])

def has_attribute(obj, key):
    if type(obj) is dict:
        return key in obj
    else:
        return hasattr(obj, key)

def set_attribute(obj, key, value):
    if type(obj) is dict:
        obj[key] = value
    else:
        return setattr(obj, key, value)

def get_attribute(obj, key, default=None):
    if type(obj) is dict:
        return obj[key] if key in obj else default
    else:
        return getattr(obj, key, default)

def find_table_name(model):
    """This function return the name of the given model as a String. If the
    model cannot be identified, it returns "none".
    :param model: a model object candidate
    :return: the table name or "none" if the object cannot be identified
    """
    if has_attribute(model, "__tablename__"):
        return model.__tablename__
    if has_attribute(model, "table"):
        return model.table.name
    if has_attribute(model, "class_"):
        return model.class_.__tablename__
    if has_attribute(model, "clauses"):
        for clause in model.clauses:
            return find_table_name(clause)
    return "none"

def extract_models(l):
    already_processed = set()
    result = []
    for selectable in [x for x in l if not x._is_function]:
        if not selectable._model in already_processed:
            already_processed.add(selectable._model)
            result += [selectable]
    return result

def extract_sub_row(row, selectables):
    """Adapt a row result to the expectation of sqlalchemy.
    :param row: a list of python objects
    :param selectables: a list entity class
    :return: the response follows what is required by sqlalchemy (if len(model)==1, a single object is fine, in
    the other case, a KeyTuple where each sub object is associated with it's entity name
    """
    if len(selectables) > 1:
        labels = []
        for selectable in selectables:
            labels += [find_table_name(selectable._model)]
        product = []
        for label in labels:
            product = product + [get_attribute(row, label)]
        return KeyedTuple(product, labels=labels)
    else:
        model_name = find_table_name(selectables[0]._model)
        return get_attribute(row, model_name)

def intersect(b1, b2):
    return [val for val in b1 if val in b2]

def flatten(lis):
    """Given a list, possibly nested to any level, return it flattened."""
    new_lis = []
    for item in lis:
        if type(item) == type([]):
            new_lis.extend(flatten(item))
        else:
            new_lis.append(item)
    return new_lis

# def flatten(l):
#     return [item for sublist in l for item in sublist]

def extract_table_data(term):
    term_value = str(term)
    if "." in term_value:
        return {"table": term_value.split(".")[0], "column": term_value.split(".")[1]}
    else:
        return None

def extract_joining_criterion(exp):
    from lib.rome.core.expression.expression import BooleanExpression
    if type(exp) is BooleanExpression:
        return map(lambda x:extract_joining_criterion(x), exp.exps)
    elif type(exp) is BinaryExpression:
        return [[extract_table_data(exp.left)] + [extract_table_data(exp.right)]]
    else:
        return []

def extract_joining_criterion_from_relationship(rel, local_table):
    local_tabledata = {"table": local_table, "column": rel.local_fk_field}
    remote_tabledata = {"table": rel.remote_object_tablename, "column": rel.remote_object_field}
    return [local_tabledata, remote_tabledata]

def correct_boolean_int(expression_str):
    expression_str = expression_str.replace("___deleted == 0", "___deleted in [0, None]")
    return expression_str

def building_tuples(lists_results, labels, criterions, hints=[]):

    """ Build tuples (join operator in relational algebra). """

    """ Create the Dataframe indexes. """
    dataframes = []
    dataindex = {}
    substitution_index = {}
    normal_keys_index = {}
    refactored_keys_index = {}
    normal_keys_to_key_index = {}
    refactored_keys_to_key_index = {}
    index = 0
    for list_results in lists_results:
        label = labels[index]
        if len(list_results) == 0:
            continue
        keys = map(lambda x: x, list_results[0]) + ["created_at", "updated_at"]

        dataframe = pd.DataFrame(data=list_results, columns=keys)

        for value in keys:
            normal_key = "%s.%s" % (label, value)
            refactored_keys = "%s___%s" % (label, value)
            normal_keys_to_key_index[normal_key] = value
            refactored_keys_to_key_index[refactored_keys] = value
        normal_keys = map(lambda x: "%s.%s" % (label, x), keys)
        normal_keys_index[label] = normal_keys
        refactored_keys = map(lambda x: "%s___%s" % (label, x), keys)
        refactored_keys_index[label] = refactored_keys
        for (a, b) in zip(normal_keys, refactored_keys):
            substitution_index[a] = b
        dataframe.columns = refactored_keys
        dataframes += [dataframe]

        """ Index the dataframe and create a reverse index. """
        dataindex[label] = index
        index += 1

    """ Collecting joining expressions. """
    joining_pairs = []
    non_joining_criterions = []
    for criterion in criterions:
        _joining_pairs = criterion.extract_joining_pairs()
        _nonjoining_criterions = criterion.extract_nonjoining_criterions()

        joining_pairs += _joining_pairs
        non_joining_criterions += _nonjoining_criterions

    """ Construct the resulting rows. """
    result = None
    processed_tables = []
    for joining_pair in joining_pairs:
        """ Preparing the tables that will be joined. """
        attribute_1 = joining_pair[0].strip()
        attribute_2 = joining_pair[1].strip()
        tablename_1 = attribute_1.split(".")[0]
        tablename_2 = attribute_2.split(".")[0]

        if tablename_1 not in dataindex or tablename_2 not in dataindex:
            return []
        index_1 = dataindex[tablename_1]
        index_2 = dataindex[tablename_2]
        dataframe_1 = dataframes[index_1] if not tablename_1 in processed_tables else result
        dataframe_2 = dataframes[index_2] if not tablename_2 in processed_tables else result

        refactored_attribute_1 = attribute_1.split(".")[0]+"___"+attribute_1.split(".")[1]
        refactored_attribute_2 = attribute_2.split(".")[0]+"___"+attribute_2.split(".")[1]

        """ Join the tables. """
        try:
            result = pd.merge(dataframe_1, dataframe_2, left_on=refactored_attribute_1, right_on=refactored_attribute_2)
        except KeyError:
            return []
        """ Update the history of processed tables. """
        processed_tables += [tablename_1, tablename_2]
        processed_tables = list(set(processed_tables))

    """ Filtering rows. """
    if result is None:
        if len(dataframes) == 0:
            return []
        result = dataframes[0]

    for non_joining_criterion in non_joining_criterions:
        expression_str = non_joining_criterion.raw_expression
        for value in substitution_index:
            if value in expression_str:
                corresponding_key = substitution_index[value]
                expression_str = expression_str.replace(value, corresponding_key)
        try:
            corrected_expression = correct_boolean_int(expression_str)
            result = result.query(corrected_expression)
        except:
            pass

    """ Building the rows. """
    result = result.transpose().to_dict()
    rows = []
    for value in result.values():
        row = []
        for label in labels:
            refactored_keys = refactored_keys_index[label] if label in refactored_keys_index else {}
            sub_row = {}
            for refactored_key in refactored_keys:
                raw_key = refactored_keys_to_key_index[refactored_key]
                sub_row[raw_key] = value[refactored_key] if refactored_key in value else None
                if type(sub_row[raw_key]) is float and math.isnan(sub_row[raw_key]):
                    sub_row[raw_key] = None
                # sub_row[raw_key] = value[refactored_key] if refactored_key in value else 0
            row += [sub_row]
        rows += [row]
    return rows

def wrap_with_lazy_value(value, only_if_necessary=True, request_uuid=None):
    if value is None:
        return None
    if only_if_necessary and type(value).__name__ in ["int", "str", "float", "unicode"]:
        return value
    elif type(value) is dict and "timezone" in value:
        decoder = get_decoder()
        return decoder.desimplify(value)
        # return LazyDate(value, request_uuid)
    else:
        return LazyValue(value, request_uuid)

# def wrap_with_lazy_value(value, only_if_necessary=True, request_uuid=None):
#     return LazyValue(value, request_uuid)

def construct_rows(models, criterions, hints, session=None, request_uuid=None):

    """This function constructs the rows that corresponds to the current orm.
    :return: a list of row, according to sqlalchemy expectation
    """

    current_milli_time = lambda: int(round(time.time() * 1000))

    part1_starttime = current_milli_time()

    if request_uuid is None:
        request_uuid = uuid.uuid1()
    else:
        request_uuid = request_uuid

    labels = []
    columns = set([])
    rows = []

    model_set = extract_models(models)

    """ Get the fields of the join result """
    for selectable in model_set:
        labels += [find_table_name(selectable._model)]
        if selectable._attributes == "*":
            try:
                selected_attributes = selectable._model._sa_class_manager
            except:
                selected_attributes = selectable._model.class_._sa_class_manager
                pass
        else:
            selected_attributes = [selectable._attributes]

        for field in selected_attributes:
            attribute = None
            if has_attribute(models, "class_"):
                attribute = selectable._model.class_._sa_class_manager[field].__str__()
            elif has_attribute(models, "_sa_class_manager"):
                attribute = selectable._model._sa_class_manager[field].__str__()
            if attribute is not None:
                columns.add(attribute)
    part2_starttime = current_milli_time()

    """ Loading objects (from database) """
    list_results = []
    for selectable in model_set:
        tablename = find_table_name(selectable._model)
        authorized_secondary_indexes = get_attribute(selectable._model, "_secondary_indexes", [])
        # tablename = selectable._model.__tablename__
        # authorized_secondary_indexes = SECONDARY_INDEXES[tablename] if tablename in SECONDARY_INDEXES else []
        selected_hints = filter(lambda x: x.table_name == tablename and (x.attribute == "id" or x.attribute in authorized_secondary_indexes), hints)
        reduced_hints = map(lambda x:(x.attribute, x.value), selected_hints)
        objects = get_objects(tablename, request_uuid=request_uuid, skip_loading=False, hints=reduced_hints)
        list_results += [objects]
    part3_starttime = current_milli_time()

    """ Building tuples """
    tuples = building_tuples(list_results, labels, criterions, hints)
    part4_starttime = current_milli_time()

    """ Filtering tuples (cartesian product) """
    indexed_rows = {}
    for product in tuples:
        if len(product) > 0:
            row = KeyedTuple(product, labels=labels)
            row_index_key = "%s" % (str(row))

            if row_index_key in indexed_rows:
                continue

            all_criterions_satisfied = True

            for criterion in criterions:
                if not criterion.is_joining_expression and not criterion.evaluate(row):
                    all_criterions_satisfied = False
                    break
            if all_criterions_satisfied:
                indexed_rows[row_index_key] = True
                rows += [extract_sub_row(row, model_set)]
    part5_starttime = current_milli_time()
    deconverter = get_decoder(request_uuid=request_uuid)

    """ Reordering tuples (+ selecting attributes) """
    final_rows = []
    showable_selection = [x for x in models if (not x.is_hidden) or x._is_function]
    part6_starttime = current_milli_time()

    """ Selecting attributes """
    if all_selectable_are_functions(models):
        final_row = []
        for selection in showable_selection:
            value = selection._function._function(rows)
            final_row += [value]
        final_row = map(lambda x: deconverter.desimplify(x), final_row)
        return [final_row]
    else:
        for row in rows:
            final_row = []
            for selection in showable_selection:
                if selection._is_function:
                    value = selection._function._function(rows)
                    final_row += [value]
                else:
                    current_table_name = find_table_name(selection._model)
                    key = current_table_name
                    if not is_novabase(row) and has_attribute(row, key):
                        value = get_attribute(row, key)
                    else:
                        value = row
                    if value is not None:
                        if selection._attributes != "*":
                            final_row += [get_attribute(value, selection._attributes)]
                        else:
                            final_row += [value]
            final_row = map(lambda x: wrap_with_lazy_value(x, request_uuid=request_uuid), final_row)

            if len(showable_selection) == 1:
                final_rows += final_row
            else:
                final_rows += [final_row]
    part7_starttime = current_milli_time()

    query_information = """{"building_query": %s, "loading_objects": %s, "building_tuples": %s, "filtering_tuples": %s, "reordering_columns": %s, "selecting_attributes": %s, "description": "%s", "timestamp": %i}""" % (
        part2_starttime - part1_starttime,
        part3_starttime - part2_starttime,
        part4_starttime - part3_starttime,
        part5_starttime - part4_starttime,
        part6_starttime - part5_starttime,
        part7_starttime - part6_starttime,
        """{\\"models\\": \\"%s\\", \\"criterions\\": \\"%s\\"}""" % (models, criterions),
        current_milli_time()
    )

    logging.info(query_information)
    if file_logger_enabled:
        file_logger.info(query_information)

    return final_rows<commit_msg>try some tweaks;<commit_after>__author__ = 'jonathan'

import logging
import time
import uuid

import pandas as pd
from sqlalchemy.sql.expression import BinaryExpression
from sqlalchemy.util._collections import KeyedTuple

from lib.rome.core.dataformat import get_decoder
from lib.rome.core.lazy import LazyValue
from lib.rome.core.utils import get_objects, is_novabase

import math

file_logger_enabled = False
try:
    file_logger = logging.getLogger('rome_file_logger')
    hdlr = logging.FileHandler('/opt/logs/rome.log')
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    file_logger.addHandler(hdlr)
    file_logger.setLevel(logging.INFO)
    file_logger_enabled = True
except:
    pass

def all_selectable_are_functions(models):
    return all(x._is_function for x in [y for y in models if not y.is_hidden])

def has_attribute(obj, key):
    if type(obj) is dict:
        return key in obj
    else:
        return hasattr(obj, key)

def set_attribute(obj, key, value):
    if type(obj) is dict:
        obj[key] = value
    else:
        return setattr(obj, key, value)

def get_attribute(obj, key, default=None):
    if type(obj) is dict:
        return obj[key] if key in obj else default
    else:
        return getattr(obj, key, default)

def find_table_name(model):
    """This function return the name of the given model as a String. If the
    model cannot be identified, it returns "none".
    :param model: a model object candidate
    :return: the table name or "none" if the object cannot be identified
    """
    if has_attribute(model, "__tablename__"):
        return model.__tablename__
    if has_attribute(model, "table"):
        return model.table.name
    if has_attribute(model, "class_"):
        return model.class_.__tablename__
    if has_attribute(model, "clauses"):
        for clause in model.clauses:
            return find_table_name(clause)
    return "none"

def extract_models(l):
    already_processed = set()
    result = []
    for selectable in [x for x in l if not x._is_function]:
        if not selectable._model in already_processed:
            already_processed.add(selectable._model)
            result += [selectable]
    return result

def extract_sub_row(row, selectables):
    """Adapt a row result to the expectation of sqlalchemy.
    :param row: a list of python objects
    :param selectables: a list entity class
    :return: the response follows what is required by sqlalchemy (if len(model)==1, a single object is fine, in
    the other case, a KeyTuple where each sub object is associated with it's entity name
    """
    if len(selectables) > 1:
        labels = []
        for selectable in selectables:
            labels += [find_table_name(selectable._model)]
        product = []
        for label in labels:
            product = product + [get_attribute(row, label)]
        return KeyedTuple(product, labels=labels)
    else:
        model_name = find_table_name(selectables[0]._model)
        return get_attribute(row, model_name)

def intersect(b1, b2):
    return [val for val in b1 if val in b2]

def flatten(lis):
    """Given a list, possibly nested to any level, return it flattened."""
    new_lis = []
    for item in lis:
        if type(item) == type([]):
            new_lis.extend(flatten(item))
        else:
            new_lis.append(item)
    return new_lis

# def flatten(l):
#     return [item for sublist in l for item in sublist]

def extract_table_data(term):
    term_value = str(term)
    if "." in term_value:
        return {"table": term_value.split(".")[0], "column": term_value.split(".")[1]}
    else:
        return None

def extract_joining_criterion(exp):
    from lib.rome.core.expression.expression import BooleanExpression
    if type(exp) is BooleanExpression:
        return map(lambda x:extract_joining_criterion(x), exp.exps)
    elif type(exp) is BinaryExpression:
        return [[extract_table_data(exp.left)] + [extract_table_data(exp.right)]]
    else:
        return []

def extract_joining_criterion_from_relationship(rel, local_table):
    local_tabledata = {"table": local_table, "column": rel.local_fk_field}
    remote_tabledata = {"table": rel.remote_object_tablename, "column": rel.remote_object_field}
    return [local_tabledata, remote_tabledata]

def correct_boolean_int(expression_str):
    expression_str = expression_str.replace("___deleted == 0", "___deleted in [0, None]")
    return expression_str

def building_tuples(lists_results, labels, criterions, hints=[]):

    """ Build tuples (join operator in relational algebra). """

    """ Create the Dataframe indexes. """
    dataframes = []
    dataindex = {}
    substitution_index = {}
    normal_keys_index = {}
    refactored_keys_index = {}
    normal_keys_to_key_index = {}
    refactored_keys_to_key_index = {}
    index = 0
    for list_results in lists_results:
        label = labels[index]
        if len(list_results) == 0:
            continue
        keys = map(lambda x: x, list_results[0]) + ["created_at", "updated_at"]

        dataframe = pd.DataFrame(data=list_results, columns=keys)

        for value in keys:
            normal_key = "%s.%s" % (label, value)
            refactored_keys = "%s___%s" % (label, value)
            normal_keys_to_key_index[normal_key] = value
            refactored_keys_to_key_index[refactored_keys] = value
        normal_keys = map(lambda x: "%s.%s" % (label, x), keys)
        normal_keys_index[label] = normal_keys
        refactored_keys = map(lambda x: "%s___%s" % (label, x), keys)
        refactored_keys_index[label] = refactored_keys
        for (a, b) in zip(normal_keys, refactored_keys):
            substitution_index[a] = b
        dataframe.columns = refactored_keys
        dataframes += [dataframe]

        """ Index the dataframe and create a reverse index. """
        dataindex[label] = index
        index += 1

    """ Collecting joining expressions. """
    joining_pairs = []
    non_joining_criterions = []
    for criterion in criterions:
        _joining_pairs = criterion.extract_joining_pairs()
        _nonjoining_criterions = criterion.extract_nonjoining_criterions()

        joining_pairs += _joining_pairs
        non_joining_criterions += _nonjoining_criterions

    """ Construct the resulting rows. """
    if len(labels) > 1 and len(filter(lambda x: len(x) == 0, lists_results)) > 0:
        return []

    result = None
    processed_tables = []
    for joining_pair in joining_pairs:
        """ Preparing the tables that will be joined. """
        attribute_1 = joining_pair[0].strip()
        attribute_2 = joining_pair[1].strip()
        tablename_1 = attribute_1.split(".")[0]
        tablename_2 = attribute_2.split(".")[0]

        if tablename_1 not in dataindex or tablename_2 not in dataindex:
            return []
        index_1 = dataindex[tablename_1]
        index_2 = dataindex[tablename_2]
        dataframe_1 = dataframes[index_1] if not tablename_1 in processed_tables else result
        dataframe_2 = dataframes[index_2] if not tablename_2 in processed_tables else result

        refactored_attribute_1 = attribute_1.split(".")[0]+"___"+attribute_1.split(".")[1]
        refactored_attribute_2 = attribute_2.split(".")[0]+"___"+attribute_2.split(".")[1]

        """ Join the tables. """
        try:
            result = pd.merge(dataframe_1, dataframe_2, left_on=refactored_attribute_1, right_on=refactored_attribute_2)
        except KeyError:
            return []
        """ Update the history of processed tables. """
        processed_tables += [tablename_1, tablename_2]
        processed_tables = list(set(processed_tables))

    """ Filtering rows. """
    if result is None:
        if len(dataframes) == 0:
            return []
        result = dataframes[0]

    for non_joining_criterion in non_joining_criterions:
        expression_str = non_joining_criterion.raw_expression
        for value in substitution_index:
            if value in expression_str:
                corresponding_key = substitution_index[value]
                expression_str = expression_str.replace(value, corresponding_key)
        try:
            corrected_expression = correct_boolean_int(expression_str)
            result = result.query(corrected_expression)
        except:
            pass

    """ Building the rows. """
    result = result.transpose().to_dict()
    rows = []
    for value in result.values():
        row = []
        for label in labels:
            refactored_keys = refactored_keys_index[label] if label in refactored_keys_index else {}
            sub_row = {}
            for refactored_key in refactored_keys:
                raw_key = refactored_keys_to_key_index[refactored_key]
                sub_row[raw_key] = value[refactored_key] if refactored_key in value else None
                if type(sub_row[raw_key]) is float and math.isnan(sub_row[raw_key]):
                    sub_row[raw_key] = None
                # sub_row[raw_key] = value[refactored_key] if refactored_key in value else 0
            row += [sub_row]
        rows += [row]
    return rows

def wrap_with_lazy_value(value, only_if_necessary=True, request_uuid=None):
    if value is None:
        return None
    if only_if_necessary and type(value).__name__ in ["int", "str", "float", "unicode"]:
        return value
    elif type(value) is dict and "timezone" in value:
        decoder = get_decoder()
        return decoder.desimplify(value)
        # return LazyDate(value, request_uuid)
    else:
        return LazyValue(value, request_uuid)

# def wrap_with_lazy_value(value, only_if_necessary=True, request_uuid=None):
#     return LazyValue(value, request_uuid)

def construct_rows(models, criterions, hints, session=None, request_uuid=None):

    """This function constructs the rows that corresponds to the current orm.
    :return: a list of row, according to sqlalchemy expectation
    """

    current_milli_time = lambda: int(round(time.time() * 1000))

    part1_starttime = current_milli_time()

    if request_uuid is None:
        request_uuid = uuid.uuid1()
    else:
        request_uuid = request_uuid

    labels = []
    columns = set([])
    rows = []

    model_set = extract_models(models)

    """ Get the fields of the join result """
    for selectable in model_set:
        labels += [find_table_name(selectable._model)]
        if selectable._attributes == "*":
            try:
                selected_attributes = selectable._model._sa_class_manager
            except:
                selected_attributes = selectable._model.class_._sa_class_manager
                pass
        else:
            selected_attributes = [selectable._attributes]

        for field in selected_attributes:
            attribute = None
            if has_attribute(models, "class_"):
                attribute = selectable._model.class_._sa_class_manager[field].__str__()
            elif has_attribute(models, "_sa_class_manager"):
                attribute = selectable._model._sa_class_manager[field].__str__()
            if attribute is not None:
                columns.add(attribute)
    part2_starttime = current_milli_time()

    """ Loading objects (from database) """
    list_results = []
    for selectable in model_set:
        tablename = find_table_name(selectable._model)
        authorized_secondary_indexes = get_attribute(selectable._model, "_secondary_indexes", [])
        # tablename = selectable._model.__tablename__
        # authorized_secondary_indexes = SECONDARY_INDEXES[tablename] if tablename in SECONDARY_INDEXES else []
        selected_hints = filter(lambda x: x.table_name == tablename and (x.attribute == "id" or x.attribute in authorized_secondary_indexes), hints)
        reduced_hints = map(lambda x:(x.attribute, x.value), selected_hints)
        objects = get_objects(tablename, request_uuid=request_uuid, skip_loading=False, hints=reduced_hints)
        list_results += [objects]
    part3_starttime = current_milli_time()

    """ Building tuples """
    tuples = building_tuples(list_results, labels, criterions, hints)
    part4_starttime = current_milli_time()

    """ Filtering tuples (cartesian product) """
    indexed_rows = {}
    for product in tuples:
        if len(product) > 0:
            row = KeyedTuple(product, labels=labels)
            row_index_key = "%s" % (str(row))

            if row_index_key in indexed_rows:
                continue

            all_criterions_satisfied = True

            for criterion in criterions:
                if not criterion.is_joining_expression and not criterion.evaluate(row):
                    all_criterions_satisfied = False
                    break
            if all_criterions_satisfied:
                indexed_rows[row_index_key] = True
                rows += [extract_sub_row(row, model_set)]
    part5_starttime = current_milli_time()
    deconverter = get_decoder(request_uuid=request_uuid)

    """ Reordering tuples (+ selecting attributes) """
    final_rows = []
    showable_selection = [x for x in models if (not x.is_hidden) or x._is_function]
    part6_starttime = current_milli_time()

    """ Selecting attributes """
    if all_selectable_are_functions(models):
        final_row = []
        for selection in showable_selection:
            value = selection._function._function(rows)
            final_row += [value]
        final_row = map(lambda x: deconverter.desimplify(x), final_row)
        return [final_row]
    else:
        for row in rows:
            final_row = []
            for selection in showable_selection:
                if selection._is_function:
                    value = selection._function._function(rows)
                    final_row += [value]
                else:
                    current_table_name = find_table_name(selection._model)
                    key = current_table_name
                    if not is_novabase(row) and has_attribute(row, key):
                        value = get_attribute(row, key)
                    else:
                        value = row
                    if value is not None:
                        if selection._attributes != "*":
                            final_row += [get_attribute(value, selection._attributes)]
                        else:
                            final_row += [value]
            final_row = map(lambda x: wrap_with_lazy_value(x, request_uuid=request_uuid), final_row)

            if len(showable_selection) == 1:
                final_rows += final_row
            else:
                final_rows += [final_row]
    part7_starttime = current_milli_time()

    query_information = """{"building_query": %s, "loading_objects": %s, "building_tuples": %s, "filtering_tuples": %s, "reordering_columns": %s, "selecting_attributes": %s, "description": "%s", "timestamp": %i}""" % (
        part2_starttime - part1_starttime,
        part3_starttime - part2_starttime,
        part4_starttime - part3_starttime,
        part5_starttime - part4_starttime,
        part6_starttime - part5_starttime,
        part7_starttime - part6_starttime,
        """{\\"models\\": \\"%s\\", \\"criterions\\": \\"%s\\"}""" % (models, criterions),
        current_milli_time()
    )

    logging.info(query_information)
    if file_logger_enabled:
        file_logger.info(query_information)

    return final_rows<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Performs some tests with pyBarcode.

All created bar codes are saved in the tests subdirectory.

You can use the index.html to watch them."""

from __future__ import unicode_literals, print_function

import sys
import os

import codecs
import unittest

from steenzout import barcode
from steenzout.barcode import get_barcode, get_barcode_class

try:
    from steenzout.barcode.writer import ImageWriter
except ImportError:
    ImageWriter = None  # lint:ok

__docformat__ = 'restructuredtext en'

PATH = os.path.dirname(os.path.abspath(__file__))
TESTPATH = os.path.join(PATH, '.')
HTMLFILE = os.path.join(TESTPATH, 'index.html')

HTML = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
    "http://www.w3.org/TR/html4/strict.dtd">

        <title>viivakoodi {version} Test</title>
        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />

viivakoodi {version} Tests

        {body}

"""

OBJECTS = ('

{name}

\n'
           '<object data="{filename}" type="image/svg+xml">\n'
           '<param name="src" value="{filename}" /></object>')

IMAGES = ('
As PNG-Image

\n'
          '
\n')

NO_PIL = '
PIL was not found. No PNG-Image created.

\n'

TESTCODES = (
    ('ean8', '40267708'),
    ('ean13', '5901234123457'),
    ('upca', '36000291453'),
    ('jan', '4901234567894'),
    ('isbn10', '3-12-517154-7'),
    ('isbn13', '978-3-16-148410-0'),
    ('issn', '1144875X'),
    ('code39', 'Example Code 39'),
    ('pzn', '487780'),
    ('code128', 'Example Code 128 998866'),
)

def test():
    if not os.path.isdir(TESTPATH):
        try:
            os.mkdir(TESTPATH)
        except OSError as e:
            print('Test not run.')
            print('Error:', e)
            sys.exit(1)
    objects = []

    def append(x, y):
        objects.append(OBJECTS.format(filename=x, name=y))

    def append_img(x, y):
        objects.append(IMAGES.format(filename=x, name=y))

    options = dict(module_width=0.495, module_height=25.0)
    for codename, code in TESTCODES:
        bcode = get_barcode(codename, code)
        if codename.startswith('i'):
            options['center_text'] = False
        else:
            options['center_text'] = True
        filename = bcode.save(os.path.join(TESTPATH, codename), options)
        print('Code: {0}, Input: {1}, Output: {2}'.format(
            bcode.name, code, bcode.get_fullcode()))
        append(filename, bcode.name)
        if ImageWriter is not None:
            bcodec = get_barcode_class(codename)
            bcode = bcodec(code, writer=ImageWriter())
            opts = dict(font_size=14, text_distance=1)
            if codename.startswith('i'):
                opts['center_text'] = False
            else:
                options['center_text'] = True
            filename = bcode.save(os.path.join(TESTPATH, codename), opts)
            append_img(filename, bcode.name)
        else:
            objects.append(NO_PIL)
    # Save htmlfile with all objects
    with codecs.open(HTMLFILE, 'w', encoding='utf-8') as f:
        obj = '\n'.join(objects)
        f.write(HTML.format(version=barcode.__version__, body=obj))

class TestBarcodeBuilds(unittest.TestCase):
    def test_ean8(self):
        ref = ('1010100011000110100100110101111010101000100'
               '100010011100101001000101')
        ean = get_barcode('ean8', '40267708')
        bc = ean.build()
        self.assertEqual(ref, bc[0])

if __name__ == '__main__':
    test()
    print('\nNow open {htmlfile} in your browser.'.format(htmlfile=HTMLFILE))
    unittest.main()
<commit_msg>cleanup original tests.<commit_after># -*- coding: utf-8 -*-
"""Performs some tests with pyBarcode.

All created bar codes are saved in the tests subdirectory.

You can use the index.html to watch them."""

from __future__ import unicode_literals, print_function

import sys
import os

import codecs

from steenzout import barcode
from steenzout.barcode import get_barcode, get_barcode_class

try:
    from steenzout.barcode.writer import ImageWriter
except ImportError:
    ImageWriter = None  # lint:ok

__docformat__ = 'restructuredtext en'

PATH = os.path.dirname(os.path.abspath(__file__))
TESTPATH = os.path.join(PATH, '.')
HTMLFILE = os.path.join(TESTPATH, 'index.html')

HTML = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
    "http://www.w3.org/TR/html4/strict.dtd">

        <title>viivakoodi {version} Test</title>
        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />

viivakoodi {version} Tests

        {body}

"""

OBJECTS = ('

{name}

\n'
           '<object data="{filename}" type="image/svg+xml">\n'
           '<param name="src" value="{filename}" /></object>')

IMAGES = ('
As PNG-Image

\n'
          '
\n')

NO_PIL = '
PIL was not found. No PNG-Image created.

\n'

TESTCODES = (
    ('ean8', '40267708'),
    ('ean13', '5901234123457'),
    ('upca', '360002914539'),
    ('jan', '4901234567894'),
    ('isbn10', '3-12-517154-7'),
    ('isbn13', '978-3-16-148410-0'),
    ('issn', '1144875X'),
    ('code39', 'Example Code 39'),
    ('pzn', '487780'),
    ('code128', 'Example Code 128 998866'),
)

def test():
    if not os.path.isdir(TESTPATH):
        try:
            os.mkdir(TESTPATH)
        except OSError as e:
            print('Test not run.')
            print('Error:', e)
            sys.exit(1)
    objects = []

    def append(x, y):
        objects.append(OBJECTS.format(filename=x, name=y))

    def append_img(x, y):
        objects.append(IMAGES.format(filename=x, name=y))

    options = dict(module_width=0.495, module_height=25.0)
    for codename, code in TESTCODES:
        bar_code = get_barcode(codename, code)
        if codename.startswith('i'):
            options['center_text'] = False
        else:
            options['center_text'] = True
        filename = bar_code.save(os.path.join(TESTPATH, codename), options)
        print('Code: %s, Input: %s, Output: %s' % (bar_code.name, code, bar_code.get_fullcode()))

        append(filename, bar_code.name)
        if ImageWriter is not None:
            code_class = get_barcode_class(codename)
            bar_code = code_class(code, writer=ImageWriter())
            opts = dict(font_size=14, text_distance=1)
            if codename.startswith('i'):
                opts['center_text'] = False
            else:
                options['center_text'] = True
            filename = bar_code.save(os.path.join(TESTPATH, codename), opts)
            append_img(filename, bar_code.name)
        else:
            objects.append(NO_PIL)

    # Save htmlfile with all objects
    with codecs.open(HTMLFILE, 'w', encoding='utf-8') as f:
        obj = '\n'.join(objects)
        f.write(HTML.format(version=barcode.__version__, body=obj))
<|endoftext|>


<commit_before># coding=utf-8
from __future__ import unicode_literals, print_function

import collections
import json
import os
import unittest

import brevity

with open('tests.json') as f:
    TESTS = json.load(f)

class BrevityTest(unittest.TestCase):

    def test_tokenize_ignore_html(self):
        text = 'this should not be linkified'
        self.assertEqual([brevity.Token(tag='text', content=text)],
                         brevity.tokenize(text))

    def test_tokenize_ignore_email(self):
        text = 'this should not.be@linkified.com'
        self.assertEqual([brevity.Token(tag='text', content=text)],
                         brevity.tokenize(text))

    def test_tokenize_ignore_cc_tlds(self):
        text = 'Despite names,\nind.ie&indie.vc are NOT #indieweb @indiewebcamp\nindiewebcamp.com/2014-review#Indie_Term_Re-use\n@iainspad @sashtown @thomatronic'
        self.assertEqual([
            brevity.Token(tag='text', content='Despite names,\nind.ie&indie.vc are NOT #indieweb @indiewebcamp\n'),
            brevity.Token(tag='link', content='indiewebcamp.com/2014-review#Indie_Term_Re-use'),
            brevity.Token(tag='text', content='\n@iainspad @sashtown @thomatronic')
        ], brevity.tokenize(text, skip_bare_cc_tlds=True))

    def test_shorten(self):
        for testcase in TESTS['shorten']:
            params = {
                k: testcase[k] for k in (
                    'text', 'permalink', 'permashortlink', 'permashortcitation',
                    'target_length', 'link_length', 'format_as_title',
                )
                if k in testcase
            }
            result = brevity.shorten(**params)
            expected = testcase['expected']
            self.assertEqual(expected, result)

    def test_autolink(self):
        for testcase in TESTS['autolink']:
            self.assertEqual(
                testcase['expected'], brevity.autolink(testcase['text']))
<commit_msg>python 2.6 compatibility<commit_after># coding=utf-8
from __future__ import unicode_literals, print_function

import collections
import json
import os
import unittest

import brevity

with open('tests.json') as f:
    TESTS = json.load(f)

class BrevityTest(unittest.TestCase):

    def test_tokenize_ignore_html(self):
        text = 'this should not be linkified'
        self.assertEqual([brevity.Token(tag='text', content=text)],
                         brevity.tokenize(text))

    def test_tokenize_ignore_email(self):
        text = 'this should not.be@linkified.com'
        self.assertEqual([brevity.Token(tag='text', content=text)],
                         brevity.tokenize(text))

    def test_tokenize_ignore_cc_tlds(self):
        text = 'Despite names,\nind.ie&indie.vc are NOT #indieweb @indiewebcamp\nindiewebcamp.com/2014-review#Indie_Term_Re-use\n@iainspad @sashtown @thomatronic'
        self.assertEqual([
            brevity.Token(tag='text', content='Despite names,\nind.ie&indie.vc are NOT #indieweb @indiewebcamp\n'),
            brevity.Token(tag='link', content='indiewebcamp.com/2014-review#Indie_Term_Re-use'),
            brevity.Token(tag='text', content='\n@iainspad @sashtown @thomatronic')
        ], brevity.tokenize(text, skip_bare_cc_tlds=True))

    def test_shorten(self):
        for testcase in TESTS['shorten']:
            params = dict([
                (k, testcase[k]) for k in (
                    'text', 'permalink', 'permashortlink', 'permashortcitation',
                    'target_length', 'link_length', 'format_as_title',
                )
                if k in testcase])
            result = brevity.shorten(**params)
            expected = testcase['expected']
            self.assertEqual(expected, result)

    def test_autolink(self):
        for testcase in TESTS['autolink']:
            self.assertEqual(
                testcase['expected'], brevity.autolink(testcase['text']))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import (
    absolute_import,   # Python 2.5+
    print_function,    # Python 2.6+
    unicode_literals,  # Python 2.6+
    with_statement,    # Python 2.5+
)

import unittest
import tempfile
import os
import shutil

from route_ctl import actions

class TestBackup(unittest.TestCase):
    """Test backup file creation."""

    def setUp(self):
        self.dir = tempfile.mkdtemp()
        chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
        self.path = os.path.join(self.dir, 'test-file.txt')
        self.expected_content = chars
        with open(self.path, 'w') as fw:
            fw.write(self.expected_content)
        self.suffix = '.backup'
        self.backup = self.path + self.suffix

    def tearDown(self):
        shutil.rmtree(self.dir)

    def test_backup_existing_file(self):
        """Existing files should be backed up."""
        self.assertTrue(os.path.isfile(self.path))
        self.assertFalse(os.path.isfile(self.backup))
        actions.do_backup(self.path, suffix=self.suffix)
        self.assertTrue(os.path.isfile(self.path))
        self.assertTrue(os.path.isfile(self.backup))
        with open(self.backup) as fr:
            content = fr.read()
            self.assertEqual(content, self.expected_content)

    def test_backup_non_existing_file(self):
        """Non-existing files should not be backed up."""
        _path = os.path.join(self.dir, 'no-such-file.txt')
        _backup = _path + self.suffix
        self.assertFalse(os.path.isfile(_path))
        self.assertFalse(os.path.isfile(_backup))
        actions.do_backup(_path, suffix=self.suffix)
        self.assertFalse(os.path.isfile(_path))
        self.assertFalse(os.path.isfile(_backup))

    def test_backup_overwrite(self):
        """Last backup should be overwritten."""
        new_content = 'SOME NEW CONTENT'
        actions.do_backup(self.path, suffix=self.suffix)
        self.assertTrue(os.path.isfile(self.path))
        self.assertTrue(os.path.isfile(self.backup))
        with open(self.backup) as fr:
            content = fr.read()
            self.assertEqual(content, self.expected_content)
        with open(self.path, 'w') as fw:
            fw.write(new_content)
        actions.do_backup(self.path, suffix=self.suffix)
        self.assertTrue(os.path.isfile(self.backup))
        with open(self.backup) as fr:
            content = fr.read()
            self.assertEqual(content, new_content)
<commit_msg>Added dir backup test<commit_after># -*- coding: utf-8 -*-

from __future__ import (
    absolute_import,   # Python 2.5+
    print_function,    # Python 2.6+
    unicode_literals,  # Python 2.6+
    with_statement,    # Python 2.5+
)

import unittest
import tempfile
import os
import shutil

from route_ctl import actions

class TestBackup(unittest.TestCase):
    """Test backup file creation."""

    def setUp(self):
        self.dir = tempfile.mkdtemp()
        chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
        self.path = os.path.join(self.dir, 'test-file.txt')
        self.expected_content = chars
        with open(self.path, 'w') as fw:
            fw.write(self.expected_content)
        self.suffix = '.backup'
        self.backup = self.path + self.suffix

    def tearDown(self):
        shutil.rmtree(self.dir)

    def test_backup_existing_file(self):
        """Existing files should be backed up."""
        self.assertTrue(os.path.isfile(self.path))
        self.assertFalse(os.path.isfile(self.backup))
        actions.do_backup(self.path, suffix=self.suffix)
        self.assertTrue(os.path.isfile(self.path))
        self.assertTrue(os.path.isfile(self.backup))
        with open(self.backup) as fr:
            content = fr.read()
            self.assertEqual(content, self.expected_content)

    def test_backup_non_existing_file(self):
        """Non-existing files should not be backed up."""
        _path = os.path.join(self.dir, 'no-such-file.txt')
        _backup = _path + self.suffix
        self.assertFalse(os.path.isfile(_path))
        self.assertFalse(os.path.isfile(_backup))
        actions.do_backup(_path, suffix=self.suffix)
        self.assertFalse(os.path.isfile(_path))
        self.assertFalse(os.path.isfile(_backup))

    def test_backup_dir(self):
        """Directories should not be backed up."""
        _path = self.dir
        _backup = _path + self.suffix
        self.assertTrue(os.path.isdir(_path))
        self.assertFalse(os.path.isdir(_backup))
        actions.do_backup(_path, suffix=self.suffix)
        self.assertTrue(os.path.isdir(_path))
        self.assertFalse(os.path.isdir(_backup))

    def test_backup_overwrite(self):
        """Last backup should be overwritten."""
        new_content = 'SOME NEW CONTENT'
        actions.do_backup(self.path, suffix=self.suffix)
        self.assertTrue(os.path.isfile(self.path))
        self.assertTrue(os.path.isfile(self.backup))
        with open(self.backup) as fr:
            content = fr.read()
            self.assertEqual(content, self.expected_content)
        with open(self.path, 'w') as fw:
            fw.write(new_content)
        actions.do_backup(self.path, suffix=self.suffix)
        self.assertTrue(os.path.isfile(self.backup))
        with open(self.backup) as fr:
            content = fr.read()
            self.assertEqual(content, new_content)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Tests for Chat Relater's Analyzer
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Run with py.test_.

.. _py.test: http://codespeak.net/py/dist/test.html

:date: 2007-07-05
:copyright: 2007 Jochen Kupperschmidt
:license: MIT, see LICENSE for details.
"""

from __future__ import with_statement
import os
from tempfile import mktemp

from py.test import raises

from chatrelater.analyze import (parse_logfile, relate_nicknames,
    compress_relations, save_data, load_data)

def test_parse_logfile():
    lines = (
        '<John> one two',
        '* some action',
        '<Jane> threefourfive',
        'Someone- A message from someone!',
        '<Mary> foobar')
    expected_nicknames = set(['John', 'Jane', 'Mary'])
    expected_loglines = [
        ('John', 'one two'),
        ('Jane', 'threefourfive'),
        ('Mary', 'foobar')]
    assert parse_logfile(lines) == (expected_nicknames, expected_loglines)

def test_relate_nicknames():
    nicknames = set(['John', 'Jane', 'Mary'])
    loglines = [
        ('John', 'heyho'),
        ('John', 'Jane, sup?'),
        ('Jane', 'John: I am fine, thanks'),
        ('Mary', 'John?'),
        ('Mary', 'John!'),
        ('Jane', 'John seems to have disappeared...')]
    expected_relations = [
        ('John', 'Jane'),
        ('Jane', 'John'),
        ('Mary', 'John'),
        ('Mary', 'John'),
        ('Jane', 'John')]
    result = list(relate_nicknames(nicknames, loglines))
    assert result == expected_relations

def test_compress_relations():
    relations = (
        ('one', 'two'),
        ('one', 'three'),
        ('two', 'one'),
        ('one', 'three'),
        ('three', 'one'))
    expected_compressed = (
        ('one', 'two', 1),
        ('one', 'three', 2),
        ('two', 'one', 1),
        ('three', 'one', 1))
    expected_compressed_unified = (
        ('one', 'two', 2),
        ('one', 'three', 3))
    assert set(compress_relations(relations, False)) == set(expected_compressed)
    assert set(compress_relations(relations, True)) == set(expected_compressed_unified)

class LoadSaveContext(object):
    """A context to create a temporary filename and remove the file afterwards."""

    def __init__(self, data):
        self.data = data

    def __enter__(self):
        self.filename = mktemp()
        save_data(self.data, self.filename)
        return self.filename

    def __exit__(self, *exc_info):
        os.unlink(self.filename)

def test_load_save_data():
    """Test saving data and then loading it again.

    Also, KeyErrors are expected if required keys are missing.
    """
    def assert_required_keys(data, should_raise=False):
        with LoadSaveContext(data) as filename:
            if should_raise:
                raises(KeyError, 'load_data(filename)')
            else:
                load_data(filename)

    # Assure that missing keys raise a ``KeyError``.
    required_keys = ('nicknames', 'relations', 'directed')
    for key in required_keys:
        keys = list(required_keys)
        keys.remove(key)
        assert_required_keys(dict.fromkeys(keys), True)
    assert_required_keys({'nicknames': None}, True)
    assert_required_keys(dict.fromkeys(required_keys), False)

    # Assure the exact data saved will be loaded afterwards.
    data = dict(nicknames=123, relations=456, directed=True)
    with LoadSaveContext(data) as filename:
        assert load_data(filename) == (123, 456, True)
<commit_msg>Reformatted test data.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Tests for Chat Relater's Analyzer
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Run with py.test_.

.. _py.test: http://codespeak.net/py/dist/test.html

:date: 2007-07-05
:copyright: 2007 Jochen Kupperschmidt
:license: MIT, see LICENSE for details.
"""

from __future__ import with_statement
import os
from tempfile import mktemp

from py.test import raises

from chatrelater.analyze import (parse_logfile, relate_nicknames,
    compress_relations, save_data, load_data)

def test_parse_logfile():
    lines = (
        '<John> one two',
        '* some action',
        '<Jane> threefourfive',
        'Someone- A message from someone!',
        '<Mary> foobar',
    )
    expected_nicknames = set(['John', 'Jane', 'Mary'])
    expected_loglines = [
        ('John', 'one two'),
        ('Jane', 'threefourfive'),
        ('Mary', 'foobar'),
    ]
    assert parse_logfile(lines) == (expected_nicknames, expected_loglines)

def test_relate_nicknames():
    nicknames = set(['John', 'Jane', 'Mary'])
    loglines = [
        ('John', 'heyho'),
        ('John', 'Jane, sup?'),
        ('Jane', 'John: I am fine, thanks'),
        ('Mary', 'John?'),
        ('Mary', 'John!'),
        ('Jane', 'John seems to have disappeared...'),
    ]
    expected_relations = [
        ('John', 'Jane'),
        ('Jane', 'John'),
        ('Mary', 'John'),
        ('Mary', 'John'),
        ('Jane', 'John'),
    ]
    result = list(relate_nicknames(nicknames, loglines))
    assert result == expected_relations

def test_compress_relations():
    relations = (
        ('one',  'two'  ),
        ('one',  'three'),
        ('two',  'one'  ),
        ('one',  'three'),
        ('three', 'one' ),
    )
    expected_compressed = (
        ('one',   'two',   1),
        ('one',   'three', 2),
        ('two',   'one',   1),
        ('three', 'one',   1),
    )
    expected_compressed_unified = (
        ('one',   'two',   2),
        ('one',   'three', 3),
    )
    assert set(compress_relations(relations, False)) == set(expected_compressed)
    assert set(compress_relations(relations, True)) == set(expected_compressed_unified)

class LoadSaveContext(object):
    """A context to create a temporary filename and remove the file afterwards."""

    def __init__(self, data):
        self.data = data

    def __enter__(self):
        self.filename = mktemp()
        save_data(self.data, self.filename)
        return self.filename

    def __exit__(self, *exc_info):
        os.unlink(self.filename)

def test_load_save_data():
    """Test saving data and then loading it again.

    Also, KeyErrors are expected if required keys are missing.
    """
    def assert_required_keys(data, should_raise=False):
        with LoadSaveContext(data) as filename:
            if should_raise:
                raises(KeyError, 'load_data(filename)')
            else:
                load_data(filename)

    # Assure that missing keys raise a ``KeyError``.
    required_keys = ('nicknames', 'relations', 'directed')
    for key in required_keys:
        keys = list(required_keys)
        keys.remove(key)
        assert_required_keys(dict.fromkeys(keys), True)
    assert_required_keys({'nicknames': None}, True)
    assert_required_keys(dict.fromkeys(required_keys), False)

    # Assure the exact data saved will be loaded afterwards.
    data = dict(nicknames=123, relations=456, directed=True)
    with LoadSaveContext(data) as filename:
        assert load_data(filename) == (123, 456, True)
<|endoftext|>


<commit_before>import datetime

from freezegun import freeze_time

try:
    import asyncio
except ImportError:
    asyncio = False

if asyncio:
    def test_time_freeze_coroutine():
        @freeze_time('1970-01-01')
        async def frozen_coroutine():
            assert datetime.date.today() == datetime.date(1970, 1, 1)

        asyncio.get_event_loop().run_until_complete(frozen_coroutine())
<commit_msg>fixed syntax error due to async def on older pythons<commit_after>import datetime
from textwrap import dedent

from freezegun import freeze_time

try:
    import asyncio
except ImportError:
    asyncio = False

if asyncio:
    def test_time_freeze_coroutine():
        @asyncio.coroutine
        @freeze_time('1970-01-01')
        def frozen_coroutine():
            assert datetime.date.today() == datetime.date(1970, 1, 1)

        asyncio.get_event_loop().run_until_complete(frozen_coroutine())

    exec(dedent('''
    def test_time_freeze_async_def():
        @freeze_time('1970-01-01')
        async def frozen_coroutine():
            assert datetime.date.today() == datetime.date(1970, 1, 1)
        asyncio.get_event_loop().run_until_complete(frozen_coroutine())
    '''))<|endoftext|>


<commit_before>from typing import List, Tuple
import pytest
import eagerpy as ep

import foolbox as fbn
import foolbox.attacks as fa

L2 = fbn.types.L2
Linf = fbn.types.Linf

# attack, attack_uses_grad, requires_real_model
attacks: List[Tuple[fbn.Attack, bool, bool]] = [
    (fa.DDNAttack(), True, False),
    (fa.DDNAttack(rescale=True), True, False),
    (fa.InversionAttack(), False, False),
    (fa.L2ContrastReductionAttack(L2(100.0)), False, False),
    (fa.BinarySearchContrastReductionAttack(binary_search_steps=15), False, False),
    (fa.LinearSearchContrastReductionAttack(steps=20), False, False),
    (fa.L2CarliniWagnerAttack(binary_search_steps=3, steps=20), True, False),
    (fa.EADAttack(binary_search_steps=3, steps=20), True, False),
    (fa.EADAttack(binary_search_steps=3, steps=20, decision_rule="L1"), True, False),
    (fa.NewtonFoolAttack(steps=20), True, False),
    (fa.L2ContrastReductionAttack(L2(100.0)).repeat(3), False, False),
    (fa.VirtualAdversarialAttack(iterations=50, xi=1, epsilon=10), True, False),
    (fa.L2BasicIterativeAttack(L2(100.0), stepsize=5.0, steps=10), True, False),
    (fa.LinfBasicIterativeAttack(Linf(1.0), stepsize=5.0, steps=10), True, False),
    (
        fa.ProjectedGradientDescentAttack(Linf(1.0), stepsize=5.0, steps=10),
        True,
        False,
    ),
    (fa.L2FastGradientAttack(L2(100.0)), True, False),
    (fa.LinfFastGradientAttack(Linf(100.0)), True, False),
    (fa.GaussianBlurAttack(steps=10), True, True),
    (fa.L2DeepFoolAttack(steps=50), True, False),
    (fa.LinfDeepFoolAttack(steps=50), True, False),
]

@pytest.mark.parametrize("attack_and_grad", attacks)
def test_untargeted_attacks(
    fmodel_and_data_ext: Tuple[Tuple[fbn.Model, ep.Tensor, ep.Tensor], bool],
    attack_and_grad: Tuple[fbn.Attack, bool, bool],
) -> None:

    attack, attack_uses_grad, requires_real_model = attack_and_grad
    (fmodel, x, y), real = fmodel_and_data_ext
    if requires_real_model and not real:
        pytest.skip()

    if isinstance(x, ep.NumPyTensor) and attack_uses_grad:
        pytest.skip()

    x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
    fmodel = fmodel.transform_bounds((0, 1))

    advs = attack(fmodel, x, y)
    assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y)

targeted_attacks: List[Tuple[fbn.Attack, bool, bool]] = [
    (fa.L2CarliniWagnerAttack(binary_search_steps=3, steps=20), True, False),
    (fa.DDNAttack(), True, False),
    # TODO: targeted EADAttack currently fails repeatedly on MobileNetv2
    # (
    #     fa.EADAttack(
    #         binary_search_steps=3, steps=20, abort_early=True, regularization=0
    #     ),
    #     True,
    #     False,
    # ),
]

@pytest.mark.parametrize("attack_and_grad", targeted_attacks)
def test_targeted_attacks(
    fmodel_and_data_ext: Tuple[Tuple[fbn.Model, ep.Tensor, ep.Tensor], bool],
    attack_and_grad: Tuple[fbn.Attack, bool, bool],
) -> None:

    attack, attack_uses_grad, requires_real_model = attack_and_grad
    (fmodel, x, y), real = fmodel_and_data_ext
    if requires_real_model and not real:
        pytest.skip()

    if isinstance(x, ep.NumPyTensor) and attack_uses_grad:
        pytest.skip()

    x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
    fmodel = fmodel.transform_bounds((0, 1))

    num_classes = fmodel(x).shape[-1]
    target_classes = (y + 1) % num_classes
    criterion = fbn.TargetedMisclassification(target_classes)
    advs = attack(fmodel, x, criterion)

    adv_before_attack = criterion(x, fmodel(x))
    adv_after_attack = criterion(advs, fmodel(advs))
    assert adv_after_attack.sum().item() > adv_before_attack.sum().item()
<commit_msg>added test for DeepFool with crossentropy<commit_after>from typing import List, Tuple
import pytest
import eagerpy as ep

import foolbox as fbn
import foolbox.attacks as fa

L2 = fbn.types.L2
Linf = fbn.types.Linf

# attack, attack_uses_grad, requires_real_model
attacks: List[Tuple[fbn.Attack, bool, bool]] = [
    (fa.DDNAttack(), True, False),
    (fa.DDNAttack(rescale=True), True, False),
    (fa.InversionAttack(), False, False),
    (fa.L2ContrastReductionAttack(L2(100.0)), False, False),
    (fa.BinarySearchContrastReductionAttack(binary_search_steps=15), False, False),
    (fa.LinearSearchContrastReductionAttack(steps=20), False, False),
    (fa.L2CarliniWagnerAttack(binary_search_steps=3, steps=20), True, False),
    (fa.EADAttack(binary_search_steps=3, steps=20), True, False),
    (fa.EADAttack(binary_search_steps=3, steps=20, decision_rule="L1"), True, False),
    (fa.NewtonFoolAttack(steps=20), True, False),
    (fa.L2ContrastReductionAttack(L2(100.0)).repeat(3), False, False),
    (fa.VirtualAdversarialAttack(iterations=50, xi=1, epsilon=10), True, False),
    (fa.L2BasicIterativeAttack(L2(100.0), stepsize=5.0, steps=10), True, False),
    (fa.LinfBasicIterativeAttack(Linf(1.0), stepsize=5.0, steps=10), True, False),
    (
        fa.ProjectedGradientDescentAttack(Linf(1.0), stepsize=5.0, steps=10),
        True,
        False,
    ),
    (fa.L2FastGradientAttack(L2(100.0)), True, False),
    (fa.LinfFastGradientAttack(Linf(100.0)), True, False),
    (fa.GaussianBlurAttack(steps=10), True, True),
    (fa.L2DeepFoolAttack(steps=50, loss="logits"), True, False),
    (fa.L2DeepFoolAttack(steps=50, loss="crossentropy"), True, False),
    (fa.LinfDeepFoolAttack(steps=50), True, False),
]

@pytest.mark.parametrize("attack_and_grad", attacks)
def test_untargeted_attacks(
    fmodel_and_data_ext: Tuple[Tuple[fbn.Model, ep.Tensor, ep.Tensor], bool],
    attack_and_grad: Tuple[fbn.Attack, bool, bool],
) -> None:

    attack, attack_uses_grad, requires_real_model = attack_and_grad
    (fmodel, x, y), real = fmodel_and_data_ext
    if requires_real_model and not real:
        pytest.skip()

    if isinstance(x, ep.NumPyTensor) and attack_uses_grad:
        pytest.skip()

    x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
    fmodel = fmodel.transform_bounds((0, 1))

    advs = attack(fmodel, x, y)
    assert fbn.accuracy(fmodel, advs, y) < fbn.accuracy(fmodel, x, y)

targeted_attacks: List[Tuple[fbn.Attack, bool, bool]] = [
    (fa.L2CarliniWagnerAttack(binary_search_steps=3, steps=20), True, False),
    (fa.DDNAttack(), True, False),
    # TODO: targeted EADAttack currently fails repeatedly on MobileNetv2
    # (
    #     fa.EADAttack(
    #         binary_search_steps=3, steps=20, abort_early=True, regularization=0
    #     ),
    #     True,
    #     False,
    # ),
]

@pytest.mark.parametrize("attack_and_grad", targeted_attacks)
def test_targeted_attacks(
    fmodel_and_data_ext: Tuple[Tuple[fbn.Model, ep.Tensor, ep.Tensor], bool],
    attack_and_grad: Tuple[fbn.Attack, bool, bool],
) -> None:

    attack, attack_uses_grad, requires_real_model = attack_and_grad
    (fmodel, x, y), real = fmodel_and_data_ext
    if requires_real_model and not real:
        pytest.skip()

    if isinstance(x, ep.NumPyTensor) and attack_uses_grad:
        pytest.skip()

    x = (x - fmodel.bounds.lower) / (fmodel.bounds.upper - fmodel.bounds.lower)
    fmodel = fmodel.transform_bounds((0, 1))

    num_classes = fmodel(x).shape[-1]
    target_classes = (y + 1) % num_classes
    criterion = fbn.TargetedMisclassification(target_classes)
    advs = attack(fmodel, x, criterion)

    adv_before_attack = criterion(x, fmodel(x))
    adv_after_attack = criterion(advs, fmodel(advs))
    assert adv_after_attack.sum().item() > adv_before_attack.sum().item()
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
#
# Copyright (C) 2012-2013 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import unicode_literals
import unittest

from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity,
                                    Tag, Template, Text, Wikilink)
from mwparserfromhell.nodes.extras import Attribute, Parameter
from mwparserfromhell.parser import tokens
from mwparserfromhell.parser.builder import Builder
from mwparserfromhell.smart_list import SmartList
from mwparserfromhell.wikicode import Wikicode

from ._test_tree_equality import TreeEqualityTestCase

wrap = lambda L: Wikicode(SmartList(L))

class TestBuilder(TreeEqualityTestCase):
    """Tests for the builder, which turns tokens into Wikicode objects."""

    def setUp(self):
        self.builder = Builder()

    def test_text(self):
        """tests for building Text nodes"""
        tests = [
            ([tokens.Text(text="foobar")], wrap([Text("foobar")])),
            ([tokens.Text(text="fóóbar")], wrap([Text("fóóbar")])),
            ([tokens.Text(text="spam"), tokens.Text(text="eggs")],
             wrap([Text("spam"), Text("eggs")])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_template(self):
        """tests for building Template nodes"""
        tests = [
            ([tokens.TemplateOpen(), tokens.Text(text="foobar"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foobar")]))])),

            ([tokens.TemplateOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.TemplateClose()],
             wrap([Template(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("1")]), wrap([Text("bar")]),
                           showkey=False)])])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateParamEquals(), tokens.Text(text="baz"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("bar")]), wrap([Text("baz")]))])])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateParamEquals(), tokens.Text(text="baz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="biz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="buzz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="3"),
              tokens.TemplateParamEquals(), tokens.Text(text="buff"),
              tokens.TemplateParamSeparator(), tokens.Text(text="baff"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("bar")]), wrap([Text("baz")])),
                 Parameter(wrap([Text("1")]), wrap([Text("biz")]),
                           showkey=False),
                 Parameter(wrap([Text("2")]), wrap([Text("buzz")]),
                           showkey=False),
                 Parameter(wrap([Text("3")]), wrap([Text("buff")])),
                 Parameter(wrap([Text("3")]), wrap([Text("baff")]),
                           showkey=False)])])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_argument(self):
        """tests for building Argument nodes"""
        tests = [
            ([tokens.ArgumentOpen(), tokens.Text(text="foobar"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foobar")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="foo"),
              tokens.ArgumentSeparator(), tokens.Text(text="bar"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foo")]), wrap([Text("bar")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="foo"),
              tokens.Text(text="bar"), tokens.ArgumentSeparator(),
              tokens.Text(text="baz"), tokens.Text(text="biz"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foo"), Text("bar")]),
                            wrap([Text("baz"), Text("biz")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_wikilink(self):
        """tests for building Wikilink nodes"""
        tests = [
            ([tokens.WikilinkOpen(), tokens.Text(text="foobar"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foobar")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="foo"),
              tokens.WikilinkSeparator(), tokens.Text(text="bar"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foo")]), wrap([Text("bar")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="foo"),
              tokens.Text(text="bar"), tokens.WikilinkSeparator(),
              tokens.Text(text="baz"), tokens.Text(text="biz"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foo"), Text("bar")]),
                            wrap([Text("baz"), Text("biz")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_html_entity(self):
        """tests for building HTMLEntity nodes"""
        tests = [
            ([tokens.HTMLEntityStart(), tokens.Text(text="nbsp"),
              tokens.HTMLEntityEnd()],
             wrap([HTMLEntity("nbsp", named=True, hexadecimal=False)])),

            ([tokens.HTMLEntityStart(), tokens.HTMLEntityNumeric(),
              tokens.Text(text="107"), tokens.HTMLEntityEnd()],
             wrap([HTMLEntity("107", named=False, hexadecimal=False)])),

            ([tokens.HTMLEntityStart(), tokens.HTMLEntityNumeric(),
              tokens.HTMLEntityHex(char="X"), tokens.Text(text="6B"),
              tokens.HTMLEntityEnd()],
             wrap([HTMLEntity("6B", named=False, hexadecimal=True,
                              hex_char="X")])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_heading(self):
        """tests for building Heading nodes"""
        tests = [
            ([tokens.HeadingStart(level=2), tokens.Text(text="foobar"),
              tokens.HeadingEnd()],
             wrap([Heading(wrap([Text("foobar")]), 2)])),

            ([tokens.HeadingStart(level=4), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.HeadingEnd()],
             wrap([Heading(wrap([Text("spam"), Text("eggs")]), 4)])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_comment(self):
        """tests for building Comment nodes"""
        tests = [
            ([tokens.CommentStart(), tokens.Text(text="foobar"),
              tokens.CommentEnd()],
             wrap([Comment(wrap([Text("foobar")]))])),

            ([tokens.CommentStart(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.CommentEnd()],
             wrap([Comment(wrap([Text("spam"), Text("eggs")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    @unittest.skip("holding this until feature/html_tags is ready")
    def test_tag(self):
        """tests for building Tag nodes"""
        pass

    def test_integration(self):
        """a test for building a combination of templates together"""
        test = [tokens.TemplateOpen(), tokens.TemplateOpen(),
                tokens.TemplateOpen(), tokens.TemplateOpen(),
                tokens.Text(text="foo"), tokens.TemplateClose(),
                tokens.Text(text="bar"), tokens.TemplateParamSeparator(),
                tokens.Text(text="baz"), tokens.TemplateParamEquals(),
                tokens.Text(text="biz"), tokens.TemplateClose(),
                tokens.Text(text="buzz"), tokens.TemplateClose(),
                tokens.Text(text="usr"), tokens.TemplateParamSeparator(),
                tokens.TemplateOpen(), tokens.Text(text="bin"),
                tokens.TemplateClose(), tokens.TemplateClose()]
        valid = wrap(
            [Template(wrap([Template(wrap([Template(wrap([Template(wrap([Text(
            "foo")])), Text("bar")]), params=[Parameter(wrap([Text("baz")]),
            wrap([Text("biz")]))]), Text("buzz")])), Text("usr")]), params=[
            Parameter(wrap([Text("1")]), wrap([Template(wrap([Text("bin")]))]),
            showkey=False)])])
        self.assertWikicodeEqual(valid, self.builder.build(test))

if __name__ == "__main__":
    unittest.main(verbosity=2)
<commit_msg>test_integration2(); finish TestBuilder<commit_after># -*- coding: utf-8  -*-
#
# Copyright (C) 2012-2013 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import unicode_literals
import unittest

from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity,
                                    Tag, Template, Text, Wikilink)
from mwparserfromhell.nodes.extras import Attribute, Parameter
from mwparserfromhell.parser import tokens
from mwparserfromhell.parser.builder import Builder
from mwparserfromhell.smart_list import SmartList
from mwparserfromhell.wikicode import Wikicode

from ._test_tree_equality import TreeEqualityTestCase

wrap = lambda L: Wikicode(SmartList(L))

class TestBuilder(TreeEqualityTestCase):
    """Tests for the builder, which turns tokens into Wikicode objects."""

    def setUp(self):
        self.builder = Builder()

    def test_text(self):
        """tests for building Text nodes"""
        tests = [
            ([tokens.Text(text="foobar")], wrap([Text("foobar")])),
            ([tokens.Text(text="fóóbar")], wrap([Text("fóóbar")])),
            ([tokens.Text(text="spam"), tokens.Text(text="eggs")],
             wrap([Text("spam"), Text("eggs")])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_template(self):
        """tests for building Template nodes"""
        tests = [
            ([tokens.TemplateOpen(), tokens.Text(text="foobar"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foobar")]))])),

            ([tokens.TemplateOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.TemplateClose()],
             wrap([Template(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("1")]), wrap([Text("bar")]),
                           showkey=False)])])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateParamEquals(), tokens.Text(text="baz"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("bar")]), wrap([Text("baz")]))])])),

            ([tokens.TemplateOpen(), tokens.Text(text="foo"),
              tokens.TemplateParamSeparator(), tokens.Text(text="bar"),
              tokens.TemplateParamEquals(), tokens.Text(text="baz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="biz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="buzz"),
              tokens.TemplateParamSeparator(), tokens.Text(text="3"),
              tokens.TemplateParamEquals(), tokens.Text(text="buff"),
              tokens.TemplateParamSeparator(), tokens.Text(text="baff"),
              tokens.TemplateClose()],
             wrap([Template(wrap([Text("foo")]), params=[
                 Parameter(wrap([Text("bar")]), wrap([Text("baz")])),
                 Parameter(wrap([Text("1")]), wrap([Text("biz")]),
                           showkey=False),
                 Parameter(wrap([Text("2")]), wrap([Text("buzz")]),
                           showkey=False),
                 Parameter(wrap([Text("3")]), wrap([Text("buff")])),
                 Parameter(wrap([Text("3")]), wrap([Text("baff")]),
                           showkey=False)])])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_argument(self):
        """tests for building Argument nodes"""
        tests = [
            ([tokens.ArgumentOpen(), tokens.Text(text="foobar"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foobar")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="foo"),
              tokens.ArgumentSeparator(), tokens.Text(text="bar"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foo")]), wrap([Text("bar")]))])),

            ([tokens.ArgumentOpen(), tokens.Text(text="foo"),
              tokens.Text(text="bar"), tokens.ArgumentSeparator(),
              tokens.Text(text="baz"), tokens.Text(text="biz"),
              tokens.ArgumentClose()],
             wrap([Argument(wrap([Text("foo"), Text("bar")]),
                            wrap([Text("baz"), Text("biz")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_wikilink(self):
        """tests for building Wikilink nodes"""
        tests = [
            ([tokens.WikilinkOpen(), tokens.Text(text="foobar"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foobar")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("spam"), Text("eggs")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="foo"),
              tokens.WikilinkSeparator(), tokens.Text(text="bar"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foo")]), wrap([Text("bar")]))])),

            ([tokens.WikilinkOpen(), tokens.Text(text="foo"),
              tokens.Text(text="bar"), tokens.WikilinkSeparator(),
              tokens.Text(text="baz"), tokens.Text(text="biz"),
              tokens.WikilinkClose()],
             wrap([Wikilink(wrap([Text("foo"), Text("bar")]),
                            wrap([Text("baz"), Text("biz")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_html_entity(self):
        """tests for building HTMLEntity nodes"""
        tests = [
            ([tokens.HTMLEntityStart(), tokens.Text(text="nbsp"),
              tokens.HTMLEntityEnd()],
             wrap([HTMLEntity("nbsp", named=True, hexadecimal=False)])),

            ([tokens.HTMLEntityStart(), tokens.HTMLEntityNumeric(),
              tokens.Text(text="107"), tokens.HTMLEntityEnd()],
             wrap([HTMLEntity("107", named=False, hexadecimal=False)])),

            ([tokens.HTMLEntityStart(), tokens.HTMLEntityNumeric(),
              tokens.HTMLEntityHex(char="X"), tokens.Text(text="6B"),
              tokens.HTMLEntityEnd()],
             wrap([HTMLEntity("6B", named=False, hexadecimal=True,
                              hex_char="X")])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_heading(self):
        """tests for building Heading nodes"""
        tests = [
            ([tokens.HeadingStart(level=2), tokens.Text(text="foobar"),
              tokens.HeadingEnd()],
             wrap([Heading(wrap([Text("foobar")]), 2)])),

            ([tokens.HeadingStart(level=4), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.HeadingEnd()],
             wrap([Heading(wrap([Text("spam"), Text("eggs")]), 4)])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_comment(self):
        """tests for building Comment nodes"""
        tests = [
            ([tokens.CommentStart(), tokens.Text(text="foobar"),
              tokens.CommentEnd()],
             wrap([Comment(wrap([Text("foobar")]))])),

            ([tokens.CommentStart(), tokens.Text(text="spam"),
              tokens.Text(text="eggs"), tokens.CommentEnd()],
             wrap([Comment(wrap([Text("spam"), Text("eggs")]))])),
        ]
        for test, valid in tests:
            self.assertWikicodeEqual(valid, self.builder.build(test))

    @unittest.skip("holding this until feature/html_tags is ready")
    def test_tag(self):
        """tests for building Tag nodes"""
        pass

    def test_integration(self):
        """a test for building a combination of templates together"""
        # {{{{{{{{foo}}bar|baz=biz}}buzz}}usr|{{bin}}}}
        test = [tokens.TemplateOpen(), tokens.TemplateOpen(),
                tokens.TemplateOpen(), tokens.TemplateOpen(),
                tokens.Text(text="foo"), tokens.TemplateClose(),
                tokens.Text(text="bar"), tokens.TemplateParamSeparator(),
                tokens.Text(text="baz"), tokens.TemplateParamEquals(),
                tokens.Text(text="biz"), tokens.TemplateClose(),
                tokens.Text(text="buzz"), tokens.TemplateClose(),
                tokens.Text(text="usr"), tokens.TemplateParamSeparator(),
                tokens.TemplateOpen(), tokens.Text(text="bin"),
                tokens.TemplateClose(), tokens.TemplateClose()]
        valid = wrap(
            [Template(wrap([Template(wrap([Template(wrap([Template(wrap([Text(
            "foo")])), Text("bar")]), params=[Parameter(wrap([Text("baz")]),
            wrap([Text("biz")]))]), Text("buzz")])), Text("usr")]), params=[
            Parameter(wrap([Text("1")]), wrap([Template(wrap([Text("bin")]))]),
            showkey=False)])])
        self.assertWikicodeEqual(valid, self.builder.build(test))

    def test_integration2(self):
        """an even more audacious test for building a horrible wikicode mess"""
        # {{a|b|{{c|[[d]]{{{e}}}}}}}[[f|{{{g}}}<!--h-->]]{{i|j= }}
        test = [tokens.TemplateOpen(), tokens.Text(text="a"),
                tokens.TemplateParamSeparator(), tokens.Text(text="b"),
                tokens.TemplateParamSeparator(), tokens.TemplateOpen(),
                tokens.Text(text="c"), tokens.TemplateParamSeparator(),
                tokens.WikilinkOpen(), tokens.Text(text="d"),
                tokens.WikilinkClose(), tokens.ArgumentOpen(),
                tokens.Text(text="e"), tokens.ArgumentClose(),
                tokens.TemplateClose(), tokens.TemplateClose(),
                tokens.WikilinkOpen(), tokens.Text(text="f"),
                tokens.WikilinkSeparator(), tokens.ArgumentOpen(),
                tokens.Text(text="g"), tokens.ArgumentClose(),
                tokens.CommentStart(), tokens.Text(text="h"),
                tokens.CommentEnd(), tokens.WikilinkClose(),
                tokens.TemplateOpen(), tokens.Text(text="i"),
                tokens.TemplateParamSeparator(), tokens.Text(text="j"),
                tokens.TemplateParamEquals(), tokens.HTMLEntityStart(),
                tokens.Text(text="nbsp"), tokens.HTMLEntityEnd(),
                tokens.TemplateClose()]
        valid = wrap(
            [Template(wrap([Text("a")]), params=[Parameter(wrap([Text("1")]),
            wrap([Text("b")]), showkey=False), Parameter(wrap([Text("2")]),
            wrap([Template(wrap([Text("c")]), params=[Parameter(wrap([Text("1")
            ]), wrap([Wikilink(wrap([Text("d")])), Argument(wrap([Text("e")]))]
            ), showkey=False)])]), showkey=False)]), Wikilink(wrap([Text("f")]
            ), wrap([Argument(wrap([Text("g")])), Comment(wrap([Text("h")]))])
            ), Template(wrap([Text("i")]), params=[Parameter(wrap([Text("j")]),
            wrap([HTMLEntity("nbsp", named=True)]))])])
        self.assertWikicodeEqual(valid, self.builder.build(test))

if __name__ == "__main__":
    unittest.main(verbosity=2)
<|endoftext|>