<commit_before>__all__ = ["make_graph", "add_edges", "Results", "find_solution"]

import numpy as np
import copy
import networkx as nx

def make_graph(node_ids, node_labels):
    """
    Make an undirected graph with the nodes specified
    in `node_ids`. Each node will have the attributes
    speficied in `node_labels`.

    Parameters
    ----------
    node_ids : iterable
        The list of node_ids. Can be a list of anything
        that networkx.Graph() can read and use.

    node_labels : dict
        A dictionary of the form {"label":[list of labels
        for all nodes], ...}.
        Defines all labels for all nodes.

    Returns
    -------
    G : networkx.Graph() instance
        The undirected graph containing the nodes and
        corresponding labels, but no edges yet.

    """

    G=nx.Graph()
    for i, nid in enumerate(node_ids):
        G.add_node(nid)
        for k in node_labels.keys():
           G.node[nid][k] = node_labels[k][i]

    return G

def add_edges(G, labels=None, hard_constraint=True, min_importance=0.1, max_importance=0.9):
    """
    Add edges to the graph, with weights.
    Weights are determined by by the importance weights on
    each label.

    If no order of labels is
    specified, then the order of keys in the dictionary
    for each node will be used.

    Parameters
    ----------
    G : networkx.Graph() instance
        The graph without edges

    labels : list of strings
        A list of labels specifying the order of attributes on each
        node to use when calculating the weights.
        This list should be in descending order (with the most important
        label *first*).
        If none are specified, then the order of keywords in the
        dictionary of attributes for each node will be used.

    hard_constraint : bool
        Boolean flag determining whether hard constraints should be used.
        In this case, this means that for the first label specified in
        `labels`, no edges will be drawn when this label is the same
        for two nodes.

    min_importance : float
        The minimum value for the set of importance values. The smaller
        this value, the more important labels will be in the final model
        The key property to consider is max_importance-min_importance, because
        it sets the *relative* importance of the labels to each other.

    max_importance: float
        The maximum value for the set of importance values, the larger
        this value, the smaller the importance of labels in the final model

    Returns
    -------
    G : networkx.Graph() instance
        The same input graph, but with edges.
    """

    ## find the total number of labels
    nlabels = len(labels)

    ## the total number of nodes
    n_nodes = G.number_of_nodes()

    ## get a list of lists of all node labels
    node_labels = []
    for l in labels:
        node_labels.append([G.node[i][l] for i in G.nodes()])

    ## weights for the different attributes
    weights = np.linspace(min_importance, max_importance, nlabels)

    ## iterate over all the different possible labels
    for i, sl in enumerate(node_labels):
        ## iterate over all nodes
        for k, n1 in enumerate(G.nodes()):
            for l, n2 in enumerate(G.nodes()):
                ## if sessions have the same label,
                ## either make no node (for first label),
                ## or weaken a node that's already there
                if hard_constraint:
                    if sl[k] == sl[l]:
                        if i == 0 or G.node[k][labels[0]] == G.node[l][labels[0]]:
                            continue
                        else:
                            G[k][l]["weight"] *= weights[i-1]
                    else:
                        if i == 0:
                            G.add_edge(n1,n2,weight=1.0)
                        else:
                            continue

                else:
                    if sl[k] == sl[l]:
                        if i == 0:
                            G.add_edge(n1,n2,weight=weights[0])
                        else:
                            G[k][l]["weight"] *= weights[i]
                    else:
                        if i == 0:
                            G.add_edge(n1,n2,weight=1.0)
                        else:
                            continue
    return G

def _sort_cliques_by_weights(G, cliques, n_elements):
    """
    Sort cliques by their weights.

    Parameters
    ----------
    G : networkx.Graph instance
        Undirected graph with nodes and edges.
        Edges must have attribute 'weight'

    cliques : iterable
        A list of lists; inner lists must have n_elements members

    n_elements : integer
        The number of elements in each clique

    Returns
    -------
    cliques : iterable
        All cliques sorted by weights in descending order

    summed_weights : iterable
        The list of summed weights, sorted in the
        same descending order as cliques

    """
    ## compute summed weights for all cliques:
    summed_weights = []
    for cl in cliques:
        ww = 0
        for i in xrange(n_elements):
            for j in xrange(n_elements):
                if i >= j:
                    continue
                else:
                    ww += G[cl[i]][cl[j]]["weight"]

        summed_weights.append(ww)

    ## sort cliques from highest weight to smallest
    sorted_cliques = cliques[np.argsort(summed_weights)[::-1]]
    ## sort weights in the same way
    summed_weights = np.sort(summed_weights)[::-1]

    return sorted_cliques, summed_weights

class Results(object):

    def __init__(self, n_elements):
        self.n_elements = n_elements
        self.groups = []
        self.all_weights = []
        self.success = True

    def update_groups(self, groups):
        self.groups.append(groups)

    def update_weights(self, weights_sum_total):
        self.all_weights.append(weights_sum_total)
        self.weights_sum_total = np.sum(self.all_weights)

def find_solution(G, n_elements, n_unused=None, results=None):
    """
    Sort nodes in G into groups of n_elements members such that
    the total sum of weights is maximized.
    If the graph includes hard constraints on the relationship between
    nodes (i.e. missing edges), it is possible that no solution is found.

    In the case of a fully connected graph, the solution will be that
    which maximizes the weights. The weights are inherent attributes of
    the Graph and must be calculated beforehand (see `add_edges` for details).

    Parameters
    ----------
    G : networkx.Graph() instance
        Undirected graph with nodes and edges. The edges must have weights
        between 0 and 1, but edges can be missing if no relationship exists
        between nodes.

    groups : iterable
        A list of lists containing all groups of n_elements members fulfilling
        the connectivity constraints that maximize the sum of weights of all
        groups being used.
        Should be initialized with an empty list, will be modified during the
        recursion to be filled with the groups.

    n_elements : integer
        The number of elements per group. Must be an integer divisor of the
        total number of nodes in the graph.

    n_unused : integer
        The number of unused nodes in the graph at every recursion step.
        If None, then it will be initialized as the total number of nodes
        in the graph.

    weights_total_sum : list
        The total sum of weights of elements in `groups`.
        If None, then it will be initialized as an empty list to count
        the sum of weights for each individual group. Will be summed at
        the end before output into a float value.
        Note: DO NOT SET THIS AT THE BEGINNING OF THE RUN!

    Returns
    -------
    success : bool
        Flag indicating success or failure of the algorithm

    groups: iterable
        A list of lists containing all groups of n_elements members fulfilling
        the connectivity constraints that maximize the sum of weights of all
        groups being used.

    weights_total_sum : float
        The total sum of all weights of the output groups

    """

    assert G.number_of_nodes() % np.float(n_elements) == 0, "Number of sessions must be " + \
                                "an integer multiple of n_elements"

    ## initialize results object
    if results is None:
        results = Results(n_elements)

    if n_unused is None:
        n_unused = G.number_of_nodes()

    ## base case
    if n_unused == 0:
        results.success = True
        return results

    ## recursion
    else:
        ## find all cliques in the graph G
        cliques = list(nx.enumerate_all_cliques(G))

        ## find all cliques that have the required number of elements
        cliques = np.array([c for c in cliques if len(c)==n_elements])

        ## sort cliques by weights
        cliques, summed_weights = _sort_cliques_by_weights(G, cliques, n_elements)

        ## find the total number of cliques with n_elements members
        ncliques = len(cliques)

        ## loop over all cliques:
        for g,(cl,ww) in enumerate(zip(cliques, summed_weights)):
            cl_topics = [G.node[c] for c in cl]

            ## add the new clique to the list of output groups
            results.update_groups(zip(cl, cl_topics))

            ## add total weight of the clique:
            results.update_weights(ww)

            ## make a new deep copy for the next recursion step
            G_new = copy.deepcopy(G)

            ## remove clique from graph
            for n in cl:
                G_new.remove_node(n)

            ## compute new unused number of nodes
            n_unused = G_new.number_of_nodes()

            ## if no unused nodes are left, return the selected groups,
            ## otherwise recurse
            results = find_solution(G_new, n_elements, n_unused, results)
            if results is not None:
                if results.success:
                        return results

            ## backtrack
            else:
                results.success = False
                results.groups.pop(-1)
                results.all_weights.pop(-1)
                continue

    if len(results.groups) == 0:
        print("No solution found!")
        results.success = False
        return results

    else:
        results.groups.pop(-1)
        results.all_weights.pop(-1)

        results.success = False
        return results

<commit_msg>Numerical versions of some of the functions<commit_after>__all__ = ["make_graph", "add_edges", "Results", "find_solution"]

import numpy as np
import copy
import networkx as nx

def make_graph(node_ids, node_labels):
    """
    Make an undirected graph with the nodes specified
    in `node_ids`. Each node will have the attributes
    speficied in `node_labels`.

    Parameters
    ----------
    node_ids : iterable
        The list of node_ids. Can be a list of anything
        that networkx.Graph() can read and use.

    node_labels : dict
        A dictionary of the form {"label":[list of labels
        for all nodes], ...}.
        Defines all labels for all nodes.

    Returns
    -------
    G : networkx.Graph() instance
        The undirected graph containing the nodes and
        corresponding labels, but no edges yet.

    """

    G=nx.Graph()
    for i, nid in enumerate(node_ids):
        G.add_node(nid)
        for k in node_labels.keys():
           G.node[nid][k] = node_labels[k][i]

    return G

def add_edges(G, labels=None, hard_constraint=True, min_importance=0.1, max_importance=0.9):
    """
    Add edges to the graph, with weights.
    Weights are determined by by the importance weights on
    each label.

    If no order of labels is
    specified, then the order of keys in the dictionary
    for each node will be used.

    Parameters
    ----------
    G : networkx.Graph() instance
        The graph without edges

    labels : list of strings
        A list of labels specifying the order of attributes on each
        node to use when calculating the weights.
        This list should be in descending order (with the most important
        label *first*).
        If none are specified, then the order of keywords in the
        dictionary of attributes for each node will be used.

    hard_constraint : bool
        Boolean flag determining whether hard constraints should be used.
        In this case, this means that for the first label specified in
        `labels`, no edges will be drawn when this label is the same
        for two nodes.

    min_importance : float
        The minimum value for the set of importance values. The smaller
        this value, the more important labels will be in the final model
        The key property to consider is max_importance-min_importance, because
        it sets the *relative* importance of the labels to each other.

    max_importance: float
        The maximum value for the set of importance values, the larger
        this value, the smaller the importance of labels in the final model

    Returns
    -------
    G : networkx.Graph() instance
        The same input graph, but with edges.
    """

    ## find the total number of labels
    nlabels = len(labels)

    ## the total number of nodes
    n_nodes = G.number_of_nodes()

    ## get a list of lists of all node labels
    node_labels = []
    for l in labels:
        node_labels.append([G.node[i][l] for i in G.nodes()])

    ## weights for the different attributes
    weights = np.linspace(min_importance, max_importance, nlabels)

    ## iterate over all the different possible labels
    for i, sl in enumerate(node_labels):
        ## iterate over all nodes
        for k, n1 in enumerate(G.nodes()):
            for l, n2 in enumerate(G.nodes()):
                ## if sessions have the same label,
                ## either make no node (for first label),
                ## or weaken a node that's already there
                if hard_constraint:
                    if sl[k] == sl[l]:
                        if i == 0 or G.node[k][labels[0]] == G.node[l][labels[0]]:
                            continue
                        else:
                            G[k][l]["weight"] *= weights[i-1]
                    else:
                        if i == 0:
                            G.add_edge(n1,n2,weight=1.0)
                        else:
                            continue

                else:
                    if sl[k] == sl[l]:
                        if i == 0:
                            G.add_edge(n1,n2,weight=weights[0])
                        else:
                            G[k][l]["weight"] *= weights[i]
                    else:
                        if i == 0:
                            G.add_edge(n1,n2,weight=1.0)
                        else:
                            continue
    return G

def add_edges_numerical(G, edges, hard_constraint=True, threshold=0.5):
    """
    Add edges to the graph, with weights.
    Weights are determined by by the importance weights on
    each label.

    If no order of labels is
    specified, then the order of keys in the dictionary
    for each node will be used.

    Parameters
    ----------
    G : networkx.Graph() instance
        The graph without edges

    hard_constraint : bool
        Boolean flag determining whether hard constraints should be used.
        In this case, this means that for the first label specified in
        `labels`, no edges will be drawn when this label is the same
        for two nodes.

    threshold: float out of [0,1]
        The threshold of the dissimilarity measure describing how dissimilar
        two sessions are. If the threshold is small, the graph has a high tolerance
        to similar sessions. That is, it is more likely to schedule similar sessions
        at the same time. If it is small, then this tolerance is much smaller. There
        will be much fewer edges between nodes. This means that there is a much
        smaller chance that similar sessions will be scheduled at the same time.
        However, fewer edges also means that there's a greater chance that no
        solution will be found at all.

    Returns
    -------
    G : networkx.Graph() instance
        The same input graph, but with edges.
    """

    n_nodes = G.number_of_nodes()

    assert np.size(edges) == (n_nodes*(n_nodes - 1))/2.0, "Incorrect number of edges!"

    # counter for edges
    i = 0

    for k, n1 in enumerate(G.nodes()):
        for l, n2 in enumerate(G.nodes()):
            if k <= l:
                continue
            # if a hard constraint is set, make no edge
            # if the edge weight is smaller than `threshold`,
            # otherwise set the weight edge to 1
            if hard_constraint:
                if edges[i] < threshold:
                    i += 1
                    continue
                else:
                    G.add_edge(n1, n2, weight=1.0)
            # otherwise just set all the weights the way they are
            else:
                G.add_edge(n1,n2,weight=edges[i])

            i += 1

    return G

def _sort_cliques_by_weights(G, cliques, n_elements):
    """
    Sort cliques by their weights.

    Parameters
    ----------
    G : networkx.Graph instance
        Undirected graph with nodes and edges.
        Edges must have attribute 'weight'

    cliques : iterable
        A list of lists; inner lists must have n_elements members

    n_elements : integer
        The number of elements in each clique

    Returns
    -------
    cliques : iterable
        All cliques sorted by weights in descending order

    summed_weights : iterable
        The list of summed weights, sorted in the
        same descending order as cliques

    """
    ## compute summed weights for all cliques:
    summed_weights = []
    for cl in cliques:
        ww = 0
        for i in xrange(n_elements):
            for j in xrange(n_elements):
                if i >= j:
                    continue
                else:
                    ww += G[cl[i]][cl[j]]["weight"]

        summed_weights.append(ww)

    ## sort cliques from highest weight to smallest
    sorted_cliques = cliques[np.argsort(summed_weights)[::-1]]
    ## sort weights in the same way
    summed_weights = np.sort(summed_weights)[::-1]

    return sorted_cliques, summed_weights

class Results(object):

    def __init__(self, n_elements):
        self.n_elements = n_elements
        self.groups = []
        self.all_weights = []
        self.success = True

    def update_groups(self, groups):
        self.groups.append(groups)

    def update_weights(self, weights_sum_total):
        self.all_weights.append(weights_sum_total)
        self.weights_sum_total = np.sum(self.all_weights)

def find_solution(G, n_elements, n_unused=None, results=None):
    """
    Sort nodes in G into groups of n_elements members such that
    the total sum of weights is maximized.
    If the graph includes hard constraints on the relationship between
    nodes (i.e. missing edges), it is possible that no solution is found.

    In the case of a fully connected graph, the solution will be that
    which maximizes the weights. The weights are inherent attributes of
    the Graph and must be calculated beforehand (see `add_edges` for details).

    Parameters
    ----------
    G : networkx.Graph() instance
        Undirected graph with nodes and edges. The edges must have weights
        between 0 and 1, but edges can be missing if no relationship exists
        between nodes.

    groups : iterable
        A list of lists containing all groups of n_elements members fulfilling
        the connectivity constraints that maximize the sum of weights of all
        groups being used.
        Should be initialized with an empty list, will be modified during the
        recursion to be filled with the groups.

    n_elements : integer
        The number of elements per group. Must be an integer divisor of the
        total number of nodes in the graph.

    n_unused : integer
        The number of unused nodes in the graph at every recursion step.
        If None, then it will be initialized as the total number of nodes
        in the graph.

    weights_total_sum : list
        The total sum of weights of elements in `groups`.
        If None, then it will be initialized as an empty list to count
        the sum of weights for each individual group. Will be summed at
        the end before output into a float value.
        Note: DO NOT SET THIS AT THE BEGINNING OF THE RUN!

    Returns
    -------
    success : bool
        Flag indicating success or failure of the algorithm

    groups: iterable
        A list of lists containing all groups of n_elements members fulfilling
        the connectivity constraints that maximize the sum of weights of all
        groups being used.

    weights_total_sum : float
        The total sum of all weights of the output groups

    """

    assert G.number_of_nodes() % np.float(n_elements) == 0, "Number of sessions must be " + \
                                "an integer multiple of n_elements"

    ## initialize results object
    if results is None:
        results = Results(n_elements)

    if n_unused is None:
        n_unused = G.number_of_nodes()

    ## base case
    if n_unused == 0:
        results.success = True
        return results

    ## recursion
    else:
        ## find all cliques in the graph G
        cliques = list(nx.enumerate_all_cliques(G))

        ## find all cliques that have the required number of elements
        cliques = np.array([c for c in cliques if len(c)==n_elements])

        ## sort cliques by weights
        cliques, summed_weights = _sort_cliques_by_weights(G, cliques, n_elements)

        ## find the total number of cliques with n_elements members
        ncliques = len(cliques)

        ## loop over all cliques:
        for g,(cl,ww) in enumerate(zip(cliques, summed_weights)):
            cl_topics = [G.node[c] for c in cl]

            ## add the new clique to the list of output groups
            results.update_groups(zip(cl, cl_topics))

            ## add total weight of the clique:
            results.update_weights(ww)

            ## make a new deep copy for the next recursion step
            G_new = copy.deepcopy(G)

            ## remove clique from graph
            for n in cl:
                G_new.remove_node(n)

            ## compute new unused number of nodes
            n_unused = G_new.number_of_nodes()

            ## if no unused nodes are left, return the selected groups,
            ## otherwise recurse
            results = find_solution(G_new, n_elements, n_unused, results)
            if results is not None:
                if results.success:
                        return results

            ## backtrack
            else:
                results.success = False
                results.groups.pop(-1)
                results.all_weights.pop(-1)
                continue

    if len(results.groups) == 0:
        print("No solution found!")
        results.success = False
        return results

    else:
        results.groups.pop(-1)
        results.all_weights.pop(-1)

        results.success = False
        return results

def find_solution_numerical(G, n_elements, n_unused=None, results=None):
    """
    Sort nodes in G into groups of n_elements members such that
    the total sum of weights is maximized.
    If the graph includes hard constraints on the relationship between
    nodes (i.e. missing edges), it is possible that no solution is found.

    In the case of a fully connected graph, the solution will be that
    which maximizes the weights. The weights are inherent attributes of
    the Graph and must be calculated beforehand (see `add_edges` for details).

    Parameters
    ----------
    G : networkx.Graph() instance
        Undirected graph with nodes and edges. The edges must have weights
        between 0 and 1, but edges can be missing if no relationship exists
        between nodes.

    n_elements : integer
        The number of elements per group. Must be an integer divisor of the
        total number of nodes in the graph.

    n_unused : integer
        The number of unused nodes in the graph at every recursion step.
        If None, then it will be initialized as the total number of nodes
        in the graph.

    weights_total_sum : list
        The total sum of weights of elements in `groups`.
        If None, then it will be initialized as an empty list to count
        the sum of weights for each individual group. Will be summed at
        the end before output into a float value.
        Note: DO NOT SET THIS AT THE BEGINNING OF THE RUN!

    Returns
    -------
    success : bool
        Flag indicating success or failure of the algorithm

    groups: iterable
        A list of lists containing all groups of n_elements members fulfilling
        the connectivity constraints that maximize the sum of weights of all
        groups being used.

    weights_total_sum : float
        The total sum of all weights of the output groups

    """

    assert G.number_of_nodes() % np.float(n_elements) == 0, "Number of sessions must be " + \
                                "an integer multiple of n_elements"

    ## initialize results object
    if results is None:
        results = Results(n_elements)

    if n_unused is None:
        n_unused = G.number_of_nodes()

    ## base case
    if n_unused == 0:
        results.success = True
        return results

    ## recursion
    else:
        ## find all cliques in the graph G
        cliques = list(nx.enumerate_all_cliques(G))

        ## find all cliques that have the required number of elements
        cliques = np.array([c for c in cliques if len(c)==n_elements])

        ## sort cliques by weights
        cliques, summed_weights = _sort_cliques_by_weights(G, cliques, n_elements)

        ## find the total number of cliques with n_elements members
        ncliques = len(cliques)

        ## loop over all cliques:
        for g,(cl,ww) in enumerate(zip(cliques, summed_weights)):
            cl_topics = [G.node[c] for c in cl]

            ## add the new clique to the list of output groups
            results.update_groups(zip(cl, cl_topics))

            ## add total weight of the clique:
            results.update_weights(ww)

            ## make a new deep copy for the next recursion step
            G_new = copy.deepcopy(G)

            ## remove clique from graph
            for n in cl:
                G_new.remove_node(n)

            ## compute new unused number of nodes
            n_unused = G_new.number_of_nodes()

            ## if no unused nodes are left, return the selected groups,
            ## otherwise recurse
            results = find_solution(G_new, n_elements, n_unused, results)
            if results is not None:
                if results.success:
                        return results

            ## backtrack
            else:
                results.success = False
                results.groups.pop(-1)
                results.all_weights.pop(-1)
                continue

    if len(results.groups) == 0:
        print("No solution found!")
        results.success = False
        return results

    else:
        results.groups.pop(-1)
        results.all_weights.pop(-1)

        results.success = False
        return results

<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst

import re
import copy
from collections import OrderedDict

from . import core
from ...table import Table
from . import cparser
from ...utils import set_locale

class FastBasic(metaclass=core.MetaBaseReader):
    """
    This class is intended to handle the same format addressed by the
    ordinary :class:`Basic` writer, but it acts as a wrapper for underlying C
    code and is therefore much faster. Unlike the other ASCII readers and
    writers, this class is not very extensible and is restricted
    by optimization requirements.
    """
    _format_name = 'fast_basic'
    _description = 'Basic table with custom delimiter using the fast C engine'
    _fast = True
    fill_extra_cols = False
    guessing = False
    strict_names = False

    def __init__(self, default_kwargs={}, **user_kwargs):
        # Make sure user does not set header_start to None for a reader
        # that expects a non-None value (i.e. a number >= 0).  This mimics
        # what happens in the Basic reader.
        if (default_kwargs.get('header_start', 0) is not None and
                user_kwargs.get('header_start', 0) is None):
            raise ValueError('header_start cannot be set to None for this Reader')

        kwargs = default_kwargs.copy()
        kwargs.update(user_kwargs)  # user kwargs take precedence over defaults
        delimiter = kwargs.pop('delimiter', ' ')
        self.delimiter = str(delimiter) if delimiter is not None else None
        self.write_comment = kwargs.get('comment', '# ')
        self.comment = kwargs.pop('comment', '#')
        if self.comment is not None:
            self.comment = str(self.comment)
        self.quotechar = str(kwargs.pop('quotechar', '"'))
        self.header_start = kwargs.pop('header_start', 0)
        # If data_start is not specified, start reading
        # data right after the header line
        data_start_default = user_kwargs.get('data_start', self.header_start +
                                    1 if self.header_start is not None else 1)
        self.data_start = kwargs.pop('data_start', data_start_default)
        self.kwargs = kwargs
        self.strip_whitespace_lines = True
        self.strip_whitespace_fields = True

    def _read_header(self):
        # Use the tokenizer by default -- this method
        # can be overridden for specialized headers
        self.engine.read_header()

    def read(self, table):
        """
        Read input data (file-like object, filename, list of strings, or
        single string) into a Table and return the result.
        """
        if self.comment is not None and len(self.comment) != 1:
            raise core.ParameterError("The C reader does not support a comment regex")
        elif self.data_start is None:
            raise core.ParameterError("The C reader does not allow data_start to be None")
        elif self.header_start is not None and self.header_start < 0 and \
             not isinstance(self, FastCommentedHeader):
            raise core.ParameterError("The C reader does not allow header_start to be "
                                      "negative except for commented-header files")
        elif self.data_start < 0:
            raise core.ParameterError("The C reader does not allow data_start to be negative")
        elif len(self.delimiter) != 1:
            raise core.ParameterError("The C reader only supports 1-char delimiters")
        elif len(self.quotechar) != 1:
            raise core.ParameterError("The C reader only supports a length-1 quote character")
        elif 'converters' in self.kwargs:
            raise core.ParameterError("The C reader does not support passing "
                                      "specialized converters")
        elif 'encoding' in self.kwargs:
            raise core.ParameterError("The C reader does not use the encoding parameter")
        elif 'Outputter' in self.kwargs:
            raise core.ParameterError("The C reader does not use the Outputter parameter")
        elif 'Inputter' in self.kwargs:
            raise core.ParameterError("The C reader does not use the Inputter parameter")
        elif 'data_Splitter' in self.kwargs or 'header_Splitter' in self.kwargs:
            raise core.ParameterError("The C reader does not use a Splitter class")

        self.strict_names = self.kwargs.pop('strict_names', False)

        fast_reader = self.kwargs.get('fast_reader', True)
        if not isinstance(fast_reader, dict):
            fast_reader = {}

        fast_reader.pop('enable', None)
        self.return_header_chars = fast_reader.pop('return_header_chars', False)
        self.kwargs['fast_reader'] = fast_reader

        self.engine = cparser.CParser(table, self.strip_whitespace_lines,
                                      self.strip_whitespace_fields,
                                      delimiter=self.delimiter,
                                      header_start=self.header_start,
                                      comment=self.comment,
                                      quotechar=self.quotechar,
                                      data_start=self.data_start,
                                      fill_extra_cols=self.fill_extra_cols,
                                      **self.kwargs)
        conversion_info = self._read_header()
        self.check_header()
        if conversion_info is not None:
            try_int, try_float, try_string = conversion_info
        else:
            try_int = {}
            try_float = {}
            try_string = {}

        with set_locale('C'):
            data, comments = self.engine.read(try_int, try_float, try_string)
        out = self.make_table(data, comments)

        if self.return_header_chars:
            out.meta['__ascii_fast_reader_header_chars__'] = self.engine.header_chars

        return out

    def make_table(self, data, comments):
        """Actually make the output table give the data and comments."""
        meta = OrderedDict()
        if comments:
            meta['comments'] = comments
        return Table(data, names=list(self.engine.get_names()), meta=meta)

    def check_header(self):
        names = self.engine.get_header_names() or self.engine.get_names()
        if self.strict_names:
            # Impose strict requirements on column names (normally used in guessing)
            bads = [" ", ",", "|", "\t", "'", '"']
            for name in names:
                if (core._is_number(name) or
                    len(name) == 0 or
                    name[0] in bads or
                    name[-1] in bads):
                    raise ValueError('Column name {0!r} does not meet strict name requirements'
                                     .format(name))
        # When guessing require at least two columns
        if self.guessing and len(names) <= 1:
            raise ValueError('Table format guessing requires at least two columns, got {}'
                             .format(names))

    def write(self, table, output):
        """
        Use a fast Cython method to write table data to output,
        where output is a filename or file-like object.
        """
        self._write(table, output, {})

    def _write(self, table, output, default_kwargs,
               header_output=True, output_types=False):

        write_kwargs = {'delimiter': self.delimiter,
                         'quotechar': self.quotechar,
                         'strip_whitespace': self.strip_whitespace_fields,
                         'comment': self.write_comment
                         }
        write_kwargs.update(default_kwargs)
        # user kwargs take precedence over default kwargs
        write_kwargs.update(self.kwargs)
        writer = cparser.FastWriter(table, **write_kwargs)
        writer.write(output, header_output, output_types)

class FastCsv(FastBasic):
    """
    A faster version of the ordinary :class:`Csv` writer that uses the
    optimized C parsing engine. Note that this reader will append empty
    field values to the end of any row with not enough columns, while
    :class:`FastBasic` simply raises an error.
    """
    _format_name = 'fast_csv'
    _description = 'Comma-separated values table using the fast C engine'
    _fast = True
    fill_extra_cols = True

    def __init__(self, **kwargs):
        super().__init__({'delimiter': ',', 'comment': None}, **kwargs)

    def write(self, table, output):
        """
        Override the default write method of `FastBasic` to
        output masked values as empty fields.
        """
        self._write(table, output, {'fill_values': [(core.masked, '')]})

class FastTab(FastBasic):
    """
    A faster version of the ordinary :class:`Tab` reader that uses
    the optimized C parsing engine.
    """
    _format_name = 'fast_tab'
    _description = 'Tab-separated values table using the fast C engine'
    _fast = True

    def __init__(self, **kwargs):
        super().__init__({'delimiter': '\t'}, **kwargs)
        self.strip_whitespace_lines = False
        self.strip_whitespace_fields = False

class FastNoHeader(FastBasic):
    """
    This class uses the fast C engine to read tables with no header line. If
    the names parameter is unspecified, the columns will be autonamed with
    "col{}".
    """
    _format_name = 'fast_no_header'
    _description = 'Basic table with no headers using the fast C engine'
    _fast = True

    def __init__(self, **kwargs):
        super().__init__({'header_start': None, 'data_start': 0}, **kwargs)

    def write(self, table, output):
        """
        Override the default writing behavior in `FastBasic` so
        that columns names are not included in output.
        """
        self._write(table, output, {}, header_output=None)

class FastCommentedHeader(FastBasic):
    """
    A faster version of the :class:`CommentedHeader` reader, which looks for
    column names in a commented line. ``header_start`` denotes the index of
    the header line among all commented lines and is 0 by default.
    """
    _format_name = 'fast_commented_header'
    _description = 'Columns name in a commented line using the fast C engine'
    _fast = True

    def __init__(self, **kwargs):
        super().__init__({}, **kwargs)
        # Mimic CommentedHeader's behavior in which data_start
        # is relative to header_start if unspecified; see #2692
        if 'data_start' not in kwargs:
            self.data_start = 0

    def make_table(self, data, comments):
        """
        Actually make the output table give the data and comments.  This is
        slightly different from the base FastBasic method in the way comments
        are handled.
        """
        meta = OrderedDict()
        if comments:
            meta['comments'] = comments[1:]
            if not meta['comments']:
                del meta['comments']

        return Table(data, names=list(self.engine.get_names()), meta=meta)

    def _read_header(self):
        tmp = self.engine.source
        commented_lines = []

        for line in tmp.splitlines():
            line = line.lstrip()
            if line and line[0] == self.comment:  # line begins with a comment
                commented_lines.append(line[1:])
                if len(commented_lines) == self.header_start + 1:
                    break

        if len(commented_lines) <= self.header_start:
            raise cparser.CParserError('not enough commented lines')

        self.engine.setup_tokenizer([commented_lines[self.header_start]])
        self.engine.header_start = 0
        self.engine.read_header()
        self.engine.setup_tokenizer(tmp)

    def write(self, table, output):
        """
        Override the default writing behavior in `FastBasic` so
        that column names are commented.
        """
        self._write(table, output, {}, header_output='comment')

class FastRdb(FastBasic):
    """
    A faster version of the :class:`Rdb` reader. This format is similar to
    tab-delimited, but it also contains a header line after the column
    name line denoting the type of each column (N for numeric, S for string).
    """
    _format_name = 'fast_rdb'
    _description = 'Tab-separated with a type definition header line'
    _fast = True

    def __init__(self, **kwargs):
        super().__init__({'delimiter': '\t', 'data_start': 2}, **kwargs)
        self.strip_whitespace_lines = False
        self.strip_whitespace_fields = False

    def _read_header(self):
        tmp = self.engine.source
        line1 = ''
        line2 = ''
        for line in tmp.splitlines():
            # valid non-comment line
            if not line1 and line.strip() and line.lstrip()[0] != self.comment:
                line1 = line
            elif not line2 and line.strip() and line.lstrip()[0] != self.comment:
                line2 = line
                break
        else:  # less than 2 lines in table
            raise ValueError('RDB header requires 2 lines')

        # tokenize the two header lines separately
        self.engine.setup_tokenizer([line2])
        self.engine.header_start = 0
        self.engine.read_header()
        types = self.engine.get_names()
        self.engine.setup_tokenizer([line1])
        self.engine.set_names([])
        self.engine.read_header()

        if len(self.engine.get_names()) != len(types):
            raise ValueError('RDB header mismatch between number of '
                             'column names and column types')

        if any(not re.match(r'\d*(N|S)$', x, re.IGNORECASE) for x in types):
            raise ValueError('RDB type definitions do not all match '
                             '[num](N|S): {0}'.format(types))

        try_int = {}
        try_float = {}
        try_string = {}

        for name, col_type in zip(self.engine.get_names(), types):
            if col_type[-1].lower() == 's':
                try_int[name] = 0
                try_float[name] = 0
                try_string[name] = 1
            else:
                try_int[name] = 1
                try_float[name] = 1
                try_string[name] = 0

        self.engine.setup_tokenizer(tmp)
        return (try_int, try_float, try_string)

    def write(self, table, output):
        """
        Override the default writing behavior in `FastBasic` to
        output a line with column types after the column name line.
        """
        self._write(table, output, {}, output_types=True)
<commit_msg>Deepcopy user_kwargs in FastBasic __init__ to isolate class input<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst

import re
import copy
from collections import OrderedDict

from . import core
from ...table import Table
from . import cparser
from ...utils import set_locale

class FastBasic(metaclass=core.MetaBaseReader):
    """
    This class is intended to handle the same format addressed by the
    ordinary :class:`Basic` writer, but it acts as a wrapper for underlying C
    code and is therefore much faster. Unlike the other ASCII readers and
    writers, this class is not very extensible and is restricted
    by optimization requirements.
    """
    _format_name = 'fast_basic'
    _description = 'Basic table with custom delimiter using the fast C engine'
    _fast = True
    fill_extra_cols = False
    guessing = False
    strict_names = False

    def __init__(self, default_kwargs={}, **user_kwargs):
        # Make sure user does not set header_start to None for a reader
        # that expects a non-None value (i.e. a number >= 0).  This mimics
        # what happens in the Basic reader.
        if (default_kwargs.get('header_start', 0) is not None and
                user_kwargs.get('header_start', 0) is None):
            raise ValueError('header_start cannot be set to None for this Reader')

        # Set up kwargs and copy any user kwargs.  Use deepcopy user kwargs
        # since they may contain a dict item which would end up as a ref to the
        # original and get munged later (e.g. in cparser.pyx validation of
        # fast_reader dict).
        kwargs = default_kwargs.copy()
        kwargs.update(copy.deepcopy(user_kwargs))

        delimiter = kwargs.pop('delimiter', ' ')
        self.delimiter = str(delimiter) if delimiter is not None else None
        self.write_comment = kwargs.get('comment', '# ')
        self.comment = kwargs.pop('comment', '#')
        if self.comment is not None:
            self.comment = str(self.comment)
        self.quotechar = str(kwargs.pop('quotechar', '"'))
        self.header_start = kwargs.pop('header_start', 0)
        # If data_start is not specified, start reading
        # data right after the header line
        data_start_default = user_kwargs.get('data_start', self.header_start +
                                    1 if self.header_start is not None else 1)
        self.data_start = kwargs.pop('data_start', data_start_default)
        self.kwargs = kwargs
        self.strip_whitespace_lines = True
        self.strip_whitespace_fields = True

    def _read_header(self):
        # Use the tokenizer by default -- this method
        # can be overridden for specialized headers
        self.engine.read_header()

    def read(self, table):
        """
        Read input data (file-like object, filename, list of strings, or
        single string) into a Table and return the result.
        """
        if self.comment is not None and len(self.comment) != 1:
            raise core.ParameterError("The C reader does not support a comment regex")
        elif self.data_start is None:
            raise core.ParameterError("The C reader does not allow data_start to be None")
        elif self.header_start is not None and self.header_start < 0 and \
             not isinstance(self, FastCommentedHeader):
            raise core.ParameterError("The C reader does not allow header_start to be "
                                      "negative except for commented-header files")
        elif self.data_start < 0:
            raise core.ParameterError("The C reader does not allow data_start to be negative")
        elif len(self.delimiter) != 1:
            raise core.ParameterError("The C reader only supports 1-char delimiters")
        elif len(self.quotechar) != 1:
            raise core.ParameterError("The C reader only supports a length-1 quote character")
        elif 'converters' in self.kwargs:
            raise core.ParameterError("The C reader does not support passing "
                                      "specialized converters")
        elif 'encoding' in self.kwargs:
            raise core.ParameterError("The C reader does not use the encoding parameter")
        elif 'Outputter' in self.kwargs:
            raise core.ParameterError("The C reader does not use the Outputter parameter")
        elif 'Inputter' in self.kwargs:
            raise core.ParameterError("The C reader does not use the Inputter parameter")
        elif 'data_Splitter' in self.kwargs or 'header_Splitter' in self.kwargs:
            raise core.ParameterError("The C reader does not use a Splitter class")

        self.strict_names = self.kwargs.pop('strict_names', False)

        # Process fast_reader kwarg, which may or may not exist (though ui.py will always
        # pass this as a dict with at least 'enable' set).
        fast_reader = self.kwargs.get('fast_reader', True)
        if not isinstance(fast_reader, dict):
            fast_reader = {}

        fast_reader.pop('enable', None)
        self.return_header_chars = fast_reader.pop('return_header_chars', False)
        # Put fast_reader dict back into kwargs.
        self.kwargs['fast_reader'] = fast_reader

        self.engine = cparser.CParser(table, self.strip_whitespace_lines,
                                      self.strip_whitespace_fields,
                                      delimiter=self.delimiter,
                                      header_start=self.header_start,
                                      comment=self.comment,
                                      quotechar=self.quotechar,
                                      data_start=self.data_start,
                                      fill_extra_cols=self.fill_extra_cols,
                                      **self.kwargs)
        conversion_info = self._read_header()
        self.check_header()
        if conversion_info is not None:
            try_int, try_float, try_string = conversion_info
        else:
            try_int = {}
            try_float = {}
            try_string = {}

        with set_locale('C'):
            data, comments = self.engine.read(try_int, try_float, try_string)
        out = self.make_table(data, comments)

        if self.return_header_chars:
            out.meta['__ascii_fast_reader_header_chars__'] = self.engine.header_chars

        return out

    def make_table(self, data, comments):
        """Actually make the output table give the data and comments."""
        meta = OrderedDict()
        if comments:
            meta['comments'] = comments
        return Table(data, names=list(self.engine.get_names()), meta=meta)

    def check_header(self):
        names = self.engine.get_header_names() or self.engine.get_names()
        if self.strict_names:
            # Impose strict requirements on column names (normally used in guessing)
            bads = [" ", ",", "|", "\t", "'", '"']
            for name in names:
                if (core._is_number(name) or
                    len(name) == 0 or
                    name[0] in bads or
                    name[-1] in bads):
                    raise ValueError('Column name {0!r} does not meet strict name requirements'
                                     .format(name))
        # When guessing require at least two columns
        if self.guessing and len(names) <= 1:
            raise ValueError('Table format guessing requires at least two columns, got {}'
                             .format(names))

    def write(self, table, output):
        """
        Use a fast Cython method to write table data to output,
        where output is a filename or file-like object.
        """
        self._write(table, output, {})

    def _write(self, table, output, default_kwargs,
               header_output=True, output_types=False):

        write_kwargs = {'delimiter': self.delimiter,
                         'quotechar': self.quotechar,
                         'strip_whitespace': self.strip_whitespace_fields,
                         'comment': self.write_comment
                         }
        write_kwargs.update(default_kwargs)
        # user kwargs take precedence over default kwargs
        write_kwargs.update(self.kwargs)
        writer = cparser.FastWriter(table, **write_kwargs)
        writer.write(output, header_output, output_types)

class FastCsv(FastBasic):
    """
    A faster version of the ordinary :class:`Csv` writer that uses the
    optimized C parsing engine. Note that this reader will append empty
    field values to the end of any row with not enough columns, while
    :class:`FastBasic` simply raises an error.
    """
    _format_name = 'fast_csv'
    _description = 'Comma-separated values table using the fast C engine'
    _fast = True
    fill_extra_cols = True

    def __init__(self, **kwargs):
        super().__init__({'delimiter': ',', 'comment': None}, **kwargs)

    def write(self, table, output):
        """
        Override the default write method of `FastBasic` to
        output masked values as empty fields.
        """
        self._write(table, output, {'fill_values': [(core.masked, '')]})

class FastTab(FastBasic):
    """
    A faster version of the ordinary :class:`Tab` reader that uses
    the optimized C parsing engine.
    """
    _format_name = 'fast_tab'
    _description = 'Tab-separated values table using the fast C engine'
    _fast = True

    def __init__(self, **kwargs):
        super().__init__({'delimiter': '\t'}, **kwargs)
        self.strip_whitespace_lines = False
        self.strip_whitespace_fields = False

class FastNoHeader(FastBasic):
    """
    This class uses the fast C engine to read tables with no header line. If
    the names parameter is unspecified, the columns will be autonamed with
    "col{}".
    """
    _format_name = 'fast_no_header'
    _description = 'Basic table with no headers using the fast C engine'
    _fast = True

    def __init__(self, **kwargs):
        super().__init__({'header_start': None, 'data_start': 0}, **kwargs)

    def write(self, table, output):
        """
        Override the default writing behavior in `FastBasic` so
        that columns names are not included in output.
        """
        self._write(table, output, {}, header_output=None)

class FastCommentedHeader(FastBasic):
    """
    A faster version of the :class:`CommentedHeader` reader, which looks for
    column names in a commented line. ``header_start`` denotes the index of
    the header line among all commented lines and is 0 by default.
    """
    _format_name = 'fast_commented_header'
    _description = 'Columns name in a commented line using the fast C engine'
    _fast = True

    def __init__(self, **kwargs):
        super().__init__({}, **kwargs)
        # Mimic CommentedHeader's behavior in which data_start
        # is relative to header_start if unspecified; see #2692
        if 'data_start' not in kwargs:
            self.data_start = 0

    def make_table(self, data, comments):
        """
        Actually make the output table give the data and comments.  This is
        slightly different from the base FastBasic method in the way comments
        are handled.
        """
        meta = OrderedDict()
        if comments:
            meta['comments'] = comments[1:]
            if not meta['comments']:
                del meta['comments']

        return Table(data, names=list(self.engine.get_names()), meta=meta)

    def _read_header(self):
        tmp = self.engine.source
        commented_lines = []

        for line in tmp.splitlines():
            line = line.lstrip()
            if line and line[0] == self.comment:  # line begins with a comment
                commented_lines.append(line[1:])
                if len(commented_lines) == self.header_start + 1:
                    break

        if len(commented_lines) <= self.header_start:
            raise cparser.CParserError('not enough commented lines')

        self.engine.setup_tokenizer([commented_lines[self.header_start]])
        self.engine.header_start = 0
        self.engine.read_header()
        self.engine.setup_tokenizer(tmp)

    def write(self, table, output):
        """
        Override the default writing behavior in `FastBasic` so
        that column names are commented.
        """
        self._write(table, output, {}, header_output='comment')

class FastRdb(FastBasic):
    """
    A faster version of the :class:`Rdb` reader. This format is similar to
    tab-delimited, but it also contains a header line after the column
    name line denoting the type of each column (N for numeric, S for string).
    """
    _format_name = 'fast_rdb'
    _description = 'Tab-separated with a type definition header line'
    _fast = True

    def __init__(self, **kwargs):
        super().__init__({'delimiter': '\t', 'data_start': 2}, **kwargs)
        self.strip_whitespace_lines = False
        self.strip_whitespace_fields = False

    def _read_header(self):
        tmp = self.engine.source
        line1 = ''
        line2 = ''
        for line in tmp.splitlines():
            # valid non-comment line
            if not line1 and line.strip() and line.lstrip()[0] != self.comment:
                line1 = line
            elif not line2 and line.strip() and line.lstrip()[0] != self.comment:
                line2 = line
                break
        else:  # less than 2 lines in table
            raise ValueError('RDB header requires 2 lines')

        # tokenize the two header lines separately
        self.engine.setup_tokenizer([line2])
        self.engine.header_start = 0
        self.engine.read_header()
        types = self.engine.get_names()
        self.engine.setup_tokenizer([line1])
        self.engine.set_names([])
        self.engine.read_header()

        if len(self.engine.get_names()) != len(types):
            raise ValueError('RDB header mismatch between number of '
                             'column names and column types')

        if any(not re.match(r'\d*(N|S)$', x, re.IGNORECASE) for x in types):
            raise ValueError('RDB type definitions do not all match '
                             '[num](N|S): {0}'.format(types))

        try_int = {}
        try_float = {}
        try_string = {}

        for name, col_type in zip(self.engine.get_names(), types):
            if col_type[-1].lower() == 's':
                try_int[name] = 0
                try_float[name] = 0
                try_string[name] = 1
            else:
                try_int[name] = 1
                try_float[name] = 1
                try_string[name] = 0

        self.engine.setup_tokenizer(tmp)
        return (try_int, try_float, try_string)

    def write(self, table, output):
        """
        Override the default writing behavior in `FastBasic` to
        output a line with column types after the column name line.
        """
        self._write(table, output, {}, output_types=True)
<|endoftext|>


<commit_before>from pathlib import Path
from typing import Sequence, Union, SupportsInt

from torch import Tensor

from avalanche.training.utils import IDatasetWithTargets, \
    ConcatDatasetWithTargets, TransformationTensorDataset
from .datasets_from_filelists import datasets_from_filelists
from .generic_cl_scenario import GenericCLScenario

def create_multi_dataset_generic_scenario(
        train_dataset_list: Sequence[IDatasetWithTargets],
        test_dataset_list: Sequence[IDatasetWithTargets],
        task_labels: Sequence[int],
        complete_test_set_only: bool = False) -> GenericCLScenario:
    """
    Creates a generic scenario given a list of datasets and the respective task
    labels. Each training dataset will be considered as a separate training
    step. Contents of the datasets will not be changed, including the targets.

    When loading the datasets from a set of fixed filelist, consider using
    the :func:`create_generic_scenario_from_filelists` helper method instead.

    In its base form, this function accepts a list of test datsets that must
    contain the same amount of datasets of the training list.
    Those pairs are then used to create the "past", "cumulative"
    (a.k.a. growing) and "future" test sets. However, in certain Continual
    Learning scenarios only the concept of "complete" test set makes sense. In
    that case, the ``complete_test_set_only`` should be set to True (see the
    parameter description for more info).

    Beware that pattern transformations must already be included in the
    datasets (when needed).

    :param train_dataset_list: A list of training datasets.
    :param test_dataset_list: A list of test datasets.
    :param task_labels: A list of task labels. Must contain the same amount of
        elements of the ``train_dataset_list`` parameter. For
        Single-Incremental-Task (a.k.a. Task-Free) scenarios, this is usually
        a list of zeros. For Multi Task scenario, this is usually a list of
        ascending task labels (starting from 0).
    :param complete_test_set_only: If True, only the complete test set will
        be returned by the scenario. This means that the ``test_dataset_list``
        parameter must be list with a single element (the complete test set).
        Defaults to False, which means that ``train_dataset_list`` and
        ``test_dataset_list`` must contain the same amount of datasets.

    :returns: A :class:`GenericCLScenario` instance.
    """

    # GenericCLScenario accepts a single training+test sets couple along with
    # the respective list of patterns indexes to include in each step.
    # This means that we have to concat the list of train and test sets
    # and create, for each step, a of indexes.
    # Each dataset describes a different step so the lists of indexes will
    # just be ranges of ascending indexes.
    train_structure = []
    concat_train_dataset = ConcatDatasetWithTargets(train_dataset_list)
    next_idx = 0
    for train_dataset in train_dataset_list:
        end_idx = next_idx + len(train_dataset)
        train_structure.append(range(next_idx, end_idx))
        next_idx = end_idx

    test_structure = []
    if complete_test_set_only:
        # If complete_test_set_only is True, we can leave test_structure = []
        # In this way, GenericCLScenario will consider the whole test set.
        #
        # We don't offer a way to reduce the test set here. However, consider
        # that the user may reduce the test set by creating a subset and passing
        # it to this function directly.
        if len(test_dataset_list) != 1:
            raise ValueError('Test must contain 1 element when'
                             'complete_test_set_only is True')
        concat_test_dataset = test_dataset_list[0]
    else:
        concat_test_dataset = ConcatDatasetWithTargets(test_dataset_list)
        test_structure = []
        next_idx = 0
        for test_dataset in test_dataset_list:
            end_idx = next_idx + len(test_dataset)
            test_structure.append(range(next_idx, end_idx))
            next_idx = end_idx

    # GenericCLScenario constructor will also check that the same amount of
    # train/test sets + task_labels have been defined.
    return GenericCLScenario(
        concat_train_dataset, concat_test_dataset, train_structure,
        test_structure, task_labels,
        return_complete_test_set_only=complete_test_set_only)

def create_generic_scenario_from_filelists(
        root: Union[str, Path],
        train_file_lists: Sequence[Union[str, Path]],
        test_file_lists: Union[Union[str, Path], Sequence[Union[str, Path]]],
        task_labels: Sequence[int],
        complete_test_set_only: bool = False,
        train_transform=None, train_target_transform=None,
        test_transform=None, test_target_transform=None) -> GenericCLScenario:
    """
    Creates a generic scenario given a list of filelists and the respective task
    labels. A separate dataset will be created for each filelist and each of
    those training datasets will be considered a separate training step.
    Contents of the datasets will not be changed, including the targets.

    In its base form, this function accepts a list of filelists for the test
    datsets that must contain the same amount of elements of the training list.
    Those pairs of datasets are then used to create the "past", "cumulative"
    (a.k.a. growing) and "future" test sets. However, in certain Continual
    Learning scenarios only the concept of "complete" test set makes sense. In
    that case, the ``complete_test_set_only`` should be set to True (see the
    parameter description for more info).

    This helper functions is the best shot when loading Caffe-style dataset
    based on filelists.

    :param root: The root path of the dataset.
    :param train_file_lists: A list of filelists describing the
        paths of the training patterns for each step.
    :param test_file_lists: A list of filelists describing the
        paths of the test patterns for each step.
    :param task_labels: A list of task labels. Must contain the same amount of
        elements of the ``train_file_lists`` parameter. For
        Single-Incremental-Task (a.k.a. Task-Free) scenarios, this is usually
        a list of zeros. For Multi Task scenario, this is usually a list of
        ascending task labels (starting from 0).
    :param complete_test_set_only: If True, only the complete test set will
        be returned by the scenario. This means that the ``test_file_lists``
        parameter must be list with a single element (the complete test set).
        Alternatively, can be a plain string or :class:`Path` object.
        Defaults to False, which means that ``train_file_lists`` and
        ``test_file_lists`` must contain the same amount of filelists paths.
    :param train_transform: The transformation to apply to training patterns.
        Defaults to None.
    :param train_target_transform: The transformation to apply to training
        patterns targets. Defaults to None.
    :param test_transform: The transformation to apply to test patterns.
        Defaults to None.
    :param test_target_transform: The transformation to apply to test
        patterns targets. Defaults to None.

    :returns: A :class:`GenericCLScenario` instance.
    """

    train_datasets, test_dataset = datasets_from_filelists(
        root, train_file_lists, test_file_lists,
        complete_test_set_only=complete_test_set_only,
        train_transform=train_transform,
        train_target_transform=train_target_transform,
        test_transform=test_transform,
        test_target_transform=test_target_transform)

    return create_multi_dataset_generic_scenario(
        train_datasets, test_dataset, task_labels,
        complete_test_set_only=complete_test_set_only)

def create_generic_scenario_from_tensors(
        train_datasets_x: Sequence[Tensor],
        train_datasets_y: Sequence[Sequence[SupportsInt]],
        test_datasets_x: Sequence[Tensor],
        test_datasets_y: Sequence[Sequence[SupportsInt]],
        task_labels: Sequence[int],
        complete_test_set_only: bool = False,
        train_transform=None, train_target_transform=None,
        test_transform=None, test_target_transform=None) -> GenericCLScenario:
    """
    Creates a generic scenario given lists of Tensors and the respective task
    labels. A separate dataset will be created from each Tensor pair (x + y)
    and each of those training datasets will be considered a separate
    training step. Contents of the datasets will not be changed, including the
    targets. Using this helper function is the lower level way to create a
    Continual Learning scenario. When possible, consider using higher level
    helpers.

    In its base form, the test lists must contain the same amount of elements of
    the training lists. Those pairs of datasets are then used to create the
    "past", "cumulative" (a.k.a. growing) and "future" test sets.
    However, in certain Continual Learning scenarios only the concept of
    "complete" test set makes sense. In that case, the
    ``complete_test_set_only`` should be set to True (see the parameter
    description for more info).

    :param train_datasets_x: A list of Tensors (one per step) containing the
        patterns of the training sets.
    :param train_datasets_y: A list of Tensors or int lists containing the
        labels of the patterns of the training sets. Must contain the same
        number of elements of ``train_datasets_x``.
    :param test_datasets_x: A list of Tensors (one per step) containing the
        patterns of the test sets.
    :param test_datasets_y: A list of Tensors or int lists containing the
        labels of the patterns of the test sets. Must contain the same
        number of elements of ``test_datasets_x``.
    :param task_labels: A list of task labels. Must contain the same amount of
        elements of the ``train_datasets_x`` parameter. For
        Single-Incremental-Task (a.k.a. Task-Free) scenarios, this is usually
        a list of zeros. For Multi Task scenario, this is usually a list of
        ascending task labels (starting from 0).
    :param complete_test_set_only: If True, only the complete test set will
        be returned by the scenario. This means that the ``test_datasets_x`` and
        ``test_datasets_y`` parameters must be lists with a single element
        (the complete test set). Defaults to False, which means that
        ``train_file_lists`` and ``test_file_lists`` must contain the same
        amount of filelists paths.
    :param train_transform: The transformation to apply to training patterns.
        Defaults to None.
    :param train_target_transform: The transformation to apply to training
        patterns targets. Defaults to None.
    :param test_transform: The transformation to apply to test patterns.
        Defaults to None.
    :param test_target_transform: The transformation to apply to test
        patterns targets. Defaults to None.

    :returns: A :class:`GenericCLScenario` instance.
    """

    if len(train_datasets_x) != len(train_datasets_y):
        raise ValueError('train_datasets_x and train_datasets_y must contain'
                         'the same amount of elements')

    if len(test_datasets_x) != len(test_datasets_y):
        raise ValueError('test_datasets_x and test_datasets_y must contain'
                         'the same amount of elements')

    train_datasets = [
        TransformationTensorDataset(
            dataset_x, dataset_y, transform=train_transform,
            target_transform=train_target_transform)
        for dataset_x, dataset_y in
        zip(train_datasets_x, train_datasets_y)]

    test_datasets = [
        TransformationTensorDataset(
            dataset_x, dataset_y, transform=test_transform,
            target_transform=test_target_transform)
        for dataset_x, dataset_y in
        zip(test_datasets_x, test_datasets_y)]

    return create_multi_dataset_generic_scenario(
        train_datasets, test_datasets, task_labels,
        complete_test_set_only=complete_test_set_only)

__all__ = ['create_multi_dataset_generic_scenario',
           'create_generic_scenario_from_filelists',
           'create_generic_scenario_from_tensors']
<commit_msg>Changed parameter names of create_generic_scenario_from_tensors<commit_after>from pathlib import Path
from typing import Sequence, Union, SupportsInt

from torch import Tensor

from avalanche.training.utils import IDatasetWithTargets, \
    ConcatDatasetWithTargets, TransformationTensorDataset
from .datasets_from_filelists import datasets_from_filelists
from .generic_cl_scenario import GenericCLScenario

def create_multi_dataset_generic_scenario(
        train_dataset_list: Sequence[IDatasetWithTargets],
        test_dataset_list: Sequence[IDatasetWithTargets],
        task_labels: Sequence[int],
        complete_test_set_only: bool = False) -> GenericCLScenario:
    """
    Creates a generic scenario given a list of datasets and the respective task
    labels. Each training dataset will be considered as a separate training
    step. Contents of the datasets will not be changed, including the targets.

    When loading the datasets from a set of fixed filelist, consider using
    the :func:`create_generic_scenario_from_filelists` helper method instead.

    In its base form, this function accepts a list of test datsets that must
    contain the same amount of datasets of the training list.
    Those pairs are then used to create the "past", "cumulative"
    (a.k.a. growing) and "future" test sets. However, in certain Continual
    Learning scenarios only the concept of "complete" test set makes sense. In
    that case, the ``complete_test_set_only`` should be set to True (see the
    parameter description for more info).

    Beware that pattern transformations must already be included in the
    datasets (when needed).

    :param train_dataset_list: A list of training datasets.
    :param test_dataset_list: A list of test datasets.
    :param task_labels: A list of task labels. Must contain the same amount of
        elements of the ``train_dataset_list`` parameter. For
        Single-Incremental-Task (a.k.a. Task-Free) scenarios, this is usually
        a list of zeros. For Multi Task scenario, this is usually a list of
        ascending task labels (starting from 0).
    :param complete_test_set_only: If True, only the complete test set will
        be returned by the scenario. This means that the ``test_dataset_list``
        parameter must be list with a single element (the complete test set).
        Defaults to False, which means that ``train_dataset_list`` and
        ``test_dataset_list`` must contain the same amount of datasets.

    :returns: A :class:`GenericCLScenario` instance.
    """

    # GenericCLScenario accepts a single training+test sets couple along with
    # the respective list of patterns indexes to include in each step.
    # This means that we have to concat the list of train and test sets
    # and create, for each step, a of indexes.
    # Each dataset describes a different step so the lists of indexes will
    # just be ranges of ascending indexes.
    train_structure = []
    concat_train_dataset = ConcatDatasetWithTargets(train_dataset_list)
    next_idx = 0
    for train_dataset in train_dataset_list:
        end_idx = next_idx + len(train_dataset)
        train_structure.append(range(next_idx, end_idx))
        next_idx = end_idx

    test_structure = []
    if complete_test_set_only:
        # If complete_test_set_only is True, we can leave test_structure = []
        # In this way, GenericCLScenario will consider the whole test set.
        #
        # We don't offer a way to reduce the test set here. However, consider
        # that the user may reduce the test set by creating a subset and passing
        # it to this function directly.
        if len(test_dataset_list) != 1:
            raise ValueError('Test must contain 1 element when'
                             'complete_test_set_only is True')
        concat_test_dataset = test_dataset_list[0]
    else:
        concat_test_dataset = ConcatDatasetWithTargets(test_dataset_list)
        test_structure = []
        next_idx = 0
        for test_dataset in test_dataset_list:
            end_idx = next_idx + len(test_dataset)
            test_structure.append(range(next_idx, end_idx))
            next_idx = end_idx

    # GenericCLScenario constructor will also check that the same amount of
    # train/test sets + task_labels have been defined.
    return GenericCLScenario(
        concat_train_dataset, concat_test_dataset, train_structure,
        test_structure, task_labels,
        return_complete_test_set_only=complete_test_set_only)

def create_generic_scenario_from_filelists(
        root: Union[str, Path],
        train_file_lists: Sequence[Union[str, Path]],
        test_file_lists: Union[Union[str, Path], Sequence[Union[str, Path]]],
        task_labels: Sequence[int],
        complete_test_set_only: bool = False,
        train_transform=None, train_target_transform=None,
        test_transform=None, test_target_transform=None) -> GenericCLScenario:
    """
    Creates a generic scenario given a list of filelists and the respective task
    labels. A separate dataset will be created for each filelist and each of
    those training datasets will be considered a separate training step.
    Contents of the datasets will not be changed, including the targets.

    In its base form, this function accepts a list of filelists for the test
    datsets that must contain the same amount of elements of the training list.
    Those pairs of datasets are then used to create the "past", "cumulative"
    (a.k.a. growing) and "future" test sets. However, in certain Continual
    Learning scenarios only the concept of "complete" test set makes sense. In
    that case, the ``complete_test_set_only`` should be set to True (see the
    parameter description for more info).

    This helper functions is the best shot when loading Caffe-style dataset
    based on filelists.

    :param root: The root path of the dataset.
    :param train_file_lists: A list of filelists describing the
        paths of the training patterns for each step.
    :param test_file_lists: A list of filelists describing the
        paths of the test patterns for each step.
    :param task_labels: A list of task labels. Must contain the same amount of
        elements of the ``train_file_lists`` parameter. For
        Single-Incremental-Task (a.k.a. Task-Free) scenarios, this is usually
        a list of zeros. For Multi Task scenario, this is usually a list of
        ascending task labels (starting from 0).
    :param complete_test_set_only: If True, only the complete test set will
        be returned by the scenario. This means that the ``test_file_lists``
        parameter must be list with a single element (the complete test set).
        Alternatively, can be a plain string or :class:`Path` object.
        Defaults to False, which means that ``train_file_lists`` and
        ``test_file_lists`` must contain the same amount of filelists paths.
    :param train_transform: The transformation to apply to training patterns.
        Defaults to None.
    :param train_target_transform: The transformation to apply to training
        patterns targets. Defaults to None.
    :param test_transform: The transformation to apply to test patterns.
        Defaults to None.
    :param test_target_transform: The transformation to apply to test
        patterns targets. Defaults to None.

    :returns: A :class:`GenericCLScenario` instance.
    """

    train_datasets, test_dataset = datasets_from_filelists(
        root, train_file_lists, test_file_lists,
        complete_test_set_only=complete_test_set_only,
        train_transform=train_transform,
        train_target_transform=train_target_transform,
        test_transform=test_transform,
        test_target_transform=test_target_transform)

    return create_multi_dataset_generic_scenario(
        train_datasets, test_dataset, task_labels,
        complete_test_set_only=complete_test_set_only)

def create_generic_scenario_from_tensors(
        train_data_x: Sequence[Tensor],
        train_data_y: Sequence[Sequence[SupportsInt]],
        test_data_x: Sequence[Tensor],
        test_data_y: Sequence[Sequence[SupportsInt]],
        task_labels: Sequence[int],
        complete_test_set_only: bool = False,
        train_transform=None, train_target_transform=None,
        test_transform=None, test_target_transform=None) -> GenericCLScenario:
    """
    Creates a generic scenario given lists of Tensors and the respective task
    labels. A separate dataset will be created from each Tensor pair (x + y)
    and each of those training datasets will be considered a separate
    training step. Contents of the datasets will not be changed, including the
    targets. Using this helper function is the lower level way to create a
    Continual Learning scenario. When possible, consider using higher level
    helpers.

    In its base form, the test lists must contain the same amount of elements of
    the training lists. Those pairs of datasets are then used to create the
    "past", "cumulative" (a.k.a. growing) and "future" test sets.
    However, in certain Continual Learning scenarios only the concept of
    "complete" test set makes sense. In that case, the
    ``complete_test_set_only`` should be set to True (see the parameter
    description for more info).

    :param train_data_x: A list of Tensors (one per step) containing the
        patterns of the training sets.
    :param train_data_y: A list of Tensors or int lists containing the
        labels of the patterns of the training sets. Must contain the same
        number of elements of ``train_datasets_x``.
    :param test_data_x: A list of Tensors (one per step) containing the
        patterns of the test sets.
    :param test_data_y: A list of Tensors or int lists containing the
        labels of the patterns of the test sets. Must contain the same
        number of elements of ``test_datasets_x``.
    :param task_labels: A list of task labels. Must contain the same amount of
        elements of the ``train_datasets_x`` parameter. For
        Single-Incremental-Task (a.k.a. Task-Free) scenarios, this is usually
        a list of zeros. For Multi Task scenario, this is usually a list of
        ascending task labels (starting from 0).
    :param complete_test_set_only: If True, only the complete test set will
        be returned by the scenario. This means that the ``test_datasets_x`` and
        ``test_datasets_y`` parameters must be lists with a single element
        (the complete test set). Defaults to False, which means that
        ``train_file_lists`` and ``test_file_lists`` must contain the same
        amount of filelists paths.
    :param train_transform: The transformation to apply to training patterns.
        Defaults to None.
    :param train_target_transform: The transformation to apply to training
        patterns targets. Defaults to None.
    :param test_transform: The transformation to apply to test patterns.
        Defaults to None.
    :param test_target_transform: The transformation to apply to test
        patterns targets. Defaults to None.

    :returns: A :class:`GenericCLScenario` instance.
    """

    if len(train_data_x) != len(train_data_y):
        raise ValueError('train_data_x and train_data_y must contain'
                         'the same amount of elements')

    if len(test_data_x) != len(test_data_y):
        raise ValueError('test_data_x and test_data_y must contain'
                         'the same amount of elements')

    train_datasets = [
        TransformationTensorDataset(
            dataset_x, dataset_y, transform=train_transform,
            target_transform=train_target_transform)
        for dataset_x, dataset_y in
        zip(train_data_x, train_data_y)]

    test_datasets = [
        TransformationTensorDataset(
            dataset_x, dataset_y, transform=test_transform,
            target_transform=test_target_transform)
        for dataset_x, dataset_y in
        zip(test_data_x, test_data_y)]

    return create_multi_dataset_generic_scenario(
        train_datasets, test_datasets, task_labels,
        complete_test_set_only=complete_test_set_only)

__all__ = ['create_multi_dataset_generic_scenario',
           'create_generic_scenario_from_filelists',
           'create_generic_scenario_from_tensors']
<|endoftext|>