<commit_before>######################
# (c) 2012 Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD 3-clause
#
# Implements structured SVM as described in Tsochantaridis et. al.
# Support Vector Machines Learning for Interdependent
# and Structures Output Spaces

import numpy as np
import cvxopt
import cvxopt.solvers

from sklearn.externals.joblib import Parallel, delayed
from sklearn.utils import gen_even_slices

from .ssvm import BaseSSVM
from ..utils import unwrap_pairwise, find_constraint

class NSlackSSVM(BaseSSVM):
    """Structured SVM solver for the n-slack QP with l1 slack penalty.

    Implements margin rescaled structural SVM using
    the n-slack formulation and cutting plane method, solved using CVXOPT.
    The optimization is restarted in each iteration.

    Parameters
    ----------
    model : StructuredModel
        Object containing the model structure. Has to implement
        `loss`, `inference` and `loss_augmented_inference`.

    max_iter : int
        Maximum number of passes over dataset to find constraints.

    C : float
        Regularization parameter

    check_constraints : bool (default=True)
        Whether to check if the new "most violated constraint" is
        more violated than previous constraints. Helpful for stopping
        and debugging, but costly.

    verbose : int (default=0)
        Verbosity.

    positive_constraint: list of ints
        Indices of parmeters that are constraint to be positive.

    break_on_bad: bool (default=False)
        Whether to break (start debug mode) when inference was approximate.

    n_jobs : int, default=1
        Number of parallel jobs for inference. -1 means as many as cpus.

    show_loss_every : int, default=0
        Controlls how often the hamming loss is computed (for monitoring
        purposes). Zero means never, otherwise it will be computed very
        show_loss_every'th epoch.

    batch_size : int, default=100
        Number of constraints after which we solve the QP again.
        batch_size=-1 means that an update is performed only after going once
        over the whole training set.

    tol : float, default=-10
        Convergence tolerance. If dual objective decreases less than tol,
        learning is stopped. The default corresponds to ignoring the behavior
        of the dual objective and stop only if no more constraints can be
        found.

    inactive_threshold : float, default=1e-5
        Threshold for dual variable of a constraint to be considered inactive.

    inactive_window : float, default=50
        Window for measuring inactivity. If a constraint is inactive for
        ``inactive_window`` iterations, it will be pruned from the QP.
        If set to 0, no constraints will be removed.

    Attributes
    ----------
    w : nd-array, shape=(model.size_psi,)
        The learned weights of the SVM.

    old_solution : dict
        The last solution found by the qp solver.

   ``loss_curve_`` : list of float
        List of loss values if show_loss_every > 0.

   ``objective_curve_`` : list of float
       Primal objective after each pass through the dataset.
    """

    def __init__(self, model, max_iter=100, C=1.0, check_constraints=True,
                 verbose=0, positive_constraint=None, n_jobs=1,
                 break_on_bad=False, show_loss_every=0, batch_size=100,
                 tol=-10, inactive_threshold=1e-10,
                 inactive_window=0, logger=None):

        BaseSSVM.__init__(self, model, max_iter, C, verbose=verbose,
                          n_jobs=n_jobs, show_loss_every=show_loss_every,
                          logger=logger)

        self.positive_constraint = positive_constraint
        self.check_constraints = check_constraints
        self.break_on_bad = break_on_bad
        self.batch_size = batch_size
        self.tol = tol
        self.inactive_threshold = inactive_threshold
        self.inactive_window = inactive_window

    def _solve_n_slack_qp(self, constraints, n_samples):
        C = self.C
        psis = [c[1] for sample in constraints for c in sample]
        losses = [c[2] for sample in constraints for c in sample]

        psi_matrix = np.vstack(psis)
        n_constraints = len(psis)
        P = cvxopt.matrix(np.dot(psi_matrix, psi_matrix.T))
        # q contains loss from margin-rescaling
        q = cvxopt.matrix(-np.array(losses, dtype=np.float))
        # constraints are a bit tricky. first, all alpha must be >zero
        idy = np.identity(n_constraints)
        tmp1 = np.zeros(n_constraints)
        # box constraint: sum of all alpha for one example must be <= C
        blocks = np.zeros((n_samples, n_constraints))
        first = 0
        for i, sample in enumerate(constraints):
            blocks[i, first: first + len(sample)] = 1
            first += len(sample)
        # positivity constraints:
        if self.positive_constraint is None:
            #empty constraints
            zero_constr = np.zeros(0)
            psis_constr = np.zeros((0, n_constraints))
        else:
            psis_constr = psi_matrix.T[self.positive_constraint]
            zero_constr = np.zeros(len(self.positive_constraint))

        # put together
        G = cvxopt.sparse(cvxopt.matrix(np.vstack((-idy, blocks,
                                                   psis_constr))))
        tmp2 = np.ones(n_samples) * C
        h = cvxopt.matrix(np.hstack((tmp1, tmp2, zero_constr)))

        # solve QP model
        cvxopt.solvers.options['feastol'] = 1e-5
        solution = cvxopt.solvers.qp(P, q, G, h)
        if solution['status'] != "optimal":
            print("regularizing QP!")
            P = cvxopt.matrix(np.dot(psi_matrix, psi_matrix.T)
                              + 1e-8 * np.eye(psi_matrix.shape[0]))
            solution = cvxopt.solvers.qp(P, q, G, h)
            if solution['status'] != "optimal":
                raise ValueError("QP solver failed. Try regularizing your QP.")

        # Lagrange multipliers
        a = np.ravel(solution['x'])
        self.prune_constraints(constraints, a)
        self.old_solution = solution

        # Support vectors have non zero lagrange multipliers
        sv = a > 1e-10
        box = np.dot(blocks, a)
        if self.verbose > 1:
            print("%d support vectors out of %d points" % (np.sum(sv),
                                                           n_constraints))
            # calculate per example box constraint:
            print("Box constraints at C: %d" % np.sum(1 - box / C < 1e-3))
            print("dual objective: %f" % solution['primal objective'])
        self.w = np.dot(a, psi_matrix)
        return -solution['primal objective']

    def _check_bad_constraint(self, y_hat, slack, old_constraints):
        if slack < 1e-5:
            return True
        y_hat_plain = unwrap_pairwise(y_hat)

        already_active = np.any([True for y__, _, _ in old_constraints
                                 if np.all(y_hat_plain ==
                                           unwrap_pairwise(y__))])
        if already_active:
            return True

        # "smart" stopping criterion
        # check if most violated constraint is more violated
        # than previous ones by more then eps.
        # If it is less violated, inference was wrong/approximate
        if self.check_constraints:
            for con in old_constraints:
                # compute slack for old constraint
                slack_tmp = max(con[2] - np.dot(self.w, con[1]), 0)
                if self.verbose > 5:
                    print("slack old constraint: %f" % slack_tmp)
                # if slack of new constraint is smaller or not
                # significantly larger, don't add constraint.
                # if smaller, complain about approximate inference.
                if slack - slack_tmp < -1e-5:
                    print("bad inference: %f" % (slack_tmp - slack))
                    if self.break_on_bad:
                        raise ValueError("bad inference: %f" % (slack_tmp -
                                                                slack))
                    return True

        return False

    def fit(self, X, Y, constraints=None, warm_start=None):
        """Learn parameters using cutting plane method.

        Parameters
        ----------
        X : iterable
            Traing instances. Contains the structured input objects.
            No requirement on the particular form of entries of X is made.

        Y : iterable
            Training labels. Contains the strctured labels for inputs in X.
            Needs to have the same length as X.

        contraints : iterable
            Known constraints for warm-starts. List of same length as X.
            Each entry is itself a list of constraints for a given instance x .
            Each constraint is of the form [y_hat, delta_psi, loss], where
            y_hat is a labeling, ``delta_psi = psi(x, y) - psi(x, y_hat)``
            and loss is the loss for predicting y_hat instead of the true label
            y.
        """
        print("Training n-slack dual structural SVM")
        if self.verbose < 2:
            cvxopt.solvers.options['show_progress'] = False
        else:
            cvxopt.solvers.options['show_progress'] = True

        self.w = np.zeros(self.model.size_psi)
        n_samples = len(X)
        if constraints is None:
            constraints = [[] for i in xrange(n_samples)]
        else:
            objective = self._solve_n_slack_qp(constraints, n_samples)
        loss_curve = []
        objective_curve = []
        self.alphas = []  # dual solutions
        # we have to update at least once after going through the dataset
        for iteration in xrange(self.max_iter):
            # main loop
            if self.verbose > 0:
                print("iteration %d" % iteration)
            new_constraints = 0
            # generate slices through dataset from batch_size
            if self.batch_size < 1 and not self.batch_size == -1:
                raise ValueError("batch_size should be integer >= 1 or -1,"
                                 "got %s." % str(self.batch_size))
            batch_size = self.batch_size if self.batch_size != -1 else len(X)
            n_batches = int(np.ceil(float(len(X)) / batch_size))
            slices = gen_even_slices(n_samples, n_batches)
            indices = np.arange(n_samples)
            for batch in slices:
                new_constraints_batch = 0
                verbose = max(0, self.verbose - 3)
                X_b = X[batch]
                Y_b = Y[batch]
                indices_b = indices[batch]
                candidate_constraints = Parallel(n_jobs=self.n_jobs,
                                                 verbose=verbose)(
                                                     delayed(find_constraint)(
                                                         self.model, x, y,
                                                         self.w)
                                                     for x, y in zip(X_b, Y_b))

                # for each slice, gather new constraints
                for i, x, y, constraint in zip(indices_b, X_b, Y_b,
                                               candidate_constraints):
                    # loop over dataset
                    y_hat, delta_psi, slack, loss = constraint

                    if self.verbose > 3:
                        print("current slack: %f" % slack)

                    if not loss > 0:
                        # can have y != y_hat but loss = 0 in latent svm.
                        # we need this here as dpsi is then != 0
                        continue

                    if self._check_bad_constraint(y_hat, slack,
                                                  constraints[i]):
                        continue

                    constraints[i].append([y_hat, delta_psi, loss])
                    new_constraints_batch += 1

                # after processing the slice, solve the qp
                if new_constraints_batch:
                    objective = self._solve_n_slack_qp(constraints, n_samples)
                    objective_curve.append(objective)
                    new_constraints += new_constraints_batch

            if new_constraints == 0:
                print("no additional constraints")
                break

            self._compute_training_loss(X, Y, iteration)

            if self.verbose > 0:
                print("new constraints: %d, "
                      "dual objective: %f" %
                      (new_constraints,
                       objective))
            if (iteration > 1 and objective_curve[-1]
                    - objective_curve[-2] < self.tol):
                print("objective converged.")
                break
            if self.verbose > 5:
                print(self.w)

            if self.logger is not None:
                self.logger(self, iteration)

        self.constraints_ = constraints
        self.loss_curve_ = loss_curve
        self.objective_curve_ = objective_curve
        if self.verbose and self.n_jobs == 1:
            print("calls to inference: %d" % self.model.inference_calls)
        return self

    def prune_constraints(self, constraints, a):
        pass
        # FIXME
        # append list for new constraint
        #self.alphas.extend([[] for i in xrange(len(a) - len(self.alphas))])
        #flat_constraints = [constraint for sample in constraints for
        #constraint #in sample]
        #assert(len(self.alphas) == len(flat_constraints))
        #for constraint, alpha in zip(self.alphas, a):
            #constraint.append(alpha)
            #constraint = constraint[-self.inactive_window:]

        ## prune unused constraints:
        ## if the max of alpha in last 50 iterations was small, throw away
        #if self.inactive_window != 0:
            #max_active = [np.max(constr[-self.inactive_window:])
                          #for constr in self.alphas]
            ## find strongest constraint that is not ground truth constraint
            #strongest = np.max(max_active[1:])
            #inactive = max_active < self.inactive_threshold * strongest
            #idx = 0
            #for sample in constraints:
                #for i, const in reversed(list(enumerate(sample))):
                    #if inactive[idx]:
                        #del const[i]
                        #del self.alphas[idx]
                    #idx += 1
<commit_msg>starting<commit_after>######################
# (c) 2012 Andreas Mueller <amueller@ais.uni-bonn.de>
# License: BSD 3-clause
#
# Implements structured SVM as described in Tsochantaridis et. al.
# Support Vector Machines Learning for Interdependent
# and Structures Output Spaces

import numpy as np
import cvxopt
import cvxopt.solvers

from sklearn.externals.joblib import Parallel, delayed
from sklearn.utils import gen_even_slices

from .ssvm import BaseSSVM
from ..utils import unwrap_pairwise, find_constraint

class NSlackSSVM(BaseSSVM):
    """Structured SVM solver for the n-slack QP with l1 slack penalty.

    Implements margin rescaled structural SVM using
    the n-slack formulation and cutting plane method, solved using CVXOPT.
    The optimization is restarted in each iteration.

    Parameters
    ----------
    model : StructuredModel
        Object containing the model structure. Has to implement
        `loss`, `inference` and `loss_augmented_inference`.

    max_iter : int
        Maximum number of passes over dataset to find constraints.

    C : float
        Regularization parameter

    check_constraints : bool (default=True)
        Whether to check if the new "most violated constraint" is
        more violated than previous constraints. Helpful for stopping
        and debugging, but costly.

    verbose : int (default=0)
        Verbosity.

    positive_constraint: list of ints
        Indices of parmeters that are constraint to be positive.

    break_on_bad: bool (default=False)
        Whether to break (start debug mode) when inference was approximate.

    n_jobs : int, default=1
        Number of parallel jobs for inference. -1 means as many as cpus.

    show_loss_every : int, default=0
        Controlls how often the hamming loss is computed (for monitoring
        purposes). Zero means never, otherwise it will be computed very
        show_loss_every'th epoch.

    batch_size : int, default=100
        Number of constraints after which we solve the QP again.
        batch_size=-1 means that an update is performed only after going once
        over the whole training set.

    tol : float, default=-10
        Convergence tolerance. If dual objective decreases less than tol,
        learning is stopped. The default corresponds to ignoring the behavior
        of the dual objective and stop only if no more constraints can be
        found.

    inactive_threshold : float, default=1e-5
        Threshold for dual variable of a constraint to be considered inactive.

    inactive_window : float, default=50
        Window for measuring inactivity. If a constraint is inactive for
        ``inactive_window`` iterations, it will be pruned from the QP.
        If set to 0, no constraints will be removed.

    Attributes
    ----------
    w : nd-array, shape=(model.size_psi,)
        The learned weights of the SVM.

    old_solution : dict
        The last solution found by the qp solver.

   ``loss_curve_`` : list of float
        List of loss values if show_loss_every > 0.

   ``objective_curve_`` : list of float
       Primal objective after each pass through the dataset.
    """

    def __init__(self, model, max_iter=100, C=1.0, check_constraints=True,
                 verbose=0, positive_constraint=None, n_jobs=1,
                 break_on_bad=False, show_loss_every=0, batch_size=100,
                 tol=-10, inactive_threshold=1e-10,
                 inactive_window=0, logger=None):

        BaseSSVM.__init__(self, model, max_iter, C, verbose=verbose,
                          n_jobs=n_jobs, show_loss_every=show_loss_every,
                          logger=logger)

        self.positive_constraint = positive_constraint
        self.check_constraints = check_constraints
        self.break_on_bad = break_on_bad
        self.batch_size = batch_size
        self.tol = tol
        self.inactive_threshold = inactive_threshold
        self.inactive_window = inactive_window

    def _solve_n_slack_qp(self, constraints, n_samples):
        C = self.C
        psis = [c[1] for sample in constraints for c in sample]
        losses = [c[2] for sample in constraints for c in sample]

        psi_matrix = np.vstack(psis)
        n_constraints = len(psis)
        P = cvxopt.matrix(np.dot(psi_matrix, psi_matrix.T))
        # q contains loss from margin-rescaling
        q = cvxopt.matrix(-np.array(losses, dtype=np.float))
        # constraints are a bit tricky. first, all alpha must be >zero
        idy = np.identity(n_constraints)
        tmp1 = np.zeros(n_constraints)
        # box constraint: sum of all alpha for one example must be <= C
        blocks = np.zeros((n_samples, n_constraints))
        first = 0
        for i, sample in enumerate(constraints):
            blocks[i, first: first + len(sample)] = 1
            first += len(sample)
        # positivity constraints:
        if self.positive_constraint is None:
            #empty constraints
            zero_constr = np.zeros(0)
            psis_constr = np.zeros((0, n_constraints))
        else:
            psis_constr = psi_matrix.T[self.positive_constraint]
            zero_constr = np.zeros(len(self.positive_constraint))

        # put together
        G = cvxopt.sparse(cvxopt.matrix(np.vstack((-idy, blocks,
                                                   psis_constr))))
        tmp2 = np.ones(n_samples) * C
        h = cvxopt.matrix(np.hstack((tmp1, tmp2, zero_constr)))

        # solve QP model
        cvxopt.solvers.options['feastol'] = 1e-5
        solution = cvxopt.solvers.qp(P, q, G, h)
        if solution['status'] != "optimal":
            print("regularizing QP!")
            P = cvxopt.matrix(np.dot(psi_matrix, psi_matrix.T)
                              + 1e-8 * np.eye(psi_matrix.shape[0]))
            solution = cvxopt.solvers.qp(P, q, G, h)
            if solution['status'] != "optimal":
                raise ValueError("QP solver failed. Try regularizing your QP.")

        # Lagrange multipliers
        a = np.ravel(solution['x'])
        self.prune_constraints(constraints, a)
        self.old_solution = solution

        # Support vectors have non zero lagrange multipliers
        sv = a > 1e-10
        box = np.dot(blocks, a)
        if self.verbose > 1:
            print("%d support vectors out of %d points" % (np.sum(sv),
                                                           n_constraints))
            # calculate per example box constraint:
            print("Box constraints at C: %d" % np.sum(1 - box / C < 1e-3))
            print("dual objective: %f" % solution['primal objective'])
        self.w = np.dot(a, psi_matrix)
        return -solution['primal objective']

    def _check_bad_constraint(self, y_hat, slack, old_constraints):
        if slack < 1e-5:
            return True
        y_hat_plain = unwrap_pairwise(y_hat)

        already_active = np.any([True for y__, _, _ in old_constraints
                                 if np.all(y_hat_plain ==
                                           unwrap_pairwise(y__))])
        if already_active:
            return True

        # "smart" stopping criterion
        # check if most violated constraint is more violated
        # than previous ones by more then eps.
        # If it is less violated, inference was wrong/approximate
        if self.check_constraints:
            for con in old_constraints:
                # compute slack for old constraint
                slack_tmp = max(con[2] - np.dot(self.w, con[1]), 0)
                if self.verbose > 5:
                    print("slack old constraint: %f" % slack_tmp)
                # if slack of new constraint is smaller or not
                # significantly larger, don't add constraint.
                # if smaller, complain about approximate inference.
                if slack - slack_tmp < -1e-5:
                    print("bad inference: %f" % (slack_tmp - slack))
                    if self.break_on_bad:
                        raise ValueError("bad inference: %f" % (slack_tmp -
                                                                slack))
                    return True

        return False

    def fit(self, X, Y, constraints=None, warm_start=None):
        """Learn parameters using cutting plane method.

        Parameters
        ----------
        X : iterable
            Traing instances. Contains the structured input objects.
            No requirement on the particular form of entries of X is made.

        Y : iterable
            Training labels. Contains the strctured labels for inputs in X.
            Needs to have the same length as X.

        contraints : iterable
            Known constraints for warm-starts. List of same length as X.
            Each entry is itself a list of constraints for a given instance x .
            Each constraint is of the form [y_hat, delta_psi, loss], where
            y_hat is a labeling, ``delta_psi = psi(x, y) - psi(x, y_hat)``
            and loss is the loss for predicting y_hat instead of the true label
            y.
        """
        print("Training n-slack dual structural SVM")
        if self.verbose < 2:
            cvxopt.solvers.options['show_progress'] = False
        else:
            cvxopt.solvers.options['show_progress'] = True

        self.w = np.zeros(self.model.size_psi)
        n_samples = len(X)
        if constraints is None:
            constraints = [[] for i in xrange(n_samples)]
        else:
            objective = self._solve_n_slack_qp(constraints, n_samples)
        loss_curve = []
        objective_curve = []
        self.alphas = [[] for i in xrange(n_samples)]  # dual solutions
        # we have to update at least once after going through the dataset
        for iteration in xrange(self.max_iter):
            # main loop
            if self.verbose > 0:
                print("iteration %d" % iteration)
            new_constraints = 0
            # generate slices through dataset from batch_size
            if self.batch_size < 1 and not self.batch_size == -1:
                raise ValueError("batch_size should be integer >= 1 or -1,"
                                 "got %s." % str(self.batch_size))
            batch_size = self.batch_size if self.batch_size != -1 else len(X)
            n_batches = int(np.ceil(float(len(X)) / batch_size))
            slices = gen_even_slices(n_samples, n_batches)
            indices = np.arange(n_samples)
            for batch in slices:
                new_constraints_batch = 0
                verbose = max(0, self.verbose - 3)
                X_b = X[batch]
                Y_b = Y[batch]
                indices_b = indices[batch]
                candidate_constraints = Parallel(n_jobs=self.n_jobs,
                                                 verbose=verbose)(
                                                     delayed(find_constraint)(
                                                         self.model, x, y,
                                                         self.w)
                                                     for x, y in zip(X_b, Y_b))

                # for each batch, gather new constraints
                for i, x, y, constraint in zip(indices_b, X_b, Y_b,
                                               candidate_constraints):
                    # loop over samples in batch
                    y_hat, delta_psi, slack, loss = constraint

                    if self.verbose > 3:
                        print("current slack: %f" % slack)

                    if not loss > 0:
                        # can have y != y_hat but loss = 0 in latent svm.
                        # we need this here as dpsi is then != 0
                        continue

                    if self._check_bad_constraint(y_hat, slack,
                                                  constraints[i]):
                        continue

                    constraints[i].append([y_hat, delta_psi, loss])
                    new_constraints_batch += 1

                # after processing the slice, solve the qp
                if new_constraints_batch:
                    objective = self._solve_n_slack_qp(constraints, n_samples)
                    objective_curve.append(objective)
                    new_constraints += new_constraints_batch

            if new_constraints == 0:
                print("no additional constraints")
                break

            self._compute_training_loss(X, Y, iteration)

            if self.verbose > 0:
                print("new constraints: %d, "
                      "dual objective: %f" %
                      (new_constraints,
                       objective))
            if (iteration > 1 and objective_curve[-1]
                    - objective_curve[-2] < self.tol):
                print("objective converged.")
                break
            if self.verbose > 5:
                print(self.w)

            if self.logger is not None:
                self.logger(self, iteration)

        self.constraints_ = constraints
        self.loss_curve_ = loss_curve
        self.objective_curve_ = objective_curve
        if self.verbose and self.n_jobs == 1:
            print("calls to inference: %d" % self.model.inference_calls)
        return self

    def prune_constraints(self, constraints, a):
        # append list for new constraint
        # self.alpha is a list which has
        # an entry per sample. each sample has an int for each constraint,
        # saying when was it last used
        k = 0
        for i, sample in enumerate(constraints):
            n_new_constraints_sample = len(self.alphas[i])
            if n_new_constraints_sample < len(sample):
                self.alphas[i] = np.hstack([self.alphas[i], [0]])
            k += n_new_constraints_sample
            assert(len(sample) == len(self.alphas[i]))
        #flat_constraints = [constraint for sample in constraints for
        #constraint #in sample]
        #assert(len(self.alphas) == len(flat_constraints))
        #for constraint, alpha in zip(self.alphas, a):
            #constraint.append(alpha)
            #constraint = constraint[-self.inactive_window:]

        ## prune unused constraints:
        ## if the max of alpha in last 50 iterations was small, throw away
        #if self.inactive_window != 0:
            #max_active = [np.max(constr[-self.inactive_window:])
                          #for constr in self.alphas]
            ## find strongest constraint that is not ground truth constraint
            #strongest = np.max(max_active[1:])
            #inactive = max_active < self.inactive_threshold * strongest
            #idx = 0
            #for sample in constraints:
                #for i, const in reversed(list(enumerate(sample))):
                    #if inactive[idx]:
                        #del const[i]
                        #del self.alphas[idx]
                    #idx += 1
<|endoftext|>


<commit_before>##
#
# cmap-parse.py
# An attempt to parse concept maps, exported from cmap tools...take one
#
# Copyright 2015 Josh Pelkey
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and limitations under the
# License.
#
##

import glob
import itertools
import networkx as nx

# choose directory where cmap txt files are located
cmap_directory = './cmaps'

# get the files
files = glob.glob (cmap_directory + '/*.txt')

# iterate over all the files and start doing stuffs
for cmap_file in files:

	# create an empty Multi-directed graph
	G = nx.MultiDiGraph ()

	# open a cmap text file
	print ('>> Opening ' + cmap_file)
	f = open (cmap_file)

	# split the lines in to a list
	lines = ((f.read ()).splitlines ())

	# iterate over the list and split each line
	# in to individual lists, delimited by tab
	for line in lines:
		edge = line.split ('\t')
		G.add_edge (edge[0], edge[2], link=edge[1])

	# number of concepts is the number of nodes
	# minus the root node
	num_concepts = G.order () - 1

	# hierarchy is the out degree of the root node
	# we assume the root is 'Sustainabiliy'
	hierarchy = G.out_degree ('Sustainability')

	# look at all paths from sustainability to all
	# other nodes. no repeated nodes (cycles)
	paths_list = []
	for n in G.nodes ():
		for path in nx.all_simple_paths (G, source='Sustainability', target=n):
			paths_list.append (path)

	# highest hierarchy defined here as the max path length
	highest_hier = max (len (x) for x in paths_list) - 1

	# let's make subgraphs of all hierarchies
	# i think we can use these subgraphs to do some
	# operations and check out cross links
	hierarchy_list = G.successors ('Sustainability')
	subgraph_list = []
	for x in hierarchy_list:
		subgraph = nx.MultiDiGraph ()
		connected_nodes = []
		for y in G.nodes ():
			if nx.has_path (G, x, y):
				connected_nodes.append (y)

		# the only reason i'm copying the whole graph here
		# is to set the name attribute right now (for debugging, testing)
		# we can remove this full copy at the end for efficiency
		subgraph = G.subgraph(connected_nodes).copy ()
		subgraph.graph['name'] = x
		subgraph_list.append (subgraph)

	# iterate through the main graph and set all hier attr to zero
	for x in G:
		G.node[x]['hier'] = 0
		#print (x)

	# iterate through the subgraph_list and set all hier attr to zero
	for x in subgraph_list:
		#for all the nodes in a subgraph
		for y in x:
			x.node[y]['hier'] = 0
			#print (y)

	# re-iterate and set heir to an incrementing number across hierarchies,
	# only if it wasn't set previously (i.e. not part of another hierarchy already)
	hierIter = 1
	for x in subgraph_list:
		for y in x:
			if x.node[y]['hier'] == 0:
				for item in subgraph_list:
					if y in item:
						item.node[y]['hier'] = hierIter
						G.node[y]['hier'] = hierIter

				#print (x.node[y]['hier'])
				#print (x.node[y])
				#print (G.node[y]['hier'])
				#print (y)
		hierIter += 1
		#print (x.nodes())

	# now i need to find all edges that have two hier node attributes that don't match. these are crosslinks
	total_crosslinks = 0
	for x in G.edges():
		if ((G.node[x[0]]['hier']) != 0) and ((G.node[x[1]]['hier']) != 0):
			if G.node[x[0]]['hier'] != G.node[x[1]]['hier']:
				#print (str (x[0]) + ' ---- ' + str (x[1]) + 'hier: ' + str (G.node[x[0]]['hier']) + '---- ' + str (G.node[x[1]]['hier']))
				total_crosslinks += 1

	# print out the stuffs
	print ('>> Num Concepts: ' + str (num_concepts))
	print ('>> Hierarchy: ' + str (hierarchy))
	print ('>> HH: ' + str (highest_hier))
	print ('>> CL: ' + str (total_crosslinks))
	# show me cycles
	#print ('>> Cycles: ' + str (nx.simple_cycles (G)))

	# make it pretty
	print ('\n')

# eof.zomg
<commit_msg>fail file parse if sustainability not in map<commit_after>##
#
# cmap-parse.py
# An attempt to parse concept maps, exported from cmap tools...take one
#
# Copyright 2015 Josh Pelkey
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and limitations under the
# License.
#
##

import glob
import itertools
import networkx as nx

# choose directory where cmap txt files are located
cmap_directory = './cmaps'

# get the files
files = glob.glob (cmap_directory + '/*.txt')

# iterate over all the files and start doing stuffs
for cmap_file in files:

	# create an empty Multi-directed graph
	G = nx.MultiDiGraph ()

	# open a cmap text file
	print ('>> Opening ' + cmap_file)
	f = open (cmap_file)

	# split the lines in to a list
	lines = ((f.read ()).splitlines ())

	# iterate over the list and split each line
	# in to individual lists, delimited by tab
	for line in lines:
		edge = line.split ('\t')
		G.add_edge (edge[0], edge[2], link=edge[1])

	# if 'Sustainability' isn't a concept, fail
	if 'Sustainability' not in G:
		print ('>> Sustainability not a concept in the map.')
		break

	# number of concepts is the number of nodes
	# minus the root node
	num_concepts = G.order () - 1

	# hierarchy is the out degree of the root node
	# we assume the root is 'Sustainabiliy'
	hierarchy = G.out_degree ('Sustainability')

	# look at all paths from sustainability to all
	# other nodes. no repeated nodes (cycles)
	paths_list = []
	for n in G.nodes ():
		for path in nx.all_simple_paths (G, source='Sustainability', target=n):
			paths_list.append (path)

	# highest hierarchy defined here as the max path length
	highest_hier = max (len (x) for x in paths_list) - 1

	# let's make subgraphs of all hierarchies
	# i think we can use these subgraphs to do some
	# operations and check out cross links
	hierarchy_list = G.successors ('Sustainability')
	subgraph_list = []
	for x in hierarchy_list:
		subgraph = nx.MultiDiGraph ()
		connected_nodes = []
		for y in G.nodes ():
			if nx.has_path (G, x, y):
				connected_nodes.append (y)

		# the only reason i'm copying the whole graph here
		# is to set the name attribute right now (for debugging, testing)
		# we can remove this full copy at the end for efficiency
		subgraph = G.subgraph(connected_nodes).copy ()
		subgraph.graph['name'] = x
		subgraph_list.append (subgraph)

	# iterate through the main graph and set all hier attr to zero
	for x in G:
		G.node[x]['hier'] = 0
		#print (x)

	# iterate through the subgraph_list and set all hier attr to zero
	for x in subgraph_list:
		#for all the nodes in a subgraph
		for y in x:
			x.node[y]['hier'] = 0
			#print (y)

	# re-iterate and set heir to an incrementing number across hierarchies,
	# only if it wasn't set previously (i.e. not part of another hierarchy already)
	hierIter = 1
	for x in subgraph_list:
		for y in x:
			if x.node[y]['hier'] == 0:
				for item in subgraph_list:
					if y in item:
						item.node[y]['hier'] = hierIter
						G.node[y]['hier'] = hierIter

				#print (x.node[y]['hier'])
				#print (x.node[y])
				#print (G.node[y]['hier'])
				#print (y)
		hierIter += 1
		#print (x.nodes())

	# now i need to find all edges that have two hier node attributes that don't match. these are crosslinks
	total_crosslinks = 0
	for x in G.edges():
		if ((G.node[x[0]]['hier']) != 0) and ((G.node[x[1]]['hier']) != 0):
			if G.node[x[0]]['hier'] != G.node[x[1]]['hier']:
				#print (str (x[0]) + ' ---- ' + str (x[1]) + 'hier: ' + str (G.node[x[0]]['hier']) + '---- ' + str (G.node[x[1]]['hier']))
				total_crosslinks += 1

	# print out the stuffs
	print ('>> Num Concepts: ' + str (num_concepts))
	print ('>> Hierarchy: ' + str (hierarchy))
	print ('>> HH: ' + str (highest_hier))
	print ('>> CL: ' + str (total_crosslinks))
	# show me cycles
	#print ('>> Cycles: ' + str (nx.simple_cycles (G)))

	# make it pretty
	print ('\n')

# eof.zomg
<|endoftext|>


<commit_before>#!/usr/bin/env python

#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import re
from os.path import isfile, join

scala_nn_root = "./spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/"
python_nn_root = "./pyspark/bigdl/nn/"

scala_to_python = {"Graph": "Model"}

def extract_scala_class(class_path):
    exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape",
                             "SplitAndSelect", "StrideSlice", "Scheduler",
                             "StaticGraph", "DynamicGraph", "DynamicContainer",
                             "SplitHeads", "CombineHeads", "VectorProduct", "Pooler"])   # noqa
    include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"])  # noqa
    content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])])  # noqa
    match = re.search(r"class ([\w]+)[^{]+", content)
    if match and any([key in match.group() for key in include_key_words]):
        return match.group(1)
    else:
        return None

def get_scala_classes(scala_nn_root):
    raw_result = [extract_scala_class(join(scala_nn_root, name)) for name in os.listdir(scala_nn_root) if isfile(join(scala_nn_root, name))]  # noqa
    return set(
        class_name for class_name in raw_result if class_name is not None)

def get_python_classes(nn_root):
    exclude_classes = set(["Criterion"])
    raw_classes = []
    python_nn_layers = [join(nn_root, name) for name in os.listdir(nn_root) if isfile(join(nn_root, name)) and name.endswith('py') and "__" not in name]  # noqa
    for p in python_nn_layers:
        with open(p) as f:
            raw_classes.extend([line for line in f.readlines() if line.startswith("class")])  # noqa
    classes = [name.split()[1].split("(")[0]for name in raw_classes]
    return set([name for name in classes if name not in exclude_classes])

scala_layers = get_scala_classes(scala_nn_root)
python_layers = get_python_classes(python_nn_root)

print("scala_layers: {0}, {1}".format(len(scala_layers), scala_layers))
print("\n")
print("python_layers: {0}, {1}".format(len(python_layers), python_layers))

print("In scala, not in python: "),
diff_count = 0
for name in scala_layers:
    if name not in python_layers and scala_to_python[name] not in python_layers:
        print(name),
        diff_count += 1

if diff_count > 0:
    raise Exception("There are difference between python and scala")
<commit_msg>[New feature] Add maskhead (#2892)<commit_after>#!/usr/bin/env python

#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import re
from os.path import isfile, join

scala_nn_root = "./spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/"
python_nn_root = "./pyspark/bigdl/nn/"

scala_to_python = {"Graph": "Model"}

def extract_scala_class(class_path):
    exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape",
                             "SplitAndSelect", "StrideSlice", "Scheduler",
                             "StaticGraph", "DynamicGraph", "DynamicContainer",
                             "SplitHeads", "CombineHeads", "VectorProduct", "Pooler",
                             "MaskHead", "MaskPostProcessor"])   # noqa
    include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"])  # noqa
    content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])])  # noqa
    match = re.search(r"class ([\w]+)[^{]+", content)
    if match and any([key in match.group() for key in include_key_words]):
        return match.group(1)
    else:
        return None

def get_scala_classes(scala_nn_root):
    raw_result = [extract_scala_class(join(scala_nn_root, name)) for name in os.listdir(scala_nn_root) if isfile(join(scala_nn_root, name))]  # noqa
    return set(
        class_name for class_name in raw_result if class_name is not None)

def get_python_classes(nn_root):
    exclude_classes = set(["Criterion"])
    raw_classes = []
    python_nn_layers = [join(nn_root, name) for name in os.listdir(nn_root) if isfile(join(nn_root, name)) and name.endswith('py') and "__" not in name]  # noqa
    for p in python_nn_layers:
        with open(p) as f:
            raw_classes.extend([line for line in f.readlines() if line.startswith("class")])  # noqa
    classes = [name.split()[1].split("(")[0]for name in raw_classes]
    return set([name for name in classes if name not in exclude_classes])

scala_layers = get_scala_classes(scala_nn_root)
python_layers = get_python_classes(python_nn_root)

print("scala_layers: {0}, {1}".format(len(scala_layers), scala_layers))
print("\n")
print("python_layers: {0}, {1}".format(len(python_layers), python_layers))

print("In scala, not in python: "),
diff_count = 0
for name in scala_layers:
    if name not in python_layers and scala_to_python[name] not in python_layers:
        print(name),
        diff_count += 1

if diff_count > 0:
    raise Exception("There are difference between python and scala")
<|endoftext|>


<commit_before>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
from .layer_function_generator import generate_layer_fn, generate_layer_fn_noattr

__activations__ = [
    'softshrink',
]

__activations_noattr__ = [
    'sigmoid',
    'logsigmoid',
    'exp',
    'tanh',
    'tanh_shrink',
    'sqrt',
    'abs',
    'ceil',
    'floor',
    'cos',
    'sin',
    'round',
    'reciprocal',
    'square',
    'softplus',
    'softsign',
]

__all__ = [
    'mean',
    'mul',
    'scale',
    'sigmoid_cross_entropy_with_logits',
    'elementwise_add',
    'elementwise_div',
    'elementwise_sub',
    'elementwise_mul',
    'elementwise_max',
    'elementwise_min',
    'elementwise_pow',
    'clip',
    'clip_by_norm',
    'logical_and',
    'logical_or',
    'logical_xor',
    'logical_not',
    'uniform_random_batch_size_like',
    'gaussian_random',
    'sampling_id',
    'gaussian_random_batch_size_like',
    'sum',
    'slice',
    'shape',
    'maxout',
] + __activations__

for _OP in set(__all__):
    globals()[_OP] = generate_layer_fn(_OP)

__all__ += __activations_noattr__

for _OP in set(__activations_noattr__):
    globals()[_OP] = generate_layer_fn_noattr(_OP)

__all__ += ["uniform_random"]

_uniform_random_ = generate_layer_fn('uniform_random')

def uniform_random(shape, dtype=None, min=None, max=None, seed=None):
    kwargs = dict()
    for name in locals():
        val = locals()[name]
        if val is not None:
            kwargs[name] = val
    return _uniform_random_(**kwargs)

uniform_random.__doc__ = _uniform_random_.__doc__ + """
Examples:

    >>> result = fluid.layers.uniform_random(shape=[32, 784])
"""

__all__ += ['hard_shrink']

_hard_shrink_ = generate_layer_fn('hard_shrink')

def hard_shrink(x, threshold=None):
    kwargs = dict()
    for name in locals():
        val = locals()[name]
        if val is not None:
            kwargs[name] = val
    return _hard_shrink_(**kwargs)

hard_shrink.__doc__ = _hard_shrink_.__doc__ + """
Examples:

    >>> data = fluid.layers.data(name="input", shape=[784])
    >>> result = fluid.layers.hard_shrink(x=data, threshold=0.3)
"""

__all__ += ['cumsum']

_cum_sum_ = generate_layer_fn('cumsum')

def cumsum(x, axis=None, exclusive=None, reverse=None):
    kwargs = dict()
    for name in locals():
        val = locals()[name]
        if val is not None:
            kwargs[name] = val

    return _cum_sum_(**kwargs)

cumsum.__doc__ = _cum_sum_.__doc__ + """
Examples:

    >>> data = fluid.layers.data(name="input", shape=[32, 784])
    >>> result = fluid.layers.cumsum(data, axis=0)
"""

__all__ += ['thresholded_relu']

_thresholded_relu_ = generate_layer_fn('thresholded_relu')

def thresholded_relu(x, threshold=None):
    kwargs = dict()
    for name in locals():
        val = locals()[name]
        if val is not None:
            kwargs[name] = val

    _thresholded_relu_(**kwargs)

thresholded_relu.__doc__ = _thresholded_relu_.__doc__ + """
Examples:

    >>> data = fluid.layers.data(name="input", shape=[1])
    >>> result = fluid.layers.thresholded_relu(data, threshold=0.4)
"""
<commit_msg>fix style<commit_after>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
from .layer_function_generator import generate_layer_fn, generate_layer_fn_noattr

__activations_noattr__ = [
    'sigmoid',
    'logsigmoid',
    'exp',
    'tanh',
    'tanh_shrink',
    'sqrt',
    'abs',
    'ceil',
    'floor',
    'cos',
    'sin',
    'round',
    'reciprocal',
    'square',
    'softplus',
    'softsign',
]

__all__ = [
    'mean',
    'mul',
    'scale',
    'sigmoid_cross_entropy_with_logits',
    'elementwise_add',
    'elementwise_div',
    'elementwise_sub',
    'elementwise_mul',
    'elementwise_max',
    'elementwise_min',
    'elementwise_pow',
    'clip',
    'clip_by_norm',
    'logical_and',
    'logical_or',
    'logical_xor',
    'logical_not',
    'uniform_random_batch_size_like',
    'gaussian_random',
    'sampling_id',
    'gaussian_random_batch_size_like',
    'sum',
    'slice',
    'shape',
    'maxout',
    'softshrink',
]

for _OP in set(__all__):
    globals()[_OP] = generate_layer_fn(_OP)

__all__ += __activations_noattr__

for _OP in set(__activations_noattr__):
    globals()[_OP] = generate_layer_fn_noattr(_OP)

__all__ += ["uniform_random"]

_uniform_random_ = generate_layer_fn('uniform_random')

def uniform_random(shape, dtype=None, min=None, max=None, seed=None):
    kwargs = dict()
    for name in locals():
        val = locals()[name]
        if val is not None:
            kwargs[name] = val
    return _uniform_random_(**kwargs)

uniform_random.__doc__ = _uniform_random_.__doc__ + """
Examples:

    >>> result = fluid.layers.uniform_random(shape=[32, 784])
"""

__all__ += ['hard_shrink']

_hard_shrink_ = generate_layer_fn('hard_shrink')

def hard_shrink(x, threshold=None):
    kwargs = dict()
    for name in locals():
        val = locals()[name]
        if val is not None:
            kwargs[name] = val
    return _hard_shrink_(**kwargs)

hard_shrink.__doc__ = _hard_shrink_.__doc__ + """
Examples:

    >>> data = fluid.layers.data(name="input", shape=[784])
    >>> result = fluid.layers.hard_shrink(x=data, threshold=0.3)
"""

__all__ += ['cumsum']

_cum_sum_ = generate_layer_fn('cumsum')

def cumsum(x, axis=None, exclusive=None, reverse=None):
    kwargs = dict()
    for name in locals():
        val = locals()[name]
        if val is not None:
            kwargs[name] = val

    return _cum_sum_(**kwargs)

cumsum.__doc__ = _cum_sum_.__doc__ + """
Examples:

    >>> data = fluid.layers.data(name="input", shape=[32, 784])
    >>> result = fluid.layers.cumsum(data, axis=0)
"""

__all__ += ['thresholded_relu']

_thresholded_relu_ = generate_layer_fn('thresholded_relu')

def thresholded_relu(x, threshold=None):
    kwargs = dict()
    for name in locals():
        val = locals()[name]
        if val is not None:
            kwargs[name] = val

    _thresholded_relu_(**kwargs)

thresholded_relu.__doc__ = _thresholded_relu_.__doc__ + """
Examples:

    >>> data = fluid.layers.data(name="input", shape=[1])
    >>> result = fluid.layers.thresholded_relu(data, threshold=0.4)
"""
<|endoftext|>


<commit_before>## Copyright (C) 2011 Aldebaran Robotics
"""Generate a binary sdk"""

#FIXME: maybe we should make --no-compress the default

import os
import logging
import datetime

import qibuild

LOGGER = logging.getLogger(__name__)

def get_package_name(project, continuous=False, version=None, arch=None):
    """Get the package name of a project.

    Recognized args are:
      continuous -> append the date the the name of the package
      version    -> if not given, will try to use version.cmake at
                    the root of the project
      arch       -> if not given, do nothing, else add this at the end
                    of the package name
    """
    res = [project.name]

    if not version:
        version = get_project_version(project)
        if version:
            res.append(version)
    else:
        res.append(version)

    if continuous:
        now = datetime.datetime.now()
        res.append(now.strftime("%Y-%m-%d-%H-%M"))

    if arch:
        res.append(arch)

    return "-".join(res)

def get_project_version(project):
    """Try to guess version from the sources of the project.

    Return None if not found.
    """
    version_cmake = os.path.join(project.directory, "version.cmake")
    if not os.path.exists(version_cmake):
        return None
    contents = None
    with open(version_cmake, "r") as fp:
        contents = fp.read()

    import re
    up_name = project.name.upper()
    match = re.match('^set\(%s_VERSION\s+"?(.*?)"?\s*\)' % up_name,
                     contents)
    if not match:
        LOGGER.warning("Invalid version.cmake. Should have a line looking like\n"
           "set(%s_VERSION <VERSION>)",  up_name)
        return None
    return match.groups()[0]

def configure_parser(parser):
    """Configure parser for this action"""
    qibuild.parsers.toc_parser(parser)
    qibuild.parsers.build_parser(parser)
    qibuild.parsers.package_parser(parser)
    parser.add_argument("project", nargs="?")
    parser.add_argument("--no-compress", dest="compress",
        action="store_false",
        help  ="Do not compress the final install directory")
    parser.set_defaults(
        cmake_flags=["CMAKE_INSTALL_PREFIX='/'"],
        compress=True)

def _do(args, build_type):
    """Called by do().
    Once with build_type=release on UNIX.
    Twice with build_type=debug and build_type=release on windows

    Returns the directory to make an archive from
    """
    args.build_type = build_type
    toc      = qibuild.toc_open(args.work_tree, args, use_env=True)
    if not args.project:
        project_name = qibuild.toc.project_from_cwd()
    else:
        project_name = args.project
    project = toc.get_project(project_name)
    inst_dir = os.path.join(toc.work_tree, "package")

    package_name = get_package_name(
        project,
        continuous=args.continuous,
        arch=args.arch,
        version=args.version)

    destdir = os.path.join(inst_dir, package_name)
    qibuild.run_action("qibuild.actions.configure", [project_name, "--no-clean-first"],
        forward_args=args)
    qibuild.run_action("qibuild.actions.make", [project_name],
        forward_args=args)
    qibuild.run_action("qibuild.actions.install", [project_name, destdir],
        forward_args=args)
    return destdir

def do(args):
    """Main entry point"""
    toc = qibuild.toc_open(args.work_tree, args, use_env=True)
    if toc.using_visual_studio and not args.runtime:
        _do(args, "debug")
    destdir = _do(args, "release")

    if args.compress:
        LOGGER.info("Compressing package")
        archive = qibuild.archive.zip(destdir)
        LOGGER.info("Package generated in %s", archive)
        # Now, clean the destdir.
        qibuild.sh.rm(destdir)
        return archive
    else:
        return destdir

<commit_msg>qibuild package.py: a couple of TODO items<commit_after>## Copyright (C) 2011 Aldebaran Robotics
"""Generate a binary sdk"""

#FIXME: maybe we should make --no-compress the default

import os
import logging
import datetime

import qibuild

LOGGER = logging.getLogger(__name__)

def get_package_name(project, continuous=False, version=None, arch=None):
    """Get the package name of a project.

    Recognized args are:
      continuous -> append the date the the name of the package
      version    -> if not given, will try to use version.cmake at
                    the root of the project
      arch       -> if not given, do nothing, else add this at the end
                    of the package name
    """
    res = [project.name]

    if not version:
        version = get_project_version(project)
        if version:
            res.append(version)
    else:
        res.append(version)

    if continuous:
        now = datetime.datetime.now()
        res.append(now.strftime("%Y-%m-%d-%H-%M"))

    if arch:
        res.append(arch)

    return "-".join(res)

def get_project_version(project):
    """Try to guess version from the sources of the project.

    Return None if not found.
    """
    version_cmake = os.path.join(project.directory, "version.cmake")
    if not os.path.exists(version_cmake):
        return None
    contents = None
    with open(version_cmake, "r") as fp:
        contents = fp.read()

    import re
    up_name = project.name.upper()
    match = re.match('^set\(%s_VERSION\s+"?(.*?)"?\s*\)' % up_name,
                     contents)
    if not match:
        LOGGER.warning("Invalid version.cmake. Should have a line looking like\n"
           "set(%s_VERSION <VERSION>)",  up_name)
        return None
    return match.groups()[0]

def configure_parser(parser):
    """Configure parser for this action"""
    qibuild.parsers.toc_parser(parser)
    qibuild.parsers.build_parser(parser)
    qibuild.parsers.package_parser(parser)
    parser.add_argument("project", nargs="?")
    parser.add_argument("--no-compress", dest="compress",
        action="store_false",
        help  ="Do not compress the final install directory")
    parser.set_defaults(
        cmake_flags=["CMAKE_INSTALL_PREFIX='/'"],
        compress=True)

def _do(args, build_type):
    """Called by do().
    Once with build_type=release on UNIX.
    Twice with build_type=debug and build_type=release on windows

    Returns the directory to make an archive from
    """
    args.build_type = build_type
    toc      = qibuild.toc_open(args.work_tree, args, use_env=True)
    if not args.project:
        project_name = qibuild.toc.project_from_cwd()
    else:
        project_name = args.project
    project = toc.get_project(project_name)
    #TODO create package directory if it does not exit
    inst_dir = os.path.join(toc.work_tree, "package")

    package_name = get_package_name(
        project,
        continuous=args.continuous,
        arch=args.arch,
        version=args.version)

    destdir = os.path.join(inst_dir, package_name)
    qibuild.run_action("qibuild.actions.configure", [project_name, "--no-clean-first"],
        forward_args=args)
    qibuild.run_action("qibuild.actions.make", [project_name],
        forward_args=args)
    qibuild.run_action("qibuild.actions.install", [project_name, destdir],
        forward_args=args)
    return destdir

def do(args):
    """Main entry point"""
    toc = qibuild.toc_open(args.work_tree, args, use_env=True)
    if toc.using_visual_studio and not args.runtime:
        _do(args, "debug")
    destdir = _do(args, "release")
    #TODO warn if dest dir is empty

    if args.compress:
        LOGGER.info("Compressing package")
        archive = qibuild.archive.zip(destdir)
        LOGGER.info("Package generated in %s", archive)
        # Now, clean the destdir.
        qibuild.sh.rm(destdir)
        return archive
    else:
        return destdir

<|endoftext|>


<commit_before># This file is part of the SPLINTER library.
# Copyright (C) 2012 Bjarne Grimstad (bjarne.grimstad@gmail.com).
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from . import splinter
from ctypes import *
from .bspline import BSpline
from .datatable import DataTable

class BSplineBuilder:

    class Smoothing:
        NONE, IDENTITY, PSPLINE = range(3)

        @staticmethod
        def isValid(value):
            return value in range(3)

    class KnotSpacing:
        SAMPLE, EQUIDISTANT, EXPERIMENTAL = range(3)

        @staticmethod
        def isValid(value):
            return value in range(3)

    def __init__(self, data, degree=3, smoothing=Smoothing.NONE, alpha=0.1, knotSpacing=KnotSpacing.SAMPLE):
        self._handle = None  # Handle for referencing the c side of this object
        self._datatable = DataTable(data)
        self._numBasisFunctions = [10**3] * self._datatable.getNumVariables()

        self._degrees = None
        self._numBasisFunctions = None
        self._knotSpacing = None
        self._smoothing = None
        self._alpha = None

        self._handle = splinter._call(splinter._getHandle().splinter_bspline_builder_init, self._datatable._getHandle())
        self.degree(degree)
        self.smoothing(smoothing)
        self.knotSpacing(knotSpacing)
        self.setAlpha(alpha)

    def degree(self, degrees):
        # If the value is a single number, make it a list of numVariables length
        if not isinstance(degrees, list):
            degrees = [degrees] * self._datatable.getNumVariables()

        if len(degrees) != self._datatable.getNumVariables():
            raise ValueError("BSplineBuilder:degree: Inconsistent number of degrees.")

        valid_degrees = range(0, 6)
        for deg in degrees:
            if deg not in valid_degrees:
                raise ValueError("BSplineBuilder:degree: Invalid degree: " + str(deg))

        self._degrees = degrees

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_degree, self._handle, (c_int * len(self._degrees))(*self._degrees), len(self._degrees))
        return self

    def numBasisFunctions(self, numBasisFunctions):
        # If the value is a single number, make it a list of numVariables length
        if not isinstance(numBasisFunctions, list):
            numBasisFunctions = [numBasisFunctions] * self._datatable.getNumVariables()

        if len(numBasisFunctions) != self._datatable.getNumVariables():
            raise ValueError("BSplineBuilder:numBasisFunctions: Inconsistent number of degrees.")

        for numBasisFunction in numBasisFunctions:
            if not isinstance(numBasisFunction, int):
                raise ValueError("BSplineBuilder:numBasisFunctions: Invalid number of basis functions (must be integer): " + str(numBasisFunction))

        self._numBasisFunctions = numBasisFunctions

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_num_basis_functions, self._handle, (c_int * len(self._numBasisFunctions))(*self._numBasisFunctions), len(self._numBasisFunctions))
        return self

    def knotSpacing(self, knotSpacing):
        if not BSplineBuilder.KnotSpacing.isValid(knotSpacing):
            raise ValueError("BSplineBuilder::knotSpacing: Invalid knotspacing: " + str(knotSpacing))

        self._knotSpacing = knotSpacing

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_knot_spacing, self._handle, self._knotSpacing)
        return self

    def smoothing(self, smoothing):
        if not BSplineBuilder.Smoothing.isValid(smoothing):
            raise ValueError("BSplineBuilder::smoothing: Invalid smoothing: " + str(smoothing))

        self._smoothing = smoothing

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_smoothing, self._handle, self._smoothing)
        return self

    def setAlpha(self, newAlpha):
        if newAlpha < 0:
            raise ValueError("BSplineBuilder:setAlpha: alpha must be non-negative.")

        self._alpha = newAlpha

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_alpha, self._handle, self._alpha)
        return self

    # Returns a handle to the created internal BSpline object
    def build(self):
        bspline_handle = splinter._call(splinter._getHandle().splinter_bspline_builder_build, self._handle)

        return BSpline(bspline_handle)

    def __del__(self):
        if self._handle is not None:
            splinter._call(splinter._getHandle().splinter_bspline_builder_delete, self._handle)
        self._handle = None
<commit_msg>Changed name from KnotSpacing.SAMPLES to KnotSpacing.AS_SAMPLED<commit_after># This file is part of the SPLINTER library.
# Copyright (C) 2012 Bjarne Grimstad (bjarne.grimstad@gmail.com).
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from . import splinter
from ctypes import *
from .bspline import BSpline
from .datatable import DataTable

class BSplineBuilder:

    class Smoothing:
        NONE, IDENTITY, PSPLINE = range(3)

        @staticmethod
        def isValid(value):
            return value in range(3)

    class KnotSpacing:
        AS_SAMPLED, EQUIDISTANT, EXPERIMENTAL = range(3)

        @staticmethod
        def isValid(value):
            return value in range(3)

    def __init__(self, data, degree=3, smoothing=Smoothing.NONE, alpha=0.1, knotSpacing=KnotSpacing.AS_SAMPLED):
        self._handle = None  # Handle for referencing the c side of this object
        self._datatable = DataTable(data)
        self._numBasisFunctions = [10**3] * self._datatable.getNumVariables()

        self._degrees = None
        self._numBasisFunctions = None
        self._knotSpacing = None
        self._smoothing = None
        self._alpha = None

        self._handle = splinter._call(splinter._getHandle().splinter_bspline_builder_init, self._datatable._getHandle())
        self.degree(degree)
        self.smoothing(smoothing)
        self.knotSpacing(knotSpacing)
        self.setAlpha(alpha)

    def degree(self, degrees):
        # If the value is a single number, make it a list of numVariables length
        if not isinstance(degrees, list):
            degrees = [degrees] * self._datatable.getNumVariables()

        if len(degrees) != self._datatable.getNumVariables():
            raise ValueError("BSplineBuilder:degree: Inconsistent number of degrees.")

        valid_degrees = range(0, 6)
        for deg in degrees:
            if deg not in valid_degrees:
                raise ValueError("BSplineBuilder:degree: Invalid degree: " + str(deg))

        self._degrees = degrees

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_degree, self._handle, (c_int * len(self._degrees))(*self._degrees), len(self._degrees))
        return self

    def numBasisFunctions(self, numBasisFunctions):
        # If the value is a single number, make it a list of numVariables length
        if not isinstance(numBasisFunctions, list):
            numBasisFunctions = [numBasisFunctions] * self._datatable.getNumVariables()

        if len(numBasisFunctions) != self._datatable.getNumVariables():
            raise ValueError("BSplineBuilder:numBasisFunctions: Inconsistent number of degrees.")

        for numBasisFunction in numBasisFunctions:
            if not isinstance(numBasisFunction, int):
                raise ValueError("BSplineBuilder:numBasisFunctions: Invalid number of basis functions (must be integer): " + str(numBasisFunction))

        self._numBasisFunctions = numBasisFunctions

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_num_basis_functions, self._handle, (c_int * len(self._numBasisFunctions))(*self._numBasisFunctions), len(self._numBasisFunctions))
        return self

    def knotSpacing(self, knotSpacing):
        if not BSplineBuilder.KnotSpacing.isValid(knotSpacing):
            raise ValueError("BSplineBuilder::knotSpacing: Invalid knotspacing: " + str(knotSpacing))

        self._knotSpacing = knotSpacing

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_knot_spacing, self._handle, self._knotSpacing)
        return self

    def smoothing(self, smoothing):
        if not BSplineBuilder.Smoothing.isValid(smoothing):
            raise ValueError("BSplineBuilder::smoothing: Invalid smoothing: " + str(smoothing))

        self._smoothing = smoothing

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_smoothing, self._handle, self._smoothing)
        return self

    def setAlpha(self, newAlpha):
        if newAlpha < 0:
            raise ValueError("BSplineBuilder:setAlpha: alpha must be non-negative.")

        self._alpha = newAlpha

        splinter._call(splinter._getHandle().splinter_bspline_builder_set_alpha, self._handle, self._alpha)
        return self

    # Returns a handle to the created internal BSpline object
    def build(self):
        bspline_handle = splinter._call(splinter._getHandle().splinter_bspline_builder_build, self._handle)

        return BSpline(bspline_handle)

    def __del__(self):
        if self._handle is not None:
            splinter._call(splinter._getHandle().splinter_bspline_builder_delete, self._handle)
        self._handle = None
<|endoftext|>


<commit_before>"""
Layer Normalization
====================
"""

import torch

import triton
import triton.language as tl

try:
    # This is https://github.com/NVIDIA/apex, NOT the apex on PyPi, so it
    # should not be added to extras_require in setup.py.
    import apex
    HAS_APEX = True
except ModuleNotFoundError:
    HAS_APEX = False

@triton.jit
def _layer_norm_fwd_fused(
    Out,
    A,
    Weight,
    Bias,
    Mean, Rstd,
    stride, N, eps,
    BLOCK_SIZE: tl.constexpr,
):
    # position of elements processed by this program
    row = tl.program_id(0)
    Out += row * stride
    A += row * stride
    # compute mean
    mean = 0
    _mean = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
    for off in range(0, N, BLOCK_SIZE):
        cols = off + tl.arange(0, BLOCK_SIZE)
        a = tl.load(A + cols, mask=cols < N, other=0., eviction_policy="evict_last").to(tl.float32)
        _mean += a
    mean = tl.sum(_mean, axis=0) / N
    # compute variance
    _var = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
    for off in range(0, N, BLOCK_SIZE):
        cols = off + tl.arange(0, BLOCK_SIZE)
        a = tl.load(A + cols, mask=cols < N, other=0., eviction_policy="evict_last").to(tl.float32)
        a = tl.where(cols < N, a - mean, 0.)
        _var += a * a
    var = tl.sum(_var, axis=0) / N
    rstd = 1 / tl.sqrt(var + eps)
    # write-back mean/rstd
    tl.store(Mean + row, mean)
    tl.store(Rstd + row, rstd)
    # multiply by weight and add bias
    for off in range(0, N, BLOCK_SIZE):
        cols = off + tl.arange(0, BLOCK_SIZE)
        mask = cols < N
        weight = tl.load(Weight + cols, mask=mask)
        bias = tl.load(Bias + cols, mask=mask)
        a = tl.load(A + cols, mask=mask, other=0., eviction_policy="evict_first").to(tl.float32)
        a_hat = (a - mean) * rstd
        out = a_hat * weight + bias
        # # write-back
        tl.store(Out + cols, out, mask=mask)

# Backward pass (DA + partial DW + partial DB)

@triton.jit
def _layer_norm_bwd_dx_fused(
    _DA,
    _DOut,
    _A,
    Weight,
    Mean, Rstd,
    stride, NumRows, NumCols, eps,
    BLOCK_SIZE_N: tl.constexpr,
):
    # position of elements processed by this program
    pid = tl.program_id(0)
    row = pid
    A = _A + row * stride
    DOut = _DOut + row * stride
    DA = _DA + row * stride
    mean = tl.load(Mean + row)
    rstd = tl.load(Rstd + row)
    # load data to SRAM
    _mean1 = tl.zeros([BLOCK_SIZE_N], dtype=tl.float32)
    _mean2 = tl.zeros([BLOCK_SIZE_N], dtype=tl.float32)
    for off in range(0, NumCols, BLOCK_SIZE_N):
        cols = off + tl.arange(0, BLOCK_SIZE_N)
        mask = cols < NumCols
        a = tl.load(A + cols, mask=mask, other=0).to(tl.float32)
        dout = tl.load(DOut + cols, mask=mask, other=0).to(tl.float32)
        weight = tl.load(Weight + cols, mask=mask, other=0).to(tl.float32)
        a_hat = (a - mean) * rstd
        wdout = weight * dout
        _mean1 += a_hat * wdout
        _mean2 += wdout
    mean1 = tl.sum(_mean1, axis=0) / NumCols
    mean2 = 0.
    mean2 = tl.sum(_mean2, axis=0) / NumCols
    for off in range(0, NumCols, BLOCK_SIZE_N):
        cols = off + tl.arange(0, BLOCK_SIZE_N)
        mask = cols < NumCols
        a = tl.load(A + cols, mask=mask, other=0).to(tl.float32)
        dout = tl.load(DOut + cols, mask=mask, other=0).to(tl.float32)
        weight = tl.load(Weight + cols, mask=mask, other=0).to(tl.float32)
        a_hat = (a - mean) * rstd
        wdout = weight * dout
        da = (wdout - (a_hat * mean1 + mean2)) * rstd
        # write-back dx
        tl.store(DA + cols, da, mask=mask)

# Backward pass (total DW + total DB)
@triton.jit
def _layer_norm_bwd_dwdb(
    A, DOut,
    Mean, Var,
    DW,
    DB,
    M, N,
    BLOCK_SIZE_M: tl.constexpr,
    BLOCK_SIZE_N: tl.constexpr,
):
    pid = tl.program_id(0)
    cols = pid * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
    dw = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
    db = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
    for i in range(0, M, BLOCK_SIZE_M):
        rows = i + tl.arange(0, BLOCK_SIZE_M)
        mask = (rows[:, None] < M) & (cols[None, :] < N)
        offs = rows[:, None] * N + cols[None, :]
        a = tl.load(A + offs, mask=mask, other=0.).to(tl.float32)
        dout = tl.load(DOut + offs, mask=mask, other=0.).to(tl.float32)
        mean = tl.load(Mean + rows, mask=rows < M, other=0.)
        rstd = tl.load(Var + rows, mask=rows < M, other=0.)
        a_hat = (a - mean[:, None]) * rstd[:, None]
        dw += dout * a_hat
        db += dout
    sum_dw = tl.sum(dw, axis=0)
    sum_db = tl.sum(db, axis=0)
    tl.store(DW + cols, sum_dw, mask=cols < N)
    tl.store(DB + cols, sum_db, mask=cols < N)

class LayerNorm(torch.autograd.Function):
    @staticmethod
    def forward(ctx, a, normalized_shape, weight, bias, eps):
        # allocate output
        out = torch.empty_like(a)
        # reshape input data into 2D tensor
        a_arg = a.reshape(-1, a.shape[-1])
        M, N = a_arg.shape
        mean = torch.empty((M,), dtype=torch.float32, device="cuda")
        rstd = torch.empty((M,), dtype=torch.float32, device="cuda")
        # Less than 64KB per feature: enqueue fused kernel
        MAX_FUSED_SIZE = 65536 // a.element_size()
        BLOCK_SIZE = min(MAX_FUSED_SIZE, triton.next_power_of_2(N))
        BLOCK_SIZE = max(BLOCK_SIZE, 128)
        BLOCK_SIZE = min(BLOCK_SIZE, 4096)
        # heuristics for number of warps
        num_warps = min(max(BLOCK_SIZE // 256, 1), 8)
        _layer_norm_fwd_fused[(M,)](
            out,
            a_arg,
            weight,
            bias,
            mean, rstd,
            a_arg.stride(0), N, eps,
            BLOCK_SIZE=BLOCK_SIZE,
            num_warps=num_warps,
        )
        ctx.save_for_backward(
            a, weight, bias, mean, rstd,
        )
        ctx.BLOCK_SIZE = BLOCK_SIZE
        ctx.num_warps = num_warps
        ctx.eps = eps
        if hasattr(bias, "config"):
            assert bias.config.grad_scale_name == weight.config.grad_scale_name
            grad_scale_name = bias.config.grad_scale_name
        else:
            grad_scale_name = None
        ctx.grad_scale_gain_bias_name = grad_scale_name
        return out

    @staticmethod
    def backward(ctx, dout):
        assert dout.is_contiguous()
        a, weight, bias, mean, var = ctx.saved_tensors
        # heuristics for amount of parallel reduction stream for DG/DB
        N = weight.shape[0]
        # allocate output
        da = torch.empty_like(dout)
        # enqueue kernel using forward pass heuristics
        # also compute partial sums for DW and DB
        x_arg = a.reshape(-1, a.shape[-1])
        M, N = x_arg.shape
        dweight = torch.empty((weight.shape[0],), dtype=weight.dtype, device=weight.device)
        dbias = torch.empty((weight.shape[0],), dtype=weight.dtype, device=weight.device)
        _layer_norm_bwd_dx_fused[(M,)](
            da,
            dout,
            a,
            weight,
            mean, var,
            x_arg.stride(0), M, N,
            ctx.eps,
            BLOCK_SIZE_N=ctx.BLOCK_SIZE,
            num_warps=ctx.num_warps,
        )
        # accumulate partial sums in separate kernel
        grid = lambda meta: [triton.cdiv(N, meta["BLOCK_SIZE_N"])]
        _layer_norm_bwd_dwdb[grid](
            a, dout,
            mean, var,
            dweight,
            dbias,
            M,
            N,
            BLOCK_SIZE_M=32,
            BLOCK_SIZE_N=128,
        )
        return (da, None, dweight, dbias, None, None,
                None, None, None, None,
                None,
                None, None, None,
                None,
                None, None, None,
                None, None, None,
                None, None, None)

def layer_norm(a, normalized_shape, weight, bias, eps):
    return LayerNorm.apply(a, normalized_shape, weight, bias, eps)

def test_layer_norm(M, N, dtype, eps=1e-5, device='cuda'):
    torch.manual_seed(0)
    # create data
    x_shape = (M, N)
    w_shape = (x_shape[-1], )
    weight = torch.rand(w_shape, dtype=dtype, device='cuda', requires_grad=True)
    bias = torch.rand(w_shape, dtype=dtype, device='cuda', requires_grad=True)
    x = -2.3 + 0.5 * torch.randn(x_shape, dtype=dtype, device='cuda')
    dy = .1 * torch.randn_like(x)
    x.requires_grad_(True)
    # forward pass
    y_tri = layer_norm(x, w_shape, weight, bias, eps)
    y_ref = torch.nn.functional.layer_norm(x, w_shape, weight, bias, eps).to(dtype)
    # backward pass (triton)
    y_tri.backward(dy, retain_graph=True)
    dx_tri, dw_tri, db_tri = [_.grad.clone() for _ in [x, weight, bias]]
    x.grad, weight.grad, bias.grad = None, None, None
    # backward pass (torch)
    y_ref.backward(dy, retain_graph=True)
    dx_ref, dw_ref, db_ref = [_.grad.clone() for _ in [x, weight, bias]]
    # compare
    triton.testing.assert_almost_equal(y_tri, y_ref)
    triton.testing.assert_almost_equal(dx_tri, dx_ref)
    triton.testing.assert_almost_equal(db_tri, db_ref, decimal=1)
    triton.testing.assert_almost_equal(dw_tri, dw_ref, decimal=1)

@triton.testing.perf_report(
    triton.testing.Benchmark(
        x_names=['N'],
        x_vals=[512 * i for i in range(2, 32)],
        line_arg='provider',
        line_vals=['triton', 'torch'] + (['apex'] if HAS_APEX else []),
        line_names=['Triton', 'Torch'] + (['Apex'] if HAS_APEX else []),
        styles=[('blue', '-'), ('green', '-'), ('orange', '-')],
        ylabel='GB/s',
        plot_name='layer-norm',
        args={'M': 4096, 'dtype': torch.float16, 'mode': 'forward'}
    )
)
def bench_layer_norm(M, N, dtype, provider, mode, eps=1e-5, device='cuda'):
    # create data
    x_shape = (M, N)
    w_shape = (x_shape[-1], )
    weight = torch.rand(w_shape, dtype=dtype, device='cuda', requires_grad=True)
    bias = torch.rand(w_shape, dtype=dtype, device='cuda', requires_grad=True)
    x = -2.3 + 0.5 * torch.randn(x_shape, dtype=dtype, device='cuda')
    dy = .1 * torch.randn_like(x)
    x.requires_grad_(True)
    # utility functions
    if provider == 'triton':
        y_fwd = lambda: layer_norm(x, w_shape, weight, bias, eps)
    if provider == 'torch':
        y_fwd = lambda: torch.nn.functional.layer_norm(x, w_shape, weight, bias, eps)
    if provider == 'apex':
        apex_layer_norm = apex.normalization.FusedLayerNorm(w_shape).to(x.device).to(x.dtype)
        y_fwd = lambda: apex_layer_norm(x)
    # forward pass
    if mode == 'forward':
        gbps = lambda ms: 2 * x.numel() * x.element_size() / ms * 1e-6
        ms, min_ms, max_ms = triton.testing.do_bench(y_fwd, rep=500)
    # backward pass
    if mode == 'backward':
        gbps = lambda ms: 3 * x.numel() * x.element_size() / ms * 1e-6
        y = y_fwd()
        ms, min_ms, max_ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True),
                                                     grad_to_none=[x], rep=500)
    return gbps(ms), gbps(max_ms), gbps(min_ms)

# test_layer_norm(1151, 8192, torch.float16)
bench_layer_norm.run(save_path='.', print_data=True)
<commit_msg>[TUTORIALS] adjust heuristics for dwdb kernel (#565)<commit_after>"""
Layer Normalization
====================
"""

import torch

import triton
import triton.language as tl

try:
    # This is https://github.com/NVIDIA/apex, NOT the apex on PyPi, so it
    # should not be added to extras_require in setup.py.
    import apex
    HAS_APEX = True
except ModuleNotFoundError:
    HAS_APEX = False

@triton.jit
def _layer_norm_fwd_fused(
    Out,
    A,
    Weight,
    Bias,
    Mean, Rstd,
    stride, N, eps,
    BLOCK_SIZE: tl.constexpr,
):
    # position of elements processed by this program
    row = tl.program_id(0)
    Out += row * stride
    A += row * stride
    # compute mean
    mean = 0
    _mean = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
    for off in range(0, N, BLOCK_SIZE):
        cols = off + tl.arange(0, BLOCK_SIZE)
        a = tl.load(A + cols, mask=cols < N, other=0., eviction_policy="evict_last").to(tl.float32)
        _mean += a
    mean = tl.sum(_mean, axis=0) / N
    # compute variance
    _var = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
    for off in range(0, N, BLOCK_SIZE):
        cols = off + tl.arange(0, BLOCK_SIZE)
        a = tl.load(A + cols, mask=cols < N, other=0., eviction_policy="evict_last").to(tl.float32)
        a = tl.where(cols < N, a - mean, 0.)
        _var += a * a
    var = tl.sum(_var, axis=0) / N
    rstd = 1 / tl.sqrt(var + eps)
    # write-back mean/rstd
    tl.store(Mean + row, mean)
    tl.store(Rstd + row, rstd)
    # multiply by weight and add bias
    for off in range(0, N, BLOCK_SIZE):
        cols = off + tl.arange(0, BLOCK_SIZE)
        mask = cols < N
        weight = tl.load(Weight + cols, mask=mask)
        bias = tl.load(Bias + cols, mask=mask)
        a = tl.load(A + cols, mask=mask, other=0., eviction_policy="evict_first").to(tl.float32)
        a_hat = (a - mean) * rstd
        out = a_hat * weight + bias
        # # write-back
        tl.store(Out + cols, out, mask=mask)

# Backward pass (DA + partial DW + partial DB)

@triton.jit
def _layer_norm_bwd_dx_fused(
    _DA,
    _DOut,
    _A,
    Weight,
    Mean, Rstd,
    stride, NumRows, NumCols, eps,
    BLOCK_SIZE_N: tl.constexpr,
):
    # position of elements processed by this program
    pid = tl.program_id(0)
    row = pid
    A = _A + row * stride
    DOut = _DOut + row * stride
    DA = _DA + row * stride
    mean = tl.load(Mean + row)
    rstd = tl.load(Rstd + row)
    # load data to SRAM
    _mean1 = tl.zeros([BLOCK_SIZE_N], dtype=tl.float32)
    _mean2 = tl.zeros([BLOCK_SIZE_N], dtype=tl.float32)
    for off in range(0, NumCols, BLOCK_SIZE_N):
        cols = off + tl.arange(0, BLOCK_SIZE_N)
        mask = cols < NumCols
        a = tl.load(A + cols, mask=mask, other=0).to(tl.float32)
        dout = tl.load(DOut + cols, mask=mask, other=0).to(tl.float32)
        weight = tl.load(Weight + cols, mask=mask, other=0).to(tl.float32)
        a_hat = (a - mean) * rstd
        wdout = weight * dout
        _mean1 += a_hat * wdout
        _mean2 += wdout
    mean1 = tl.sum(_mean1, axis=0) / NumCols
    mean2 = 0.
    mean2 = tl.sum(_mean2, axis=0) / NumCols
    for off in range(0, NumCols, BLOCK_SIZE_N):
        cols = off + tl.arange(0, BLOCK_SIZE_N)
        mask = cols < NumCols
        a = tl.load(A + cols, mask=mask, other=0).to(tl.float32)
        dout = tl.load(DOut + cols, mask=mask, other=0).to(tl.float32)
        weight = tl.load(Weight + cols, mask=mask, other=0).to(tl.float32)
        a_hat = (a - mean) * rstd
        wdout = weight * dout
        da = (wdout - (a_hat * mean1 + mean2)) * rstd
        # write-back dx
        tl.store(DA + cols, da, mask=mask)

# Backward pass (total DW + total DB)
@triton.jit
def _layer_norm_bwd_dwdb(
    A, DOut,
    Mean, Var,
    DW,
    DB,
    M, N,
    BLOCK_SIZE_M: tl.constexpr,
    BLOCK_SIZE_N: tl.constexpr,
):
    pid = tl.program_id(0)
    cols = pid * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
    dw = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
    db = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
    UNROLL: tl.constexpr = 4
    for i in range(0, M, BLOCK_SIZE_M * UNROLL):
        for j in range(UNROLL):
            rows = i + j * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
            mask = (rows[:, None] < M) & (cols[None, :] < N)
            offs = rows[:, None] * N + cols[None, :]
            a = tl.load(A + offs, mask=mask, other=0.).to(tl.float32)
            dout = tl.load(DOut + offs, mask=mask, other=0.).to(tl.float32)
            mean = tl.load(Mean + rows, mask=rows < M, other=0.)
            rstd = tl.load(Var + rows, mask=rows < M, other=0.)
            a_hat = (a - mean[:, None]) * rstd[:, None]
            dw += dout * a_hat
            db += dout
    sum_dw = tl.sum(dw, axis=0)
    sum_db = tl.sum(db, axis=0)
    tl.store(DW + cols, sum_dw, mask=cols < N)
    tl.store(DB + cols, sum_db, mask=cols < N)

class LayerNorm(torch.autograd.Function):
    @staticmethod
    def forward(ctx, a, normalized_shape, weight, bias, eps):
        # allocate output
        out = torch.empty_like(a)
        # reshape input data into 2D tensor
        a_arg = a.reshape(-1, a.shape[-1])
        M, N = a_arg.shape
        mean = torch.empty((M,), dtype=torch.float32, device="cuda")
        rstd = torch.empty((M,), dtype=torch.float32, device="cuda")
        # Less than 64KB per feature: enqueue fused kernel
        MAX_FUSED_SIZE = 65536 // a.element_size()
        BLOCK_SIZE = min(MAX_FUSED_SIZE, triton.next_power_of_2(N))
        BLOCK_SIZE = max(BLOCK_SIZE, 128)
        BLOCK_SIZE = min(BLOCK_SIZE, 4096)
        # heuristics for number of warps
        num_warps = min(max(BLOCK_SIZE // 256, 1), 8)
        _layer_norm_fwd_fused[(M,)](
            out,
            a_arg,
            weight,
            bias,
            mean, rstd,
            a_arg.stride(0), N, eps,
            BLOCK_SIZE=BLOCK_SIZE,
            num_warps=num_warps,
        )
        ctx.save_for_backward(
            a, weight, bias, mean, rstd,
        )
        ctx.BLOCK_SIZE = BLOCK_SIZE
        ctx.num_warps = num_warps
        ctx.eps = eps
        if hasattr(bias, "config"):
            assert bias.config.grad_scale_name == weight.config.grad_scale_name
            grad_scale_name = bias.config.grad_scale_name
        else:
            grad_scale_name = None
        ctx.grad_scale_gain_bias_name = grad_scale_name
        return out

    @staticmethod
    def backward(ctx, dout):
        assert dout.is_contiguous()
        a, weight, bias, mean, var = ctx.saved_tensors
        # heuristics for amount of parallel reduction stream for DG/DB
        N = weight.shape[0]
        # allocate output
        da = torch.empty_like(dout)
        # enqueue kernel using forward pass heuristics
        # also compute partial sums for DW and DB
        x_arg = a.reshape(-1, a.shape[-1])
        M, N = x_arg.shape
        dweight = torch.empty((weight.shape[0],), dtype=weight.dtype, device=weight.device)
        dbias = torch.empty((weight.shape[0],), dtype=weight.dtype, device=weight.device)
        _layer_norm_bwd_dx_fused[(M,)](
            da,
            dout,
            a,
            weight,
            mean, var,
            x_arg.stride(0), M, N,
            ctx.eps,
            BLOCK_SIZE_N=ctx.BLOCK_SIZE,
            num_warps=ctx.num_warps,
        )
        if N > 10240:
            BLOCK_SIZE_N = 128
            BLOCK_SIZE_M = 32
            num_warps = 4
        else:
            # maximize occupancy for small N
            BLOCK_SIZE_N = 16
            BLOCK_SIZE_M = 16
            num_warps = 8
        grid = lambda meta: [triton.cdiv(N, meta["BLOCK_SIZE_N"])]
        _layer_norm_bwd_dwdb[grid](
            a, dout,
            mean, var,
            dweight,
            dbias,
            M,
            N,
            BLOCK_SIZE_M=BLOCK_SIZE_M,
            BLOCK_SIZE_N=BLOCK_SIZE_N,
            num_warps=num_warps
        )
        return (da, None, dweight, dbias, None)

def layer_norm(a, normalized_shape, weight, bias, eps):
    return LayerNorm.apply(a, normalized_shape, weight, bias, eps)

def test_layer_norm(M, N, dtype, eps=1e-5, device='cuda'):
    torch.manual_seed(0)
    # create data
    x_shape = (M, N)
    w_shape = (x_shape[-1], )
    weight = torch.rand(w_shape, dtype=dtype, device='cuda', requires_grad=True)
    bias = torch.rand(w_shape, dtype=dtype, device='cuda', requires_grad=True)
    x = -2.3 + 0.5 * torch.randn(x_shape, dtype=dtype, device='cuda')
    dy = .1 * torch.randn_like(x)
    x.requires_grad_(True)
    # forward pass
    y_tri = layer_norm(x, w_shape, weight, bias, eps)
    y_ref = torch.nn.functional.layer_norm(x, w_shape, weight, bias, eps).to(dtype)
    # backward pass (triton)
    y_tri.backward(dy, retain_graph=True)
    dx_tri, dw_tri, db_tri = [_.grad.clone() for _ in [x, weight, bias]]
    x.grad, weight.grad, bias.grad = None, None, None
    # backward pass (torch)
    y_ref.backward(dy, retain_graph=True)
    dx_ref, dw_ref, db_ref = [_.grad.clone() for _ in [x, weight, bias]]
    # compare
    triton.testing.assert_almost_equal(y_tri, y_ref)
    triton.testing.assert_almost_equal(dx_tri, dx_ref)
    triton.testing.assert_almost_equal(db_tri, db_ref, decimal=1)
    triton.testing.assert_almost_equal(dw_tri, dw_ref, decimal=1)

@triton.testing.perf_report(
    triton.testing.Benchmark(
        x_names=['N'],
        x_vals=[512 * i for i in range(2, 32)],
        line_arg='provider',
        line_vals=['triton', 'torch'] + (['apex'] if HAS_APEX else []),
        line_names=['Triton', 'Torch'] + (['Apex'] if HAS_APEX else []),
        styles=[('blue', '-'), ('green', '-'), ('orange', '-')],
        ylabel='GB/s',
        plot_name='layer-norm',
        args={'M': 4096, 'dtype': torch.float16, 'mode': 'forward'}
    )
)
def bench_layer_norm(M, N, dtype, provider, mode, eps=1e-5, device='cuda'):
    # create data
    x_shape = (M, N)
    w_shape = (x_shape[-1], )
    weight = torch.rand(w_shape, dtype=dtype, device='cuda', requires_grad=True)
    bias = torch.rand(w_shape, dtype=dtype, device='cuda', requires_grad=True)
    x = -2.3 + 0.5 * torch.randn(x_shape, dtype=dtype, device='cuda')
    dy = .1 * torch.randn_like(x)
    x.requires_grad_(True)
    # utility functions
    if provider == 'triton':
        y_fwd = lambda: layer_norm(x, w_shape, weight, bias, eps)
    if provider == 'torch':
        y_fwd = lambda: torch.nn.functional.layer_norm(x, w_shape, weight, bias, eps)
    if provider == 'apex':
        apex_layer_norm = apex.normalization.FusedLayerNorm(w_shape).to(x.device).to(x.dtype)
        y_fwd = lambda: apex_layer_norm(x)
    # forward pass
    if mode == 'forward':
        gbps = lambda ms: 2 * x.numel() * x.element_size() / ms * 1e-6
        ms, min_ms, max_ms = triton.testing.do_bench(y_fwd, rep=500)
    # backward pass
    if mode == 'backward':
        gbps = lambda ms: 3 * x.numel() * x.element_size() / ms * 1e-6
        y = y_fwd()
        ms, min_ms, max_ms = triton.testing.do_bench(lambda: y.backward(dy, retain_graph=True),
                                                     grad_to_none=[x], rep=500)
    return gbps(ms), gbps(max_ms), gbps(min_ms)

# test_layer_norm(1151, 8192, torch.float16)
bench_layer_norm.run(save_path='.', print_data=True)
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

from abc import ABCMeta, abstractmethod
from weakref import WeakKeyDictionary

from six import add_metaclass, text_type

NoValue = object()

@add_metaclass(ABCMeta)
class BaseParameter(object):
    def __init__(self, type_, required):
        self._name = None
        self._required = required
        self._type = type_
        self._data = WeakKeyDictionary()

    def get_value(self, instance):
        return self._data.get(instance, NoValue)

    def __get__(self, instance, owner):
        if instance is None:
            return self
        return self._get_value(instance, owner)

    def __set__(self, instance, value):
        if self._type is not None and not isinstance(value, self._type):
            raise TypeError('Expected value {!r} to be of type {!r}'.format(
                value, self._type))
        self._data[instance] = value

    def set_attribute_name(self, name):
        self._name = name

    def _get_value(self, instance, owner):
        value = self.get_value(instance)
        if self._required and value is NoValue:
            raise ValueError(
                'Missing required attribute {!r}'.format(self._name))
        return value

    @abstractmethod
    def format(self, value):
        """Format the key and value for Google Analytics.

        """

class Parameter(BaseParameter):

    def __init__(self, target_name, type_=None, required=False):
        super(Parameter, self).__init__(type_, required)
        self._target_name = target_name

    def format(self, value):
        return (self._target_name, value)

class _CustomParameter(BaseParameter):

    FORMAT = None
    TYPE = None

    def __init__(self, index, required=False):
        super(_CustomParameter, self).__init__(self.TYPE, required)
        self._index = index

    def format(self, value):
        return (self.FORMAT.format(self._index), value)

class CustomDimension(_CustomParameter):

    FORMAT = 'cd{:d}'
    TYPE = text_type

class CustomMetric(_CustomParameter):

    FORMAT = 'cm{:d}'
    TYPE = int

class Encoder(object):
    def __init__(self, **kwargs):
        for name, value in kwargs.items():
            setattr(self, name, value)

    def __setattr__(self, name, value):
        if name not in self._tracked_attributes:
            raise AttributeError(name)
        super(Encoder, self).__setattr__(name, value)

    def encode(self):
        """Encode the object to a dictionary with the keys corresponding to
        the expected Google Analytics attributes.

        """
        encoded = {}
        type_ = type(self)
        for attribute_name in self._tracked_attributes:
            item = getattr(self, attribute_name)
            if item is NoValue:
                continue
            formatter = getattr(type_, attribute_name)
            key, value = formatter.format(item)
            encoded[key] = value
        return encoded

    def to_dict(self):
        """Dump the object value as a dictionary. This can be used in
        reonstructing a new object.

        """
        encoded = {}
        type_ = type(self)
        for attribute_name in self._tracked_attributes:
            descriptor = getattr(type_, attribute_name)
            value = descriptor.get_value(self)
            if value is NoValue:
                continue
            encoded[attribute_name] = value
        return encoded

class EventEncoder(type):

    def __new__(cls, class_name, bases, class_dict):
        bases = tuple(base for base in bases if base is not object)
        if Encoder not in bases:
            bases = bases + (Encoder,)
        tracked_attributes = set()
        for base in bases:
            tracked_attributes.update(
                set(getattr(base, '_tracked_attributes', set())))
        for key, value in list(class_dict.items()):
            if isinstance(value, BaseParameter):
                value.set_attribute_name(key)
                tracked_attributes.add(key)
        class_dict['_tracked_attributes'] = tuple(tracked_attributes)
        return type.__new__(cls, class_name, bases, class_dict)
<commit_msg>Fix typo<commit_after>from __future__ import absolute_import, unicode_literals

from abc import ABCMeta, abstractmethod
from weakref import WeakKeyDictionary

from six import add_metaclass, text_type

NoValue = object()

@add_metaclass(ABCMeta)
class BaseParameter(object):
    def __init__(self, type_, required):
        self._name = None
        self._required = required
        self._type = type_
        self._data = WeakKeyDictionary()

    def get_value(self, instance):
        return self._data.get(instance, NoValue)

    def __get__(self, instance, owner):
        if instance is None:
            return self
        return self._get_value(instance, owner)

    def __set__(self, instance, value):
        if self._type is not None and not isinstance(value, self._type):
            raise TypeError('Expected value {!r} to be of type {!r}'.format(
                value, self._type))
        self._data[instance] = value

    def set_attribute_name(self, name):
        self._name = name

    def _get_value(self, instance, owner):
        value = self.get_value(instance)
        if self._required and value is NoValue:
            raise ValueError(
                'Missing required attribute {!r}'.format(self._name))
        return value

    @abstractmethod
    def format(self, value):
        """Format the key and value for Google Analytics.

        """

class Parameter(BaseParameter):

    def __init__(self, target_name, type_=None, required=False):
        super(Parameter, self).__init__(type_, required)
        self._target_name = target_name

    def format(self, value):
        return (self._target_name, value)

class _CustomParameter(BaseParameter):

    FORMAT = None
    TYPE = None

    def __init__(self, index, required=False):
        super(_CustomParameter, self).__init__(self.TYPE, required)
        self._index = index

    def format(self, value):
        return (self.FORMAT.format(self._index), value)

class CustomDimension(_CustomParameter):

    FORMAT = 'cd{:d}'
    TYPE = text_type

class CustomMetric(_CustomParameter):

    FORMAT = 'cm{:d}'
    TYPE = int

class Encoder(object):
    def __init__(self, **kwargs):
        for name, value in kwargs.items():
            setattr(self, name, value)

    def __setattr__(self, name, value):
        if name not in self._tracked_attributes:
            raise AttributeError(name)
        super(Encoder, self).__setattr__(name, value)

    def encode(self):
        """Encode the object to a dictionary with the keys corresponding to
        the expected Google Analytics attributes.

        """
        encoded = {}
        type_ = type(self)
        for attribute_name in self._tracked_attributes:
            item = getattr(self, attribute_name)
            if item is NoValue:
                continue
            formatter = getattr(type_, attribute_name)
            key, value = formatter.format(item)
            encoded[key] = value
        return encoded

    def to_dict(self):
        """Dump the object value as a dictionary. This can be used in
        reconstructing a new object.

        """
        encoded = {}
        type_ = type(self)
        for attribute_name in self._tracked_attributes:
            descriptor = getattr(type_, attribute_name)
            value = descriptor.get_value(self)
            if value is NoValue:
                continue
            encoded[attribute_name] = value
        return encoded

class EventEncoder(type):

    def __new__(cls, class_name, bases, class_dict):
        bases = tuple(base for base in bases if base is not object)
        if Encoder not in bases:
            bases = bases + (Encoder,)
        tracked_attributes = set()
        for base in bases:
            tracked_attributes.update(
                set(getattr(base, '_tracked_attributes', set())))
        for key, value in list(class_dict.items()):
            if isinstance(value, BaseParameter):
                value.set_attribute_name(key)
                tracked_attributes.add(key)
        class_dict['_tracked_attributes'] = tuple(tracked_attributes)
        return type.__new__(cls, class_name, bases, class_dict)
<|endoftext|>