<commit_before>import numpy as np
from scipy.special import gamma,psi
from scipy.spatial.distance import cdist

# Get the k nearest neighbors
# between points in a random variable/vector.
# Uses Euclidean style distances.
def k_nearest_neighbors(X, k=1):
    #length
    nX = len(X)
    #initialize knn dict
    knn = {key: [] for key in xrange(nX)}
    #make sure X has the right shape for the cdist function
    X = np.reshape(X, (nX,-1))
    dists_arr = cdist(X, X)
    distances = [[i,j,dists_arr[i,j]] for i in xrange(nX-1) for j in xrange(i+1,nX)]
    #sort distances
    distances.sort(key=lambda x: x[2])
    #pick up the k nearest
    for d in distances:
        i = d[0]
        j = d[1]
        dist = d[2]
        if len(knn[i]) < k:
            knn[i].append([j, dist])
        if len(knn[j]) < k:
            knn[j].append([i, dist])
    return knn

# knn kth neighbor distances for entropy calcs.
def kth_nearest_neighbor_distances(X, k=1):

    #length
    nX = len(X)
    #make sure X has the right shape for the cdist function
    X = np.reshape(X, (nX,-1))
    dists_arr = cdist(X, X)
    #sorts each row
    dists_arr.sort()
    return [dists_arr[i][k] for i in xrange(nX)]

def shannon_entropy(X, k=1, kth_dists=None):
    # KL entropy estimator from
    #  https://arxiv.org/pdf/1506.06501v1.pdf
    # Also see:
    # https://www.cs.tut.fi/~timhome/tim/tim/core/differential_entropy_kl_details.htm
    # and
    # Kozachenko, L. F. & Leonenko, N. N. 1987 Sample estimate of entropy
    #    of a random vector. Probl. Inf. Transm. 23, 95-101.
    # the kth nearest neighbor distances
    r_k = kth_dists
    if kth_dists is None:
        r_k = kth_nearest_neighbor_distances(X, k=k)
    #length
    n = len(X)
    #dimension
    d = 1
    if len(X.shape) == 2:
        d = X.shape[1]
    # volume of unit ball in d^n
    v_unit_ball = np.pi**(0.5*d)/gamma(0.5*d + 1.0)
    # log distances
    lr_k = np.log(r_k)
    # Shannon entropy estimate
    H = psi(n) - psi(k) + np.log(v_unit_ball) + (np.float(d)/np.float(n))*( lr_k.sum())

    return H

# Not entirely sure this one is correct.
def shannon_entropy_pc(X, k=1, kth_dists=None):
    # entropy estimator from
    # F. Perez-Cruz, (2008). Estimation of Information Theoretic Measures
    #     for Continuous Random Variables. Advances in Neural Information
    #     Processing Systems 21 (NIPS). Vancouver (Canada), December.
    #    https://papers.nips.cc/paper/3417-estimation-of-information-theoretic-measures-for-continuous-random-variables.pdf
    #
    #
    r_k = kth_dists
    if kth_dists is None:
        r_k = np.array(kth_nearest_neighbor_distances(X, k=k))
    n = len(X)
    d = 1
    if len(X.shape) == 2:
        d = X.shape[1]
    #volume of the unit ball
    v_unit_ball = np.pi**(0.5*d)/gamma(0.5*d + 1.0)
    # probability estimator using knn distances
    p_k_hat = (k / (n -1.0)) * (1.0/v_unit_ball) * (1.0/r_k**d)
    # log probability
    log_p_k_hat = np.log(p_k_hat)
    #entropy estimator
    h_k_hat = log_p_k_hat.sum() / (-1.0*n)
    return h_k_hat

#mutual information
def mutual_information(var_tuple, k=2):
    nvar = len(var_tuple)
    #make sure the input arrays are properly shaped for hstacking
    var_tuple = tuple( var_tuple[i].reshape(len(var_tuple[i],-1)) for i in xrange(nvar) )
    #compute the individual entropies of each variable
    Hx = [shannon_entropy(var_tuple[i],k=k) for i in xrange(nvar)]
    Hx = np.array(Hx)
    # and get the sum
    Hxtot = Hx.sum()
    #now get the entropy of the joint distribution
    joint = np.hstack(var_tuple)
    Hjoint = shannon_entropy(joint, k=k)
    #get the mutual information
    MI = Hxtot - Hjoint
    #set to zero if value is negative
    if MI < 0.0: MI = 0.0
    #return
    return MI

# conditional mutual information
def conditional_mutual_information(var_tuple, cond_tuple, k=2):
    nvar = len(var_tuple)
    ncon = len(cond_tuple)
    #make sure the input arrays are properly shaped for hstacking
    var_tuple = tuple( var_tuple[i].reshape(len(var_tuple[i],-1)) for i in xrange(nvar) )
    cond_tuple = tuple( cond_tuple[i].reshape(len(cond_tuple[i],-1)) for i in xrange(ncon) )
    # compute pair joint entropies
    Hxz = [shannon_entropy(np.hstack(var_tuple[i]+cond_tuple), k=k) for i in xrange(nvar)]
    Hxz = np.array(Hxz)
    jtup = var_tuple + cond_tuple
    joint = np.hstack( jtup )
    Hj = shannon_entropy(joint, k=k)
    Hz = 0.0
    if len(cond_tuple) > 1:
        joint = np.hstack(cond_tuple)
        Hz = shannon_entropy(joint, k=k)
    else:
        Hz = shannon_entropy(cond_tuple[0], k=k)
    Hxzsum = Hxz.sum()
   # print "Hxzsum: ",Hxzsum, " Hj: ",Hj, " Hz: ",Hz
    MIc = Hxzsum - Hj - Hz
   # print "MIc: ",MIc
    if MIc < 0.0: MIc = 0.0
    #print MIc
    return MIc
<commit_msg>parentheses fix<commit_after>import numpy as np
from scipy.special import gamma,psi
from scipy.spatial.distance import cdist

# Get the k nearest neighbors
# between points in a random variable/vector.
# Uses Euclidean style distances.
def k_nearest_neighbors(X, k=1):
    #length
    nX = len(X)
    #initialize knn dict
    knn = {key: [] for key in xrange(nX)}
    #make sure X has the right shape for the cdist function
    X = np.reshape(X, (nX,-1))
    dists_arr = cdist(X, X)
    distances = [[i,j,dists_arr[i,j]] for i in xrange(nX-1) for j in xrange(i+1,nX)]
    #sort distances
    distances.sort(key=lambda x: x[2])
    #pick up the k nearest
    for d in distances:
        i = d[0]
        j = d[1]
        dist = d[2]
        if len(knn[i]) < k:
            knn[i].append([j, dist])
        if len(knn[j]) < k:
            knn[j].append([i, dist])
    return knn

# knn kth neighbor distances for entropy calcs.
def kth_nearest_neighbor_distances(X, k=1):

    #length
    nX = len(X)
    #make sure X has the right shape for the cdist function
    X = np.reshape(X, (nX,-1))
    dists_arr = cdist(X, X)
    #sorts each row
    dists_arr.sort()
    return [dists_arr[i][k] for i in xrange(nX)]

def shannon_entropy(X, k=1, kth_dists=None):
    # KL entropy estimator from
    #  https://arxiv.org/pdf/1506.06501v1.pdf
    # Also see:
    # https://www.cs.tut.fi/~timhome/tim/tim/core/differential_entropy_kl_details.htm
    # and
    # Kozachenko, L. F. & Leonenko, N. N. 1987 Sample estimate of entropy
    #    of a random vector. Probl. Inf. Transm. 23, 95-101.
    # the kth nearest neighbor distances
    r_k = kth_dists
    if kth_dists is None:
        r_k = kth_nearest_neighbor_distances(X, k=k)
    #length
    n = len(X)
    #dimension
    d = 1
    if len(X.shape) == 2:
        d = X.shape[1]
    # volume of unit ball in d^n
    v_unit_ball = np.pi**(0.5*d)/gamma(0.5*d + 1.0)
    # log distances
    lr_k = np.log(r_k)
    # Shannon entropy estimate
    H = psi(n) - psi(k) + np.log(v_unit_ball) + (np.float(d)/np.float(n))*( lr_k.sum())

    return H

# Not entirely sure this one is correct.
def shannon_entropy_pc(X, k=1, kth_dists=None):
    # entropy estimator from
    # F. Perez-Cruz, (2008). Estimation of Information Theoretic Measures
    #     for Continuous Random Variables. Advances in Neural Information
    #     Processing Systems 21 (NIPS). Vancouver (Canada), December.
    #    https://papers.nips.cc/paper/3417-estimation-of-information-theoretic-measures-for-continuous-random-variables.pdf
    #
    #
    r_k = kth_dists
    if kth_dists is None:
        r_k = np.array(kth_nearest_neighbor_distances(X, k=k))
    n = len(X)
    d = 1
    if len(X.shape) == 2:
        d = X.shape[1]
    #volume of the unit ball
    v_unit_ball = np.pi**(0.5*d)/gamma(0.5*d + 1.0)
    # probability estimator using knn distances
    p_k_hat = (k / (n -1.0)) * (1.0/v_unit_ball) * (1.0/r_k**d)
    # log probability
    log_p_k_hat = np.log(p_k_hat)
    #entropy estimator
    h_k_hat = log_p_k_hat.sum() / (-1.0*n)
    return h_k_hat

#mutual information
def mutual_information(var_tuple, k=2):
    nvar = len(var_tuple)
    #make sure the input arrays are properly shaped for hstacking
    var_tuple = tuple( var_tuple[i].reshape(len(var_tuple[i]),-1) for i in xrange(nvar) )
    #compute the individual entropies of each variable
    Hx = [shannon_entropy(var_tuple[i],k=k) for i in xrange(nvar)]
    Hx = np.array(Hx)
    # and get the sum
    Hxtot = Hx.sum()
    #now get the entropy of the joint distribution
    joint = np.hstack(var_tuple)
    Hjoint = shannon_entropy(joint, k=k)
    #get the mutual information
    MI = Hxtot - Hjoint
    #set to zero if value is negative
    if MI < 0.0: MI = 0.0
    #return
    return MI

# conditional mutual information
def conditional_mutual_information(var_tuple, cond_tuple, k=2):
    nvar = len(var_tuple)
    ncon = len(cond_tuple)
    #make sure the input arrays are properly shaped for hstacking
    var_tuple = tuple( var_tuple[i].reshape(len(var_tuple[i]),-1) for i in xrange(nvar) )
    cond_tuple = tuple( cond_tuple[i].reshape(len(cond_tuple[i]),-1) for i in xrange(ncon) )
    # compute pair joint entropies
    Hxz = [shannon_entropy(np.hstack(var_tuple[i]+cond_tuple), k=k) for i in xrange(nvar)]
    Hxz = np.array(Hxz)
    jtup = var_tuple + cond_tuple
    joint = np.hstack( jtup )
    Hj = shannon_entropy(joint, k=k)
    Hz = 0.0
    if len(cond_tuple) > 1:
        joint = np.hstack(cond_tuple)
        Hz = shannon_entropy(joint, k=k)
    else:
        Hz = shannon_entropy(cond_tuple[0], k=k)
    Hxzsum = Hxz.sum()
   # print "Hxzsum: ",Hxzsum, " Hj: ",Hj, " Hz: ",Hz
    MIc = Hxzsum - Hj - Hz
   # print "MIc: ",MIc
    if MIc < 0.0: MIc = 0.0
    #print MIc
    return MIc
<|endoftext|>


<commit_before>"""
nodes.py: definition of an EmaneNode class for implementing configuration
control of an EMANE emulation. An EmaneNode has several attached NEMs that
share the same MAC+PHY model.
"""

from core import logger
from core.coreobj import PyCoreNet
from core.enumerations import LinkTypes
from core.enumerations import NodeTypes
from core.enumerations import RegisterTlvs

try:
    from emane.events import LocationEvent
except ImportError:
    try:
        from emanesh.events import LocationEvent
    except ImportError:
        logger.warn("compatible emane python bindings not installed")

class EmaneNet(PyCoreNet):
    """
    EMANE network base class.
    """
    apitype = NodeTypes.EMANE.value
    linktype = LinkTypes.WIRELESS.value
    # icon used
    type = "wlan"

class EmaneNode(EmaneNet):
    """
    EMANE node contains NEM configuration and causes connected nodes
    to have TAP interfaces (instead of VEth). These are managed by the
    Emane controller object that exists in a session.
    """

    def __init__(self, session, objid=None, name=None, start=True):
        PyCoreNet.__init__(self, session, objid, name, start)
        self.conf = ""
        self.up = False
        self.nemidmap = {}
        self.model = None
        self.mobility = None

    def linkconfig(self, netif, bw=None, delay=None, loss=None, duplicate=None, jitter=None, netif2=None):
        """
        The CommEffect model supports link configuration.
        """
        if not self.model:
            return
        return self.model.linkconfig(netif=netif, bw=bw, delay=delay, loss=loss,
                                     duplicate=duplicate, jitter=jitter, netif2=netif2)

    def config(self, conf):
        self.conf = conf

    def shutdown(self):
        pass

    def link(self, netif1, netif2):
        pass

    def unlink(self, netif1, netif2):
        pass

    def updatemodel(self, config):
        if not self.model:
            raise ValueError("no model set to update for node(%s)", self.objid)
        logger.info("node(%s) updating model(%s): %s", self.objid, self.model.name, config)
        self.model.set_configs(config, node_id=self.objid)

    def setmodel(self, model, config):
        """
        set the EmaneModel associated with this node
        """
        logger.info("adding model: %s", model.name)
        if model.config_type == RegisterTlvs.WIRELESS.value:
            # EmaneModel really uses values from ConfigurableManager
            #  when buildnemxml() is called, not during init()
            self.model = model(session=self.session, object_id=self.objid)
            self.model.update_config(config)
        elif model.config_type == RegisterTlvs.MOBILITY.value:
            self.mobility = model(session=self.session, object_id=self.objid)
            self.mobility.update_config(config)

    def setnemid(self, netif, nemid):
        """
        Record an interface to numerical ID mapping. The Emane controller
        object manages and assigns these IDs for all NEMs.
        """
        self.nemidmap[netif] = nemid

    def getnemid(self, netif):
        """
        Given an interface, return its numerical ID.
        """
        if netif not in self.nemidmap:
            return None
        else:
            return self.nemidmap[netif]

    def getnemnetif(self, nemid):
        """
        Given a numerical NEM ID, return its interface. This returns the
        first interface that matches the given NEM ID.
        """
        for netif in self.nemidmap:
            if self.nemidmap[netif] == nemid:
                return netif
        return None

    def netifs(self, sort=True):
        """
        Retrieve list of linked interfaces sorted by node number.
        """
        return sorted(self._netif.values(), key=lambda ifc: ifc.node.objid)

    def installnetifs(self):
        """
        Install TAP devices into their namespaces. This is done after
        EMANE daemons have been started, because that is their only chance
        to bind to the TAPs.
        """
        if self.session.emane.genlocationevents() and self.session.emane.service is None:
            warntxt = "unable to publish EMANE events because the eventservice "
            warntxt += "Python bindings failed to load"
            logger.error(warntxt)

        for netif in self.netifs():
            if not self.session.emane.genlocationevents():
                netif.poshook = None
                continue
            # at this point we register location handlers for generating
            # EMANE location events
            netif.poshook = self.setnemposition
            x, y, z = netif.node.position.get()
            self.setnemposition(netif, x, y, z)

    def deinstallnetifs(self):
        """
        Uninstall TAP devices. This invokes their shutdown method for
        any required cleanup; the device may be actually removed when
        emanetransportd terminates.
        """
        for netif in self.netifs():
            if "virtual" in netif.transport_type.lower():
                netif.shutdown()
            netif.poshook = None

    def setnemposition(self, netif, x, y, z):
        """
        Publish a NEM location change event using the EMANE event service.
        """
        if self.session.emane.service is None:
            logger.info("position service not available")
            return
        nemid = self.getnemid(netif)
        ifname = netif.localname
        if nemid is None:
            logger.info("nemid for %s is unknown" % ifname)
            return
        lat, long, alt = self.session.location.getgeo(x, y, z)
        logger.info("setnemposition %s (%s) x,y,z=(%d,%d,%s)(%.6f,%.6f,%.6f)", ifname, nemid, x, y, z, lat, long, alt)
        event = LocationEvent()

        # altitude must be an integer or warning is printed
        # unused: yaw, pitch, roll, azimuth, elevation, velocity
        alt = int(round(alt))
        event.append(nemid, latitude=lat, longitude=long, altitude=alt)
        self.session.emane.service.publish(0, event)

    def setnempositions(self, moved_netifs):
        """
        Several NEMs have moved, from e.g. a WaypointMobilityModel
        calculation. Generate an EMANE Location Event having several
        entries for each netif that has moved.
        """
        if len(moved_netifs) == 0:
            return

        if self.session.emane.service is None:
            logger.info("position service not available")
            return

        event = LocationEvent()
        i = 0
        for netif in moved_netifs:
            nemid = self.getnemid(netif)
            ifname = netif.localname
            if nemid is None:
                logger.info("nemid for %s is unknown" % ifname)
                continue
            x, y, z = netif.node.getposition()
            lat, long, alt = self.session.location.getgeo(x, y, z)
            logger.info("setnempositions %d %s (%s) x,y,z=(%d,%d,%s)(%.6f,%.6f,%.6f)",
                        i, ifname, nemid, x, y, z, lat, long, alt)
            # altitude must be an integer or warning is printed
            alt = int(round(alt))
            event.append(nemid, latitude=lat, longitude=long, altitude=alt)
            i += 1

        self.session.emane.service.publish(0, event)
<commit_msg>added back in core emane address creation, only when not external<commit_after>"""
nodes.py: definition of an EmaneNode class for implementing configuration
control of an EMANE emulation. An EmaneNode has several attached NEMs that
share the same MAC+PHY model.
"""

from core import logger
from core.coreobj import PyCoreNet
from core.enumerations import LinkTypes
from core.enumerations import NodeTypes
from core.enumerations import RegisterTlvs

try:
    from emane.events import LocationEvent
except ImportError:
    try:
        from emanesh.events import LocationEvent
    except ImportError:
        logger.warn("compatible emane python bindings not installed")

class EmaneNet(PyCoreNet):
    """
    EMANE network base class.
    """
    apitype = NodeTypes.EMANE.value
    linktype = LinkTypes.WIRELESS.value
    # icon used
    type = "wlan"

class EmaneNode(EmaneNet):
    """
    EMANE node contains NEM configuration and causes connected nodes
    to have TAP interfaces (instead of VEth). These are managed by the
    Emane controller object that exists in a session.
    """

    def __init__(self, session, objid=None, name=None, start=True):
        PyCoreNet.__init__(self, session, objid, name, start)
        self.conf = ""
        self.up = False
        self.nemidmap = {}
        self.model = None
        self.mobility = None

    def linkconfig(self, netif, bw=None, delay=None, loss=None, duplicate=None, jitter=None, netif2=None):
        """
        The CommEffect model supports link configuration.
        """
        if not self.model:
            return
        return self.model.linkconfig(netif=netif, bw=bw, delay=delay, loss=loss,
                                     duplicate=duplicate, jitter=jitter, netif2=netif2)

    def config(self, conf):
        self.conf = conf

    def shutdown(self):
        pass

    def link(self, netif1, netif2):
        pass

    def unlink(self, netif1, netif2):
        pass

    def updatemodel(self, config):
        if not self.model:
            raise ValueError("no model set to update for node(%s)", self.objid)
        logger.info("node(%s) updating model(%s): %s", self.objid, self.model.name, config)
        self.model.set_configs(config, node_id=self.objid)

    def setmodel(self, model, config):
        """
        set the EmaneModel associated with this node
        """
        logger.info("adding model: %s", model.name)
        if model.config_type == RegisterTlvs.WIRELESS.value:
            # EmaneModel really uses values from ConfigurableManager
            #  when buildnemxml() is called, not during init()
            self.model = model(session=self.session, object_id=self.objid)
            self.model.update_config(config)
        elif model.config_type == RegisterTlvs.MOBILITY.value:
            self.mobility = model(session=self.session, object_id=self.objid)
            self.mobility.update_config(config)

    def setnemid(self, netif, nemid):
        """
        Record an interface to numerical ID mapping. The Emane controller
        object manages and assigns these IDs for all NEMs.
        """
        self.nemidmap[netif] = nemid

    def getnemid(self, netif):
        """
        Given an interface, return its numerical ID.
        """
        if netif not in self.nemidmap:
            return None
        else:
            return self.nemidmap[netif]

    def getnemnetif(self, nemid):
        """
        Given a numerical NEM ID, return its interface. This returns the
        first interface that matches the given NEM ID.
        """
        for netif in self.nemidmap:
            if self.nemidmap[netif] == nemid:
                return netif
        return None

    def netifs(self, sort=True):
        """
        Retrieve list of linked interfaces sorted by node number.
        """
        return sorted(self._netif.values(), key=lambda ifc: ifc.node.objid)

    def installnetifs(self):
        """
        Install TAP devices into their namespaces. This is done after
        EMANE daemons have been started, because that is their only chance
        to bind to the TAPs.
        """
        if self.session.emane.genlocationevents() and self.session.emane.service is None:
            warntxt = "unable to publish EMANE events because the eventservice "
            warntxt += "Python bindings failed to load"
            logger.error(warntxt)

        for netif in self.netifs():
            external = self.session.emane.get_config("external", self.objid, self.model.name)
            if external == "0":
                logger.info("I AM NOT SKIPPING EMANE ADDRESSES")
                netif.setaddrs()
            else:
                logger.info("I AM SKIPPING EMANE ADDRESSES")

            if not self.session.emane.genlocationevents():
                netif.poshook = None
                continue

            # at this point we register location handlers for generating
            # EMANE location events
            netif.poshook = self.setnemposition
            x, y, z = netif.node.position.get()
            self.setnemposition(netif, x, y, z)

    def deinstallnetifs(self):
        """
        Uninstall TAP devices. This invokes their shutdown method for
        any required cleanup; the device may be actually removed when
        emanetransportd terminates.
        """
        for netif in self.netifs():
            if "virtual" in netif.transport_type.lower():
                netif.shutdown()
            netif.poshook = None

    def setnemposition(self, netif, x, y, z):
        """
        Publish a NEM location change event using the EMANE event service.
        """
        if self.session.emane.service is None:
            logger.info("position service not available")
            return
        nemid = self.getnemid(netif)
        ifname = netif.localname
        if nemid is None:
            logger.info("nemid for %s is unknown" % ifname)
            return
        lat, long, alt = self.session.location.getgeo(x, y, z)
        logger.info("setnemposition %s (%s) x,y,z=(%d,%d,%s)(%.6f,%.6f,%.6f)", ifname, nemid, x, y, z, lat, long, alt)
        event = LocationEvent()

        # altitude must be an integer or warning is printed
        # unused: yaw, pitch, roll, azimuth, elevation, velocity
        alt = int(round(alt))
        event.append(nemid, latitude=lat, longitude=long, altitude=alt)
        self.session.emane.service.publish(0, event)

    def setnempositions(self, moved_netifs):
        """
        Several NEMs have moved, from e.g. a WaypointMobilityModel
        calculation. Generate an EMANE Location Event having several
        entries for each netif that has moved.
        """
        if len(moved_netifs) == 0:
            return

        if self.session.emane.service is None:
            logger.info("position service not available")
            return

        event = LocationEvent()
        i = 0
        for netif in moved_netifs:
            nemid = self.getnemid(netif)
            ifname = netif.localname
            if nemid is None:
                logger.info("nemid for %s is unknown" % ifname)
                continue
            x, y, z = netif.node.getposition()
            lat, long, alt = self.session.location.getgeo(x, y, z)
            logger.info("setnempositions %d %s (%s) x,y,z=(%d,%d,%s)(%.6f,%.6f,%.6f)",
                        i, ifname, nemid, x, y, z, lat, long, alt)
            # altitude must be an integer or warning is printed
            alt = int(round(alt))
            event.append(nemid, latitude=lat, longitude=long, altitude=alt)
            i += 1

        self.session.emane.service.publish(0, event)
<|endoftext|>


<commit_before># ___________________________________________________________________________
# Slot Names

CHARACTER_VALUE_INDEX = 0        # Page 630 of the blue book

STREAM_ARRAY_INDEX = 0           # Page 631 of the blue book
STREAM_INDEX_INDEX = 1
STREAM_READ_LIMIT_INDEX = 2
STREAM_WRITE_LIMIT_INDEX = 3

CLASS_SUPERCLASS_INDEX = 0
CLASS_METHODDICT_INDEX = 1
CLASS_FORMAT_INDEX = 2
CLASS_NAME_INDEX = 6                # in the mini.image, at least

# MethodDict
METHODDICT_TALLY_INDEX = 0
METHODDICT_VALUES_INDEX = 1
METHODDICT_NAMES_INDEX  = 2

# Message
MESSAGE_SELECTOR_INDEX = 0
MESSAGE_ARGUMENTS_INDEX = 1
MESSAGE_LOOKUP_CLASS_INDEX = 2

# ContextPart
CTXPART_SENDER_INDEX = 0
CTXPART_PC_INDEX = 1
CTXPART_STACKP_INDEX = 2

METHOD_HEADER_INDEX = 0

# BlockContext < ContextPart
BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX = 3
BLKCTX_INITIAL_IP_INDEX = 4
BLKCTX_HOME_INDEX = 5
BLKCTX_STACK_START = 6

# MethodContext < ContextPart
MTHDCTX_METHOD = 3
MTHDCTX_RECEIVER_MAP = 4
MTHDCTX_RECEIVER = 5
MTHDCTX_TEMP_FRAME_START = 6

# ___________________________________________________________________________
# Miscellaneous constants

LITERAL_START = 1 # index of the first literal after the method header
BYTES_PER_WORD = 4

# ___________________________________________________________________________
# Special objects indices

SO_NIL = 0
SO_FALSE = 1
SO_TRUE = 2
SO_SCHEDULERASSOCIATIONPOINTER = 3
SO_BITMAP_CLASS = 4
SO_SMALLINTEGER_CLASS = 5
SO_STRING_CLASS = 6
SO_ARRAY_CLASS = 7
SO_SMALLTALK = 8 # Deperacted
SO_FLOAT_CLASS = 9
SO_METHODCONTEXT_CLASS = 10
SO_BLOCKCONTEXT_CLASS = 11
SO_POINT_CLASS = 12
SO_LARGEPOSITIVEINTEGER_CLASS = 13
SO_DISPLAY_CLASS = 14
SO_MESSAGE_CLASS = 15
SO_COMPILEDMETHOD_CLASS = 16
SO_LOW_SPACE_SEMAPHORE = 17
SO_SEMAPHORE_CLASS = 18
SO_CHARACTER_CLASS = 19
SO_DOES_NOT_UNDERSTAND = 20
SO_CANNOT_RETURN = 21
SO_PROCESS_SIGNALIGN_LOW_SPACE = 22 # the process that triggered the low space semaphore. mostly nil
SO_SPECIAL_SELECTORS_ARRAY = 23
SO_CHARACTER_TABLE_ARRAY = 24
SO_MUST_BE_BOOLEAN = 25
SO_BYTEARRAY_CLASS = 26
SO_PROCESS_CLASS = 27
SO_COMPACT_CLASSES_ARRAY = 28
SO_DELAY_SEMAPHORE = 29
SO_USER_INTERRUPT_SEMAPHORE = 30
SO_FLOAT_ZERO = 31
SO_LARGEPOSITIVEINTEGER_ZERO = 32
SO_A_POINT = 33
SO_CANNOT_INTERPRET = 34
SO_A_METHODCONTEXT = 35 # deprecated in closure images
SO_BLOCKCLOSURE_CLASS = 36
SO_A_BLOCKCONTEXT = 37 # deprecated in closure images
SO_EXTERNAL_OBJECTS_ARRAY = 38
SO_PSEUDOCONTEXT_CLASS = 39
SO_TRANSLATEDMETHOD_CLASS = 40
SO_FINALIZATION_SEMPAHORE = 41
SO_LARGENEGATIVEINTEGER_CLASS = 42

# XXX more missing?
classes_in_special_object_table = {
#    "Bitmap" : SO_BITMAP_CLASS,
    "SmallInteger" : SO_SMALLINTEGER_CLASS,
    "String" : SO_STRING_CLASS,
    "Array" : SO_ARRAY_CLASS,
    "Float" : SO_FLOAT_CLASS,
    "MethodContext" : SO_METHODCONTEXT_CLASS,
    "BlockContext" : SO_BLOCKCONTEXT_CLASS,
    "BlockClosure" : SO_BLOCKCLOSURE_CLASS,
    "Point" : SO_POINT_CLASS,
    "LargePositiveInteger" : SO_LARGEPOSITIVEINTEGER_CLASS,
#    "Display" : SO_DISPLAY_CLASS,
#    "Message" : SO_MESSAGE_CLASS,
    "CompiledMethod" : SO_COMPILEDMETHOD_CLASS,
    "Semaphore" : SO_SEMAPHORE_CLASS,
    "Character" : SO_CHARACTER_CLASS,
    "ByteArray" : SO_BYTEARRAY_CLASS,
    "Process" : SO_PROCESS_CLASS,
#    "PseudoContext" : SO_PSEUDOCONTEXT_CLASS,
#    "TranslatedMethod" : SO_TRANSLATEDMETHOD_CLASS,
    # "LargeNegativeInteger" : SO_LARGENEGATIVEINTEGER_CLASS, # Not available in mini.image
}

objects_in_special_object_table = {
    "nil": SO_NIL,
    "true": SO_TRUE,
    "false": SO_FALSE,
    "charactertable": SO_CHARACTER_TABLE_ARRAY,
    "schedulerassociationpointer" : SO_SCHEDULERASSOCIATIONPOINTER,
    "smalltalkdict" : SO_SMALLTALK,
}

LONG_BIT = 32
TAGGED_MAXINT = 2 ** (LONG_BIT - 2) - 1
TAGGED_MININT = -2 ** (LONG_BIT - 2)
<commit_msg>Revert adding BlockClosures to constants.py, because it breaks the tests<commit_after># ___________________________________________________________________________
# Slot Names

CHARACTER_VALUE_INDEX = 0        # Page 630 of the blue book

STREAM_ARRAY_INDEX = 0           # Page 631 of the blue book
STREAM_INDEX_INDEX = 1
STREAM_READ_LIMIT_INDEX = 2
STREAM_WRITE_LIMIT_INDEX = 3

CLASS_SUPERCLASS_INDEX = 0
CLASS_METHODDICT_INDEX = 1
CLASS_FORMAT_INDEX = 2
CLASS_NAME_INDEX = 6                # in the mini.image, at least

# MethodDict
METHODDICT_TALLY_INDEX = 0
METHODDICT_VALUES_INDEX = 1
METHODDICT_NAMES_INDEX  = 2

# Message
MESSAGE_SELECTOR_INDEX = 0
MESSAGE_ARGUMENTS_INDEX = 1
MESSAGE_LOOKUP_CLASS_INDEX = 2

# ContextPart
CTXPART_SENDER_INDEX = 0
CTXPART_PC_INDEX = 1
CTXPART_STACKP_INDEX = 2

METHOD_HEADER_INDEX = 0

# BlockContext < ContextPart
BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX = 3
BLKCTX_INITIAL_IP_INDEX = 4
BLKCTX_HOME_INDEX = 5
BLKCTX_STACK_START = 6

# MethodContext < ContextPart
MTHDCTX_METHOD = 3
MTHDCTX_RECEIVER_MAP = 4
MTHDCTX_RECEIVER = 5
MTHDCTX_TEMP_FRAME_START = 6

# ___________________________________________________________________________
# Miscellaneous constants

LITERAL_START = 1 # index of the first literal after the method header
BYTES_PER_WORD = 4

# ___________________________________________________________________________
# Special objects indices

SO_NIL = 0
SO_FALSE = 1
SO_TRUE = 2
SO_SCHEDULERASSOCIATIONPOINTER = 3
SO_BITMAP_CLASS = 4
SO_SMALLINTEGER_CLASS = 5
SO_STRING_CLASS = 6
SO_ARRAY_CLASS = 7
SO_SMALLTALK = 8 # Deperacted
SO_FLOAT_CLASS = 9
SO_METHODCONTEXT_CLASS = 10
SO_BLOCKCONTEXT_CLASS = 11
SO_POINT_CLASS = 12
SO_LARGEPOSITIVEINTEGER_CLASS = 13
SO_DISPLAY_CLASS = 14
SO_MESSAGE_CLASS = 15
SO_COMPILEDMETHOD_CLASS = 16
SO_LOW_SPACE_SEMAPHORE = 17
SO_SEMAPHORE_CLASS = 18
SO_CHARACTER_CLASS = 19
SO_DOES_NOT_UNDERSTAND = 20
SO_CANNOT_RETURN = 21
SO_PROCESS_SIGNALIGN_LOW_SPACE = 22 # the process that triggered the low space semaphore. mostly nil
SO_SPECIAL_SELECTORS_ARRAY = 23
SO_CHARACTER_TABLE_ARRAY = 24
SO_MUST_BE_BOOLEAN = 25
SO_BYTEARRAY_CLASS = 26
SO_PROCESS_CLASS = 27
SO_COMPACT_CLASSES_ARRAY = 28
SO_DELAY_SEMAPHORE = 29
SO_USER_INTERRUPT_SEMAPHORE = 30
SO_FLOAT_ZERO = 31
SO_LARGEPOSITIVEINTEGER_ZERO = 32
SO_A_POINT = 33
SO_CANNOT_INTERPRET = 34
SO_A_METHODCONTEXT = 35 # deprecated in closure images
SO_BLOCKCLOSURE_CLASS = 36
SO_A_BLOCKCONTEXT = 37 # deprecated in closure images
SO_EXTERNAL_OBJECTS_ARRAY = 38
SO_PSEUDOCONTEXT_CLASS = 39
SO_TRANSLATEDMETHOD_CLASS = 40
SO_FINALIZATION_SEMPAHORE = 41
SO_LARGENEGATIVEINTEGER_CLASS = 42

# XXX more missing?
classes_in_special_object_table = {
#    "Bitmap" : SO_BITMAP_CLASS,
    "SmallInteger" : SO_SMALLINTEGER_CLASS,
    "String" : SO_STRING_CLASS,
    "Array" : SO_ARRAY_CLASS,
    "Float" : SO_FLOAT_CLASS,
    "MethodContext" : SO_METHODCONTEXT_CLASS,
    "BlockContext" : SO_BLOCKCONTEXT_CLASS,
    # "BlockClosure" : SO_BLOCKCLOSURE_CLASS,
    "Point" : SO_POINT_CLASS,
    "LargePositiveInteger" : SO_LARGEPOSITIVEINTEGER_CLASS,
#    "Display" : SO_DISPLAY_CLASS,
#    "Message" : SO_MESSAGE_CLASS,
    "CompiledMethod" : SO_COMPILEDMETHOD_CLASS,
    "Semaphore" : SO_SEMAPHORE_CLASS,
    "Character" : SO_CHARACTER_CLASS,
    "ByteArray" : SO_BYTEARRAY_CLASS,
    "Process" : SO_PROCESS_CLASS,
#    "PseudoContext" : SO_PSEUDOCONTEXT_CLASS,
#    "TranslatedMethod" : SO_TRANSLATEDMETHOD_CLASS,
    # "LargeNegativeInteger" : SO_LARGENEGATIVEINTEGER_CLASS, # Not available in mini.image
}

objects_in_special_object_table = {
    "nil": SO_NIL,
    "true": SO_TRUE,
    "false": SO_FALSE,
    "charactertable": SO_CHARACTER_TABLE_ARRAY,
    "schedulerassociationpointer" : SO_SCHEDULERASSOCIATIONPOINTER,
    "smalltalkdict" : SO_SMALLTALK,
}

LONG_BIT = 32
TAGGED_MAXINT = 2 ** (LONG_BIT - 2) - 1
TAGGED_MININT = -2 ** (LONG_BIT - 2)
<|endoftext|>


<commit_before># Copyright 2019 Google, Inc.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from m5.objects.FastModelGIC import FastModelGIC, SCFastModelGIC
from m5.objects.RealView import VExpress_GEM5_Base, HDLcd

class VExpressFastmodel(VExpress_GEM5_Base):
    gic = FastModelGIC(
        sc_gic=SCFastModelGIC(
            reg_base=0x2c000000,
            reg_base_per_redistributor="0.0.0.0=0x2c010000",
            spi_count=988,
        ))

    hdlcd = HDLcd(
        pxl_clk=VExpress_GEM5_Base.dcc.osc_pxl, pio_addr=0x2b000000, int_num=95)

    def __init__(self, *args, **kwargs):
        super(VExpressFastmodel, self).__init__(*args, **kwargs)

    def _on_chip_devices(self):
        devices = super(VExpressFastmodel, self)._on_chip_devices()
        devices += [ self.gic, self.hdlcd ]
        devices.remove(self.generic_timer)
        return devices

    def setupBootLoader(self, cur_sys, loc):
        super(VExpressFastmodel,self).setupBootLoader(cur_sys, loc)
        # Override the boot_loader setting
        cur_sys.boot_loader = [ loc('boot_v2.arm64') ]
<commit_msg>dev-arm: Fix setupBootloader for VExpressFastmodel<commit_after># Copyright 2019 Google, Inc.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from m5.objects.FastModelGIC import FastModelGIC, SCFastModelGIC
from m5.objects.RealView import VExpress_GEM5_Base, HDLcd

class VExpressFastmodel(VExpress_GEM5_Base):
    gic = FastModelGIC(
        sc_gic=SCFastModelGIC(
            reg_base=0x2c000000,
            reg_base_per_redistributor="0.0.0.0=0x2c010000",
            spi_count=988,
        ))

    hdlcd = HDLcd(
        pxl_clk=VExpress_GEM5_Base.dcc.osc_pxl, pio_addr=0x2b000000, int_num=95)

    def __init__(self, *args, **kwargs):
        super(VExpressFastmodel, self).__init__(*args, **kwargs)

    def _on_chip_devices(self):
        devices = super(VExpressFastmodel, self)._on_chip_devices()
        devices += [ self.gic, self.hdlcd ]
        devices.remove(self.generic_timer)
        return devices

    def setupBootLoader(self, cur_sys, loc, boot_loader=None):
        if boot_loader is None:
            boot_loader = [ loc('boot_v2.arm64') ]
        super(VExpressFastmodel, self).setupBootLoader(
                cur_sys, boot_loader)
<|endoftext|>


<commit_before>import sys
import re
import os

from django.conf import settings
from logging import getLogger
from django.db.utils import IntegrityError

from restclients.models.sws import Person
from restclients.models.canvas import CanvasUser
from restclients.canvas.users import Users as CanvasUsers
from restclients.canvas.accounts import Accounts as CanvasAccounts
from restclients.sws.campus import get_all_campuses
from restclients.sws.college import get_all_colleges
from restclients.sws.department import get_departments_by_college
from restclients.exceptions import DataFailureException

from suds.client import Client
from suds.transport.http import HttpTransport
from suds import WebFault

from astra.models import Admin, Account
from sis_provisioner.models import User
from sis_provisioner.loader import load_user
from sis_provisioner.policy import UserPolicy

import urllib2
import socket
import ssl
import httplib

class ASTRAException(Exception):
    pass

class HTTPSTransportV3(HttpTransport):
    def __init__(self, *args, **kwargs):
        HttpTransport.__init__(self, *args, **kwargs)

    def u2open(self, u2request):
        tm = self.options.timeout
        url = urllib2.build_opener(HTTPSClientAuthHandler())
        if self.u2ver() < 2.6:
            socket.setdefaulttimeout(tm)
            return url.open(u2request)
        else:
            return url.open(u2request, timeout=tm)

class HTTPSClientAuthHandler(urllib2.HTTPSHandler):
    def __init__(self):
        urllib2.HTTPSHandler.__init__(self)

    def https_open(self, req):
        return self.do_open(HTTPSConnectionClientCertV3, req)

    def getConnection(self, host, timeout=300):
        return HTTPSConnectionClientCertV3(host)

class HTTPSConnectionClientCertV3(httplib.HTTPSConnection):
    def __init__(self, *args, **kwargs):
        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
        self.key_file = settings.ASTRA_KEY
        self.cert_file = settings.ASTRA_CERT

    def connect(self):
        sock = socket.create_connection((self.host, self.port), self.timeout)
        if self._tunnel_host:
            self.sock = sock
            self._tunnel()
        try:
            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
                                        ssl_version=ssl.PROTOCOL_TLSv1)
        except ssl.SSLError, e:
            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
                                        ssl_version=ssl.PROTOCOL_SSLv3)

class ASTRA():
    """Load admin table with ASTRA-defined administrators
    """
    def __init__(self, options={}):
        self._astra = Client(settings.ASTRA_WSDL,
                             transport=HTTPSTransportV3())
        # prepare to map spans of control to campus and college resource values
        self._user_policy = UserPolicy()
        self._campuses = get_all_campuses()
        self._colleges = get_all_colleges()
        self._pid = os.getpid()
        self._log = getLogger(__name__)
        self._re_non_academic_code = re.compile(r'^canvas_([0-9]+)$')
        self._canvas_ids = {}
        self._verbosity = int(options.get('verbosity', 0))

    def _request(self, methodName, params={}):
        port = 'AuthzProvider'
        try:
            result = self._astra.service[port][methodName](params)
            return result
        except WebFault, err:
            self._log.error(err)
        except:
            self._log.error('Other error: ' + str(sys.exc_info()[1]))

        return None

    def _getAuthz(self, authFilter):
        return self._request('GetAuthz', authFilter)

    def get_version(self):
        return self._request('GetVersion', {})

    def _add_admin(self, **kwargs):
        netid = kwargs['net_id']
        regid = kwargs['reg_id']
        self._log.info('ADD: %s is %s in %s' % (netid,
                                                kwargs['role'],
                                                kwargs['account_id']))

        try:
            User.objects.get(reg_id=regid)
        except User.DoesNotExist:
            try:
                person = self._user_policy.get_person_by_netid(netid)
                name = person.full_name if (
                    isinstance(person, Person)) else person.display_name

                self._log.info('Provisioning admin: %s, %s, %s)' % (
                    person.uwnetid, person.uwregid, name))

                canvas = CanvasUsers()
                try:
                    user = canvas.get_user_by_sis_id(person.uwregid)
                except DataFailureException:
                    user = canvas.create_user(CanvasUser(name=name,
                                              login_id=person.uwnetid,
                                              sis_user_id=person.uwregid))

                load_user(person)

            except Exception, err:
                self._log.info('Skipped admin: %s (%s)' % (netid, err))
                return

        try:
            admin = Admin.objects.get(net_id=netid,
                                      reg_id=regid,
                                      account_id=kwargs['account_id'],
                                      canvas_id=kwargs['canvas_id'],
                                      role=kwargs['role'])
        except Admin.DoesNotExist:
            admin = Admin(net_id=netid,
                          reg_id=regid,
                          account_id=kwargs['account_id'],
                          canvas_id=kwargs['canvas_id'],
                          role=kwargs['role'],
                          queue_id=self._pid)

        admin.is_deleted = None
        admin.deleted_date = None
        admin.save()

    def _quote_id_level(self, level):
        return re.sub(r'[ :]', '-', level.lower())

    def _get_campus_from_code(self, code):
        for c in self._campuses:
            if c.label.lower() == code.lower():
                return c.label.lower()

        raise ASTRAException('Unknown Campus Code: %s' % code)

    def _get_college_from_code(self, campus, code):
        for college in self._colleges:
            if (campus.lower() == college.campus_label.lower() and
                    code.lower() == college.label.lower()):
                return college

        raise ASTRAException('Unknown College Code: %s' % code)

    def _valid_department_code(self, college, code):
        depts = get_departments_by_college(college)
        for dept in depts:
            if dept.label.lower() == code.lower():
                return code

        raise ASTRAException('Unknown Department Code: %s' % code)

    def _generate_sis_account_id(self, soc):
        id = []
        campus = None
        college = None

        if not isinstance(soc, list):
            raise ASTRAException('NO Span of Control')

        if soc[0]:
            if (soc[0]._type == 'CanvasNonAcademic' or
                    soc[0]._type == 'CanvasTestAccount'):
                try:
                    return (soc[0]._code,
                            self._re_non_academic_code.match(soc[0]._code).group(1))
                except Exception, err:
                    raise ASTRAException('Unknown non-academic code: %s %s' % (
                        soc[0]._code, err))
            elif soc[0]._type == 'SWSCampus':
                campus = self._get_campus_from_code(soc[0]._code)
                id.append(settings.SIS_IMPORT_ROOT_ACCOUNT_ID)
                id.append(self._quote_id_level(campus))
            else:
                raise ASTRAException('Unknown SOC type: %s %s' % (
                    soc[0]._type, soc[0]))

        if len(soc) > 1:
            if soc[1]._type == 'swscollege':
                if campus:
                    college = self._get_college_from_code(campus, soc[1]._code)
                    id.append(self._quote_id_level(college.name))
                else:
                    raise ASTRAException('College without campus: %s' % (
                        soc[1]._code))
            else:
                raise ASTRAException('Unknown second level SOC: %s' % (
                    soc[1]._type))

            if len(soc) > 2:
                if soc[2]._type == 'swsdepartment':
                    if (campus and college and
                            self._valid_department_code(college, soc[2]._code)):
                        id.append(self._quote_id_level(soc[2]._code))
                    else:
                        raise ASTRAException('Unknown third level SOC: %s' % (
                            soc[0]))

        sis_id = ':'.join(id)

        if sis_id not in self._canvas_ids:
            canvas_account = CanvasAccounts().get_account_by_sis_id(sis_id)
            self._canvas_ids[sis_id] = canvas_account.account_id

        return (sis_id, self._canvas_ids[sis_id])

    def load_all_admins(self, options={}):
        # loader running?
        queued = Admin.objects.queued()
        if len(queued):
            # look for pid matching queue_id, adjust gripe accordingly
            try:
                os.kill(queued[0].queue_id, 0)
                raise ASTRAException('loader already running %s' % (
                    queued[0].queue_id))
            except:
                override = options.get('override', 0)
                if override > 0 and override == queued[0].queue_id:
                    Admin.objects.dequeue(queue_id=override)
                    if len(Admin.objects.queued()):
                        raise ASTRAException('unable to override process %s' % (
                            override))
                else:
                    raise ASTRAException('loader blocked by process %s' % (
                        queued[0].queue_id))

        # query ASTRA
        authFilter = self._astra.factory.create('authFilter')
        authFilter.privilege._code = settings.ASTRA_APPLICATION
        authFilter.environment._code = settings.ASTRA_ENVIRONMENT
        authFilter.astraRole._code = 'User'

        authz = self._getAuthz(authFilter)
        if not authz:
            self._log.error('ASTRA GetAuthz failed. Aborting Canvas admin update.')
            return

        # flag and mark all records deleted to catch ASTRA fallen
        Admin.objects.queue_all(queue_id=self._pid)

        # restore records with latest auths
        if 'authCollection' in authz and 'auth' in authz.authCollection:
            for auth in authz.authCollection.auth:
                try:
                    if auth.role._code not in settings.ASTRA_ROLE_MAPPING:
                        raise ASTRAException("Unknown Role Code: %s" % (
                            auth.role._code))
                    elif '_regid' not in auth.party:
                        raise ASTRAException("No regid in party: %s" % (
                            auth.party))

                    if 'spanOfControlCollection' in auth:
                        if ('spanOfControl' in auth.spanOfControlCollection and
                                isinstance(auth.spanOfControlCollection.spanOfControl, list)):
                            (account_id, canvas_id) = self._generate_sis_account_id(auth.spanOfControlCollection.spanOfControl)
                        else:
                            canvas_id = settings.RESTCLIENTS_CANVAS_ACCOUNT_ID
                            account_id = "canvas_%s" % canvas_id

                        self._add_admin(net_id=auth.party._uwNetid,
                                        reg_id=auth.party._regid,
                                        account_id=account_id,
                                        canvas_id=canvas_id,
                                        role=auth.role._code,
                                        is_deleted=None)
                    else:
                        raise ASTRAException("Missing required span of control: %s" % (auth.party))

                except ASTRAException, errstr:
                    self._log.error('%s\n  AUTH: %s' % (errstr, auth))

        # log who fell from ASTRA
        for d in Admin.objects.get_deleted():
            self._log.info('REMOVE: %s as %s in %s' % (
                d.net_id, d.role, d.account_id))

        # tidy up
        Admin.objects.dequeue()

class Accounts():
    """Load account table with Canvas accounts
    """
    def __init__(self):
        self._accounts = CanvasAccounts()
        self._re_sis = re.compile(r'^%s:(%s)(:[^:]+){0,3}$' % (
            settings.SIS_IMPORT_ROOT_ACCOUNT_ID,
            '|'.join([c.label.lower() for c in get_all_campuses()])))
        self._log = getLogger(__name__)

    def load_all_accounts(self):
        root_id = settings.RESTCLIENTS_CANVAS_ACCOUNT_ID
        accounts = [self._accounts.get_account(root_id)]
        accounts.extend(self._accounts.get_all_sub_accounts(root_id))

        Account.objects.all().update(is_deleted=True)

        for account in accounts:
            self.load_account(account)

    def load_account(self, account):
        sis_id = None
        account_type = Account.ADHOC_TYPE
        if account.account_id == int(settings.RESTCLIENTS_CANVAS_ACCOUNT_ID):
            account_type = Account.ROOT_TYPE
        elif account.sis_account_id is not None:
            sis_id = account.sis_account_id
            if self._re_sis.match(account.sis_account_id):
                account_type = Account.SDB_TYPE

        try:
            a = Account.objects.get(canvas_id=account.account_id)
            a.sis_id = sis_id
            a.account_name = account.name
            a.account_type = account_type
        except Account.DoesNotExist:
            a = Account(canvas_id=account.account_id,
                        sis_id=sis_id,
                        account_name=account.name,
                        account_type=account_type)

        a.is_deleted = None
        try:
            a.save()
        except IntegrityError, err:
            self._log.error('ACCOUNT LOAD FAILED: canvas_id = %s, sis_id = %s, name = "%s": %s'
                            % (account.account_id, sis_id, account.name, err))
            raise

        return a
<commit_msg>use sis provisioner dao, adds email to create_user<commit_after>import sys
import re
import os

from django.conf import settings
from logging import getLogger
from django.db.utils import IntegrityError

from restclients.models.canvas import CanvasUser
from restclients.canvas.users import Users as CanvasUsers
from restclients.canvas.accounts import Accounts as CanvasAccounts
from restclients.sws.campus import get_all_campuses
from restclients.sws.college import get_all_colleges
from restclients.sws.department import get_departments_by_college
from restclients.exceptions import DataFailureException

from suds.client import Client
from suds.transport.http import HttpTransport
from suds import WebFault

from astra.models import Admin, Account
from sis_provisioner.models import User
from sis_provisioner.loader import load_user
from sis_provisioner.dao.user import get_person_by_netid, user_fullname,\
    user_email

import urllib2
import socket
import ssl
import httplib

class ASTRAException(Exception):
    pass

class HTTPSTransportV3(HttpTransport):
    def __init__(self, *args, **kwargs):
        HttpTransport.__init__(self, *args, **kwargs)

    def u2open(self, u2request):
        tm = self.options.timeout
        url = urllib2.build_opener(HTTPSClientAuthHandler())
        if self.u2ver() < 2.6:
            socket.setdefaulttimeout(tm)
            return url.open(u2request)
        else:
            return url.open(u2request, timeout=tm)

class HTTPSClientAuthHandler(urllib2.HTTPSHandler):
    def __init__(self):
        urllib2.HTTPSHandler.__init__(self)

    def https_open(self, req):
        return self.do_open(HTTPSConnectionClientCertV3, req)

    def getConnection(self, host, timeout=300):
        return HTTPSConnectionClientCertV3(host)

class HTTPSConnectionClientCertV3(httplib.HTTPSConnection):
    def __init__(self, *args, **kwargs):
        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
        self.key_file = settings.ASTRA_KEY
        self.cert_file = settings.ASTRA_CERT

    def connect(self):
        sock = socket.create_connection((self.host, self.port), self.timeout)
        if self._tunnel_host:
            self.sock = sock
            self._tunnel()
        try:
            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
                                        ssl_version=ssl.PROTOCOL_TLSv1)
        except ssl.SSLError, e:
            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
                                        ssl_version=ssl.PROTOCOL_SSLv3)

class ASTRA():
    """Load admin table with ASTRA-defined administrators
    """
    def __init__(self, options={}):
        self._astra = Client(settings.ASTRA_WSDL,
                             transport=HTTPSTransportV3())
        # prepare to map spans of control to campus and college resource values
        self._campuses = get_all_campuses()
        self._colleges = get_all_colleges()
        self._pid = os.getpid()
        self._log = getLogger(__name__)
        self._re_non_academic_code = re.compile(r'^canvas_([0-9]+)$')
        self._canvas_ids = {}
        self._verbosity = int(options.get('verbosity', 0))

    def _request(self, methodName, params={}):
        port = 'AuthzProvider'
        try:
            result = self._astra.service[port][methodName](params)
            return result
        except WebFault, err:
            self._log.error(err)
        except:
            self._log.error('Other error: ' + str(sys.exc_info()[1]))

        return None

    def _getAuthz(self, authFilter):
        return self._request('GetAuthz', authFilter)

    def get_version(self):
        return self._request('GetVersion', {})

    def _add_admin(self, **kwargs):
        netid = kwargs['net_id']
        regid = kwargs['reg_id']
        self._log.info('ADD: %s is %s in %s' % (netid,
                                                kwargs['role'],
                                                kwargs['account_id']))

        try:
            User.objects.get(reg_id=regid)
        except User.DoesNotExist:
            try:
                person = get_person_by_netid(netid)

                self._log.info('Provisioning admin: %s (%s)' % (
                    person.uwnetid, person.uwregid))

                canvas = CanvasUsers()
                try:
                    user = canvas.get_user_by_sis_id(person.uwregid)
                except DataFailureException:
                    user = canvas.create_user(
                        CanvasUser(name=user_fullname(person),
                                   login_id=person.uwnetid,
                                   sis_user_id=person.uwregid,
                                   email=user_email(person)))

                load_user(person)

            except Exception, err:
                self._log.info('Skipped admin: %s (%s)' % (netid, err))
                return

        try:
            admin = Admin.objects.get(net_id=netid,
                                      reg_id=regid,
                                      account_id=kwargs['account_id'],
                                      canvas_id=kwargs['canvas_id'],
                                      role=kwargs['role'])
        except Admin.DoesNotExist:
            admin = Admin(net_id=netid,
                          reg_id=regid,
                          account_id=kwargs['account_id'],
                          canvas_id=kwargs['canvas_id'],
                          role=kwargs['role'],
                          queue_id=self._pid)

        admin.is_deleted = None
        admin.deleted_date = None
        admin.save()

    def _quote_id_level(self, level):
        return re.sub(r'[ :]', '-', level.lower())

    def _get_campus_from_code(self, code):
        for c in self._campuses:
            if c.label.lower() == code.lower():
                return c.label.lower()

        raise ASTRAException('Unknown Campus Code: %s' % code)

    def _get_college_from_code(self, campus, code):
        for college in self._colleges:
            if (campus.lower() == college.campus_label.lower() and
                    code.lower() == college.label.lower()):
                return college

        raise ASTRAException('Unknown College Code: %s' % code)

    def _valid_department_code(self, college, code):
        depts = get_departments_by_college(college)
        for dept in depts:
            if dept.label.lower() == code.lower():
                return code

        raise ASTRAException('Unknown Department Code: %s' % code)

    def _generate_sis_account_id(self, soc):
        id = []
        campus = None
        college = None

        if not isinstance(soc, list):
            raise ASTRAException('NO Span of Control')

        if soc[0]:
            if (soc[0]._type == 'CanvasNonAcademic' or
                    soc[0]._type == 'CanvasTestAccount'):
                try:
                    return (soc[0]._code,
                            self._re_non_academic_code.match(soc[0]._code).group(1))
                except Exception, err:
                    raise ASTRAException('Unknown non-academic code: %s %s' % (
                        soc[0]._code, err))
            elif soc[0]._type == 'SWSCampus':
                campus = self._get_campus_from_code(soc[0]._code)
                id.append(settings.SIS_IMPORT_ROOT_ACCOUNT_ID)
                id.append(self._quote_id_level(campus))
            else:
                raise ASTRAException('Unknown SOC type: %s %s' % (
                    soc[0]._type, soc[0]))

        if len(soc) > 1:
            if soc[1]._type == 'swscollege':
                if campus:
                    college = self._get_college_from_code(campus, soc[1]._code)
                    id.append(self._quote_id_level(college.name))
                else:
                    raise ASTRAException('College without campus: %s' % (
                        soc[1]._code))
            else:
                raise ASTRAException('Unknown second level SOC: %s' % (
                    soc[1]._type))

            if len(soc) > 2:
                if soc[2]._type == 'swsdepartment':
                    if (campus and college and
                            self._valid_department_code(college, soc[2]._code)):
                        id.append(self._quote_id_level(soc[2]._code))
                    else:
                        raise ASTRAException('Unknown third level SOC: %s' % (
                            soc[0]))

        sis_id = ':'.join(id)

        if sis_id not in self._canvas_ids:
            canvas_account = CanvasAccounts().get_account_by_sis_id(sis_id)
            self._canvas_ids[sis_id] = canvas_account.account_id

        return (sis_id, self._canvas_ids[sis_id])

    def load_all_admins(self, options={}):
        # loader running?
        queued = Admin.objects.queued()
        if len(queued):
            # look for pid matching queue_id, adjust gripe accordingly
            try:
                os.kill(queued[0].queue_id, 0)
                raise ASTRAException('loader already running %s' % (
                    queued[0].queue_id))
            except:
                override = options.get('override', 0)
                if override > 0 and override == queued[0].queue_id:
                    Admin.objects.dequeue(queue_id=override)
                    if len(Admin.objects.queued()):
                        raise ASTRAException('unable to override process %s' % (
                            override))
                else:
                    raise ASTRAException('loader blocked by process %s' % (
                        queued[0].queue_id))

        # query ASTRA
        authFilter = self._astra.factory.create('authFilter')
        authFilter.privilege._code = settings.ASTRA_APPLICATION
        authFilter.environment._code = settings.ASTRA_ENVIRONMENT
        authFilter.astraRole._code = 'User'

        authz = self._getAuthz(authFilter)
        if not authz:
            self._log.error('ASTRA GetAuthz failed. Aborting Canvas admin update.')
            return

        # flag and mark all records deleted to catch ASTRA fallen
        Admin.objects.queue_all(queue_id=self._pid)

        # restore records with latest auths
        if 'authCollection' in authz and 'auth' in authz.authCollection:
            for auth in authz.authCollection.auth:
                try:
                    if auth.role._code not in settings.ASTRA_ROLE_MAPPING:
                        raise ASTRAException("Unknown Role Code: %s" % (
                            auth.role._code))
                    elif '_regid' not in auth.party:
                        raise ASTRAException("No regid in party: %s" % (
                            auth.party))

                    if 'spanOfControlCollection' in auth:
                        if ('spanOfControl' in auth.spanOfControlCollection and
                                isinstance(auth.spanOfControlCollection.spanOfControl, list)):
                            (account_id, canvas_id) = self._generate_sis_account_id(auth.spanOfControlCollection.spanOfControl)
                        else:
                            canvas_id = settings.RESTCLIENTS_CANVAS_ACCOUNT_ID
                            account_id = "canvas_%s" % canvas_id

                        self._add_admin(net_id=auth.party._uwNetid,
                                        reg_id=auth.party._regid,
                                        account_id=account_id,
                                        canvas_id=canvas_id,
                                        role=auth.role._code,
                                        is_deleted=None)
                    else:
                        raise ASTRAException("Missing required span of control: %s" % (auth.party))

                except ASTRAException, errstr:
                    self._log.error('%s\n  AUTH: %s' % (errstr, auth))

        # log who fell from ASTRA
        for d in Admin.objects.get_deleted():
            self._log.info('REMOVE: %s as %s in %s' % (
                d.net_id, d.role, d.account_id))

        # tidy up
        Admin.objects.dequeue()

class Accounts():
    """Load account table with Canvas accounts
    """
    def __init__(self):
        self._accounts = CanvasAccounts()
        self._re_sis = re.compile(r'^%s:(%s)(:[^:]+){0,3}$' % (
            settings.SIS_IMPORT_ROOT_ACCOUNT_ID,
            '|'.join([c.label.lower() for c in get_all_campuses()])))
        self._log = getLogger(__name__)

    def load_all_accounts(self):
        root_id = settings.RESTCLIENTS_CANVAS_ACCOUNT_ID
        accounts = [self._accounts.get_account(root_id)]
        accounts.extend(self._accounts.get_all_sub_accounts(root_id))

        Account.objects.all().update(is_deleted=True)

        for account in accounts:
            self.load_account(account)

    def load_account(self, account):
        sis_id = None
        account_type = Account.ADHOC_TYPE
        if account.account_id == int(settings.RESTCLIENTS_CANVAS_ACCOUNT_ID):
            account_type = Account.ROOT_TYPE
        elif account.sis_account_id is not None:
            sis_id = account.sis_account_id
            if self._re_sis.match(account.sis_account_id):
                account_type = Account.SDB_TYPE

        try:
            a = Account.objects.get(canvas_id=account.account_id)
            a.sis_id = sis_id
            a.account_name = account.name
            a.account_type = account_type
        except Account.DoesNotExist:
            a = Account(canvas_id=account.account_id,
                        sis_id=sis_id,
                        account_name=account.name,
                        account_type=account_type)

        a.is_deleted = None
        try:
            a.save()
        except IntegrityError, err:
            self._log.error('ACCOUNT LOAD FAILED: canvas_id = %s, sis_id = %s, name = "%s": %s'
                            % (account.account_id, sis_id, account.name, err))
            raise

        return a
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys
import os
import argparse
import json
import logging

from penman.__about__ import __version__
from penman.model import Model
from penman import layout
from penman.codec import PENMANCodec
from penman import transform

# Names of functions allowed for ordering triples/branches; we cannot
# resolve them to the actual functions until the model is loaded.  If
# the value of the key is not a method of the model, the value is
# passed as a keyword argument with the value `True`.
REARRANGE_KEYS = {
    'random': 'random_order',
    'canonical': 'canonical_order',
    'alphanumeric': 'alphanumeric_order',
    'inverted-last': 'is_role_inverted',
    'attributes-first': 'attributes_first',
}
RECONFIGURE_KEYS = {
    'original': 'original_order',
    'random': 'random_order',
    'canonical': 'canonical_order',
}

def process(f,
            model,
            out,
            err,
            check,
            normalize_options,
            format_options,
            triples):
    """Read graphs from *f* and write to *out*."""

    exitcode = 0
    codec = PENMANCodec(model=model)
    trees = codec.iterparse(f)

    first = True
    for t in trees:
        if first:
            first = False
        else:
            print(file=out)

        g = _process_in(t, model, normalize_options)
        if check:
            exitcode = _check(g, model)
        if triples:
            s = codec.format_triples(
                g.triples,
                indent=bool(format_options.get('indent', True)))
        else:
            t = _process_out(g, model, normalize_options)
            s = codec.format(t, **format_options)

        print(s, file=out)

    return exitcode

def _process_in(t, model, normalize_options):
    """Encode tree *t* and return the string."""
    # tree transformations
    if normalize_options['canonicalize_roles']:
        t = transform.canonicalize_roles(t, model)

    g = layout.interpret(t, model)

    # graph transformations
    if normalize_options['reify_edges']:
        g = transform.reify_edges(g, model)
    if normalize_options['dereify_edges']:
        g = transform.dereify_edges(g, model)
    if normalize_options['reify_attributes']:
        g = transform.reify_attributes(g)
    if normalize_options['indicate_branches']:
        g = transform.indicate_branches(g, model)

    return g

def _process_out(g, model, normalize_options):
    if normalize_options['reconfigure']:
        key, kwargs = normalize_options['reconfigure']
        t = layout.reconfigure(g, key=key, **kwargs)
        g = layout.interpret(t, model)
    else:
        t = layout.configure(g, model=model)
    if normalize_options['rearrange']:
        key, kwargs = normalize_options['rearrange']
        layout.rearrange(t, key=key, **kwargs)
    if normalize_options['make_variables']:
        t.reset_variables(normalize_options['make_variables'])

    return t

def _check(g, model):
    i = 1
    errors = model.errors(g)
    if errors:
        for triple, errors in errors.items():
            if triple:
                context = '({}) '.format(' '.join(map(str, triple)))
            else:
                context = ''
            for error in errors:
                g.metadata[f'error-{i}'] = context + error
            i += 1
        return 1
    else:
        return 0

def _order_funcs(KEY_FUNCS):

    def split_arg(arg):
        values = arg.split(',')
        for value in values:
            if value not in KEY_FUNCS:
                raise argparse.ArgumentTypeError(
                    'invalid choice: {!r} (choose from {})'
                    .format(value, ', '.join(map(repr, KEY_FUNCS))))
        return values

    return split_arg

def _make_sort_key(keys, model, KEY_FUNCS):
    kwargs = {}
    funcs = []
    for key in keys:
        name = KEY_FUNCS[key]
        func = getattr(model, name, None)
        if func is None:
            kwargs[name] = True
        else:
            funcs.append(func)

    def sort_key(role, funcs=funcs):
        return [func(role) for func in funcs]

    return sort_key, kwargs

def main():
    parser = argparse.ArgumentParser(
        prog='penman',
        description='Read and write graphs in the PENMAN notation.',
    )
    parser.add_argument(
        '-V', '--version', action='version',
        version='Penman v{}'.format(__version__))
    parser.add_argument(
        '-v', '--verbose', action='count', dest='verbosity', default=0,
        help='increase verbosity')
    parser.add_argument(
        '-q', '--quiet', action='store_true',
        help='suppress output on <stdout> and <stderr>')
    parser.add_argument(
        'FILE', nargs='*',
        help='read graphs from FILEs instead of stdin')
    model_group = parser.add_mutually_exclusive_group()
    model_group.add_argument(
        '--model', metavar='FILE',
        type=argparse.FileType('r'),
        help='JSON model file describing the semantic model')
    model_group.add_argument(
        '--amr', action='store_true',
        help='use the AMR model')
    parser.add_argument(
        '--check', action='store_true',
        help='check graphs for compliance with the model')
    form = parser.add_argument_group('formatting options')
    form.add_argument(
        '--indent', metavar='N',
        help='indent N spaces per level ("no" for no newlines)')
    form.add_argument(
        '--compact', action='store_true',
        help='compactly print node attributes on one line')
    form.add_argument(
        '--triples', action='store_true',
        help='print graphs as triple conjunctions')
    norm = parser.add_argument_group('normalization options')
    norm.add_argument(
        '--make-variables', metavar='FMT',
        help="recreate node variables with FMT (e.g.: '{prefix}{j}')")
    norm.add_argument(
        '--rearrange', metavar='KEY',
        type=_order_funcs(REARRANGE_KEYS),
        help='reorder the branches of the tree')
    norm.add_argument(
        '--reconfigure', metavar='KEY',
        type=_order_funcs(RECONFIGURE_KEYS),
        help='reconfigure the graph layout with reordered triples')
    norm.add_argument(
        '--canonicalize-roles', action='store_true',
        help='canonicalize role forms')
    norm.add_argument(
        '--reify-edges', action='store_true',
        help='reify all eligible edges')
    norm.add_argument(
        '--dereify-edges', action='store_true',
        help='dereify all eligible edges')
    norm.add_argument(
        '--reify-attributes', action='store_true',
        help='reify all attributes')
    norm.add_argument(
        '--indicate-branches', action='store_true',
        help='insert triples to indicate tree structure')

    args = parser.parse_args()

    if args.quiet:
        args.verbosity = 0
        sys.stdout.close()
        sys.stdout = open(os.devnull, 'w')
    else:
        args.verbosity = min(args.verbosity, 3)

    logger = logging.getLogger('penman')
    logger.setLevel(logging.ERROR - (args.verbosity * 10))

    if args.amr:
        from penman.models.amr import model
    elif args.model:
        model = Model(**json.load(args.model))
    else:
        model = Model()

    if args.rearrange:
        args.rearrange = _make_sort_key(
            args.rearrange, model, REARRANGE_KEYS)

    if args.reconfigure:
        args.reconfigure = _make_sort_key(
            args.reconfigure, model, RECONFIGURE_KEYS)

    indent = -1
    if args.indent:
        if args.indent.lower() in ("no", "none", "false"):
            indent = None
        else:
            try:
                indent = int(args.indent)
                if indent < -1:
                    raise ValueError
            except ValueError:
                sys.exit('error: --indent value must be "no" or an '
                         'integer >= -1')

    normalize_options = {
        'make_variables': args.make_variables,
        'rearrange': args.rearrange,
        'reconfigure': args.reconfigure,
        'canonicalize_roles': args.canonicalize_roles,
        'reify_edges': args.reify_edges,
        'dereify_edges': args.dereify_edges,
        'reify_attributes': args.reify_attributes,
        'indicate_branches': args.indicate_branches,
    }
    format_options = {
        'indent': indent,
        'compact': args.compact,
    }

    if args.FILE:
        for file in args.FILE:
            with open(file) as f:
                exitcode = process(
                    f, model, sys.stdout, sys.stderr, args.check,
                    normalize_options, format_options, args.triples)
    else:
        exitcode = process(
            sys.stdin, model, sys.stdout, sys.stderr, args.check,
            normalize_options, format_options, args.triples)

    sys.exit(exitcode)

if __name__ == '__main__':
    main()
<commit_msg>Flake8 complexity fix<commit_after># -*- coding: utf-8 -*-

import sys
import os
import argparse
import json
import logging

from penman.__about__ import __version__
from penman.model import Model
from penman import layout
from penman.codec import PENMANCodec
from penman import transform

# Names of functions allowed for ordering triples/branches; we cannot
# resolve them to the actual functions until the model is loaded.  If
# the value of the key is not a method of the model, the value is
# passed as a keyword argument with the value `True`.
REARRANGE_KEYS = {
    'random': 'random_order',
    'canonical': 'canonical_order',
    'alphanumeric': 'alphanumeric_order',
    'inverted-last': 'is_role_inverted',
    'attributes-first': 'attributes_first',
}
RECONFIGURE_KEYS = {
    'original': 'original_order',
    'random': 'random_order',
    'canonical': 'canonical_order',
}

def process(f,
            model,
            out,
            err,
            check,
            normalize_options,
            format_options,
            triples):
    """Read graphs from *f* and write to *out*."""

    exitcode = 0
    codec = PENMANCodec(model=model)
    trees = codec.iterparse(f)

    first = True
    for t in trees:
        if first:
            first = False
        else:
            print(file=out)

        g = _process_in(t, model, normalize_options)
        if check:
            exitcode = _check(g, model)
        if triples:
            s = codec.format_triples(
                g.triples,
                indent=bool(format_options.get('indent', True)))
        else:
            t = _process_out(g, model, normalize_options)
            s = codec.format(t, **format_options)

        print(s, file=out)

    return exitcode

def _process_in(t, model, normalize_options):
    """Encode tree *t* and return the string."""
    # tree transformations
    if normalize_options['canonicalize_roles']:
        t = transform.canonicalize_roles(t, model)

    g = layout.interpret(t, model)

    # graph transformations
    if normalize_options['reify_edges']:
        g = transform.reify_edges(g, model)
    if normalize_options['dereify_edges']:
        g = transform.dereify_edges(g, model)
    if normalize_options['reify_attributes']:
        g = transform.reify_attributes(g)
    if normalize_options['indicate_branches']:
        g = transform.indicate_branches(g, model)

    return g

def _process_out(g, model, normalize_options):
    if normalize_options['reconfigure']:
        key, kwargs = normalize_options['reconfigure']
        t = layout.reconfigure(g, key=key, **kwargs)
        g = layout.interpret(t, model)
    else:
        t = layout.configure(g, model=model)
    if normalize_options['rearrange']:
        key, kwargs = normalize_options['rearrange']
        layout.rearrange(t, key=key, **kwargs)
    if normalize_options['make_variables']:
        t.reset_variables(normalize_options['make_variables'])

    return t

def _check(g, model):
    i = 1
    errors = model.errors(g)
    if errors:
        for triple, errors in errors.items():
            if triple:
                context = '({}) '.format(' '.join(map(str, triple)))
            else:
                context = ''
            for error in errors:
                g.metadata[f'error-{i}'] = context + error
            i += 1
        return 1
    else:
        return 0

def _order_funcs(KEY_FUNCS):

    def split_arg(arg):
        values = arg.split(',')
        for value in values:
            if value not in KEY_FUNCS:
                raise argparse.ArgumentTypeError(
                    'invalid choice: {!r} (choose from {})'
                    .format(value, ', '.join(map(repr, KEY_FUNCS))))
        return values

    return split_arg

def _make_sort_key(keys, model, KEY_FUNCS):
    kwargs = {}
    funcs = []
    for key in keys:
        name = KEY_FUNCS[key]
        func = getattr(model, name, None)
        if func is None:
            kwargs[name] = True
        else:
            funcs.append(func)

    def sort_key(role, funcs=funcs):
        return [func(role) for func in funcs]

    return sort_key, kwargs

def main():
    parser = argparse.ArgumentParser(
        prog='penman',
        description='Read and write graphs in the PENMAN notation.',
    )
    parser.add_argument(
        '-V', '--version', action='version',
        version='Penman v{}'.format(__version__))
    parser.add_argument(
        '-v', '--verbose', action='count', dest='verbosity', default=0,
        help='increase verbosity')
    parser.add_argument(
        '-q', '--quiet', action='store_true',
        help='suppress output on <stdout> and <stderr>')
    parser.add_argument(
        'FILE', nargs='*',
        help='read graphs from FILEs instead of stdin')
    model_group = parser.add_mutually_exclusive_group()
    model_group.add_argument(
        '--model', metavar='FILE',
        type=argparse.FileType('r'),
        help='JSON model file describing the semantic model')
    model_group.add_argument(
        '--amr', action='store_true',
        help='use the AMR model')
    parser.add_argument(
        '--check', action='store_true',
        help='check graphs for compliance with the model')
    form = parser.add_argument_group('formatting options')
    form.add_argument(
        '--indent', metavar='N',
        help='indent N spaces per level ("no" for no newlines)')
    form.add_argument(
        '--compact', action='store_true',
        help='compactly print node attributes on one line')
    form.add_argument(
        '--triples', action='store_true',
        help='print graphs as triple conjunctions')
    norm = parser.add_argument_group('normalization options')
    norm.add_argument(
        '--make-variables', metavar='FMT',
        help="recreate node variables with FMT (e.g.: '{prefix}{j}')")
    norm.add_argument(
        '--rearrange', metavar='KEY',
        type=_order_funcs(REARRANGE_KEYS),
        help='reorder the branches of the tree')
    norm.add_argument(
        '--reconfigure', metavar='KEY',
        type=_order_funcs(RECONFIGURE_KEYS),
        help='reconfigure the graph layout with reordered triples')
    norm.add_argument(
        '--canonicalize-roles', action='store_true',
        help='canonicalize role forms')
    norm.add_argument(
        '--reify-edges', action='store_true',
        help='reify all eligible edges')
    norm.add_argument(
        '--dereify-edges', action='store_true',
        help='dereify all eligible edges')
    norm.add_argument(
        '--reify-attributes', action='store_true',
        help='reify all attributes')
    norm.add_argument(
        '--indicate-branches', action='store_true',
        help='insert triples to indicate tree structure')

    args = parser.parse_args()

    if args.quiet:
        args.verbosity = 0
        sys.stdout.close()
        sys.stdout = open(os.devnull, 'w')
    else:
        args.verbosity = min(args.verbosity, 3)

    logger = logging.getLogger('penman')
    logger.setLevel(logging.ERROR - (args.verbosity * 10))

    model = _get_model(args.amr, args.model)

    if args.rearrange:
        args.rearrange = _make_sort_key(
            args.rearrange, model, REARRANGE_KEYS)

    if args.reconfigure:
        args.reconfigure = _make_sort_key(
            args.reconfigure, model, RECONFIGURE_KEYS)

    indent = _indent(args.indent)

    normalize_options = {
        'make_variables': args.make_variables,
        'rearrange': args.rearrange,
        'reconfigure': args.reconfigure,
        'canonicalize_roles': args.canonicalize_roles,
        'reify_edges': args.reify_edges,
        'dereify_edges': args.dereify_edges,
        'reify_attributes': args.reify_attributes,
        'indicate_branches': args.indicate_branches,
    }
    format_options = {
        'indent': indent,
        'compact': args.compact,
    }

    if args.FILE:
        for file in args.FILE:
            with open(file) as f:
                exitcode = process(
                    f, model, sys.stdout, sys.stderr, args.check,
                    normalize_options, format_options, args.triples)
    else:
        exitcode = process(
            sys.stdin, model, sys.stdout, sys.stderr, args.check,
            normalize_options, format_options, args.triples)

    sys.exit(exitcode)

def _get_model(amr, model_file):
    if amr:
        from penman.models.amr import model
    elif model_file:
        model = Model(**json.load(model_file))
    else:
        model = Model()
    return model

def _indent(indent):
    if indent:
        if indent.lower() in ("no", "none", "false"):
            indent = None
        else:
            try:
                indent = int(indent)
                if indent < -1:
                    raise ValueError
            except ValueError:
                sys.exit('error: --indent value must be "no" or an '
                         'integer >= -1')
    else:
        indent = -1
    return indent

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Authors: Fred Buchanan
License: Apache 2.0
Description: Coconut test suite
"""

from __future__ import print_function, absolute_import, unicode_literals, division

from .intagration_tests import OldCocoTest<commit_msg>Fixed importing in tests<commit_after>#!/usr/bin/env python
"""
Authors: Fred Buchanan
License: Apache 2.0
Description: Coconut test suite
"""

from __future__ import print_function, absolute_import, unicode_literals, division

from tests.intagration_tests import OldCocoTest<|endoftext|>


<commit_before>import random
from ..actions import Buff, Give, Summon
from ..enums import CardClass, CardType
from ..game import Game
from ..cards.utils import RandomMinion
from ..dsl.picker import RandomCardPicker

class BlackrockShowdownBrawl(Game):
	"""
	Showdown at Blackrock Mountain

	These two epic bosses of Blackrock Mountain are
	settling things once and for all... with your help!
	"""

	NEFARIAN_DECK = ([
		"BRMA14_3",
		"BRMA14_5",
		"BRMA14_7",
		"BRMA14_9",
		"BRMA10_5H", "BRMA10_5H", "BRMA10_5H",
		"BRMC_83", "BRMC_83",
		"BRMC_84", "BRMC_84",
		"BRMC_86",
		"BRMC_88", "BRMC_88",
		"BRMC_97",
		"BRMC_98",
		"BRM_018", "BRM_018",
		"BRM_029",
		"BRM_031",
		"BRM_033", "BRM_033", "BRM_033",
		"BRM_034", "BRM_034",
		"EX1_303", "EX1_303",
		"EX1_562",
		"EX1_570", "EX1_570",
	], "TBA01_4")

	RAGNAROS_DECK = ([
		"BRMA_01", "BRMA_01",
		"BRMC_85",
		"BRMC_87",
		"BRMC_89", "BRMC_89",
		"BRMC_90", "BRMC_90", "BRMC_90",
		"BRMC_91", "BRMC_91",
		"BRMC_92",
		"BRMC_95",
		"BRMC_95h", "BRMC_95h",
		"BRMC_96",
		"BRMC_99",
		"BRMC_100", "BRMC_100",
		"CS2_032", "CS2_032",
		"CS2_042", "CS2_042",
		"EX1_241", "EX1_241",
		"EX1_249",
		"EX1_319", "EX1_319",
		"EX1_620", "EX1_620",
	], "TBA01_1")

	@classmethod
	def new_game(cls, *players):
		decks = random.sample((cls.NEFARIAN_DECK, cls.RAGNAROS_DECK), 2)
		for player, deck in zip(players, decks):
			player.prepare_deck(deck[0], hero=deck[1])
		return cls(players)

	def prepare(self):
		super().prepare()
		for player in self.players:
			if player.hero.id == self.NEFARIAN_DECK[1]:
				player.max_mana = 4
				player.hero.armor = 30
			else:
				player.summon("BRMC_94")  # Sulfuras

class BananaBrawl(Game):
	"""
	Banana Brawl!

	Mukla is a year older, and he LOVES bananas! Whenever
	one of your minions dies, he gives you a Banana to
	celebrate!
	"""

	class RandomBanana(RandomCardPicker):
		cards = ("EX1_014t", "TB_006", "TB_007", "TB_008")

	def _schedule_death(self, card):
		ret = super()._schedule_death(card)
		if card.type == CardType.MINION:
			ret.append(Give(card.controller, self.RandomBanana()))
		return ret

class SpidersEverywhereBrawl(Game):
	"""
	Spiders, Spiders EVERYWHERE!

	Spiders have overrun everything, including your deck!
	Whatever class you play, your deck will be TEEMING with
	Webspinners.
	"""

	def __init__(self, players):
		from .. import cards
		super().__init__(players)
		for player in players:
			hero = player.starting_hero
			player_class = getattr(cards, hero).card_class
			spells = cards.filter(card_class=player_class, type=CardType.SPELL)
			deck = ["FP1_011"] * 23
			for i in range(7):
				deck.append(random.choice(spells))
			player.prepare_deck(deck, hero)

class GreatSummonerBrawl(Game):
	"""
	The Great Summoner Competition

	Summoners from across the world have come to compete. When
	you cast a spell, a random minion of the same cost is summoned
	for you!
	"""

	def _play(self, card, *args):
		if card.type == CardType.SPELL:
			action = Summon(card.controller, RandomMinion(cost=card.cost))
			self.queue_actions(card.controller, [action])
		return super()._play(card, *args)

class CrossroadsEncounterBrawl(Game):
	"""
	Encounter at the Crossroads

	Encounter at the crossroads! Pick a class.
	Let's see what's in your deck this time!
	"""

	def __init__(self, players):
		from .. import cards
		super().__init__(players)
		for player in players:
			hero = player.starting_hero
			player_class = getattr(cards, hero).card_class
			pool = cards.filter(card_class=player_class, collectible=True)
			deck = [random.choice(pool) for i in range(15)]
			pool = cards.filter(card_class=CardClass.INVALID, collectible=True)
			deck += [random.choice(pool) for i in range(15)]
			player.prepare_deck(deck, hero)

class HeartOfTheSunwellBrawl(Game):
	"""
	Heart of the Sunwell

	In the Sunwell lies unlimited power, and that power is
	yours! Start each game with 10 mana and see what you
	can do with it!
	"""

	def prepare(self):
		super().prepare()
		for player in self.players:
			player.max_mana = 10

class TooManyPortalsBrawl(Game):
	"""
	Too Many Portals!

	The master mages of Dalaran have gone too far this time,
	opening up hundreds of portals!  Choose a class and use a
	few spells and a WHOLE lot of portals to defeat your rivals!
	"""
	UNSTABLE_PORTAL = "GVG_003"

	def __init__(self, players):
		from .. import cards
		super().__init__(players)
		for player in players:
			hero = player.starting_hero
			player_class = getattr(cards, hero).card_class
			spells = cards.filter(card_class=player_class, type=CardType.SPELL)
			deck = [self.UNSTABLE_PORTAL] * 23
			for i in range(7):
				deck.append(random.choice(spells))
			player.prepare_deck(deck, hero)

class MaskedBallBrawl(Game):
	"""
	The Masked Ball

	At the SI:7 mansion in Stormwind they have a grand masked ball every year.
	Everyone is in disguise! When a minion dies, its disguise is revealed, showing the
	minion to actually be a different random minion that costs (2) less and ready for
	another fight!
	"""

	def _play(self, card, *args):
		if card.type == CardType.MINION and card.cost >= 2:
			action = Buff(card, "TB_Pilot1")
			self.queue_actions(card, [action])
		return super()._play(card, *args)

class GrandTournamentBrawl(Game):
	"""
	Grand Tournament Match

	On the road to The Grand Tournament, two knights face off!
	One relies on jousting skill, while the other inspires troops to
	great deeds. Who will win?
	"""

	ALLERIA_DECK = ([
		"AT_010",
		"AT_058", "AT_058",
		"AT_060",
		"AT_061", "AT_061",
		"AT_063",
		"AT_102", "AT_102",
		"AT_103"
		"AT_108", "AT_108",
		"AT_111",
		"AT_112", "AT_112",
		"AT_128",
		"CS2_084", "CS2_084",
		"DS1_178",
		"DS1_184", "DS1_184",
		"DS1_185", "DS1_185",
		"EX1_538", "EX1_538",
		"EX1_609",
		"EX1_611",
		"GVG_073",
		"NEW1_031", "NEW1_031",
	], "HERO_05a")

	MEDIVH_DECK = ([
		"AT_001", "AT_001",
		"AT_003", "AT_003",
		"AT_005", "AT_005",
		"AT_007", "AT_007",
		"AT_008", "AT_008",
		"AT_009",
		"AT_080", "AT_080",
		"AT_082", "AT_082",
		"AT_083",
		"AT_085", "AT_085",
		"AT_087",
		"AT_089", "AT_089",
		"AT_097",
		"AT_099",
		"AT_114", "AT_114",
		"AT_119",
		"AT_127",
		"AT_120", "AT_120",
		"CS2_028",
	], "HERO_08a")

	@classmethod
	def new_game(cls, *players):
		decks = random.sample((cls.ALLERIA_DECK, cls.MEDIVH_DECK), 2)
		for player, deck in zip(players, decks):
			player.prepare_deck(deck[0], hero=deck[1])
		return cls(players)
<commit_msg>Implement It's Raining Mana! tavern brawl<commit_after>import random
from ..actions import Buff, Give, Summon
from ..enums import CardClass, CardType
from ..game import Game
from ..cards.utils import RandomMinion
from ..dsl.picker import RandomCardPicker

class BlackrockShowdownBrawl(Game):
	"""
	Showdown at Blackrock Mountain

	These two epic bosses of Blackrock Mountain are
	settling things once and for all... with your help!
	"""

	NEFARIAN_DECK = ([
		"BRMA14_3",
		"BRMA14_5",
		"BRMA14_7",
		"BRMA14_9",
		"BRMA10_5H", "BRMA10_5H", "BRMA10_5H",
		"BRMC_83", "BRMC_83",
		"BRMC_84", "BRMC_84",
		"BRMC_86",
		"BRMC_88", "BRMC_88",
		"BRMC_97",
		"BRMC_98",
		"BRM_018", "BRM_018",
		"BRM_029",
		"BRM_031",
		"BRM_033", "BRM_033", "BRM_033",
		"BRM_034", "BRM_034",
		"EX1_303", "EX1_303",
		"EX1_562",
		"EX1_570", "EX1_570",
	], "TBA01_4")

	RAGNAROS_DECK = ([
		"BRMA_01", "BRMA_01",
		"BRMC_85",
		"BRMC_87",
		"BRMC_89", "BRMC_89",
		"BRMC_90", "BRMC_90", "BRMC_90",
		"BRMC_91", "BRMC_91",
		"BRMC_92",
		"BRMC_95",
		"BRMC_95h", "BRMC_95h",
		"BRMC_96",
		"BRMC_99",
		"BRMC_100", "BRMC_100",
		"CS2_032", "CS2_032",
		"CS2_042", "CS2_042",
		"EX1_241", "EX1_241",
		"EX1_249",
		"EX1_319", "EX1_319",
		"EX1_620", "EX1_620",
	], "TBA01_1")

	@classmethod
	def new_game(cls, *players):
		decks = random.sample((cls.NEFARIAN_DECK, cls.RAGNAROS_DECK), 2)
		for player, deck in zip(players, decks):
			player.prepare_deck(deck[0], hero=deck[1])
		return cls(players)

	def prepare(self):
		super().prepare()
		for player in self.players:
			if player.hero.id == self.NEFARIAN_DECK[1]:
				player.max_mana = 4
				player.hero.armor = 30
			else:
				player.summon("BRMC_94")  # Sulfuras

class BananaBrawl(Game):
	"""
	Banana Brawl!

	Mukla is a year older, and he LOVES bananas! Whenever
	one of your minions dies, he gives you a Banana to
	celebrate!
	"""

	class RandomBanana(RandomCardPicker):
		cards = ("EX1_014t", "TB_006", "TB_007", "TB_008")

	def _schedule_death(self, card):
		ret = super()._schedule_death(card)
		if card.type == CardType.MINION:
			ret.append(Give(card.controller, self.RandomBanana()))
		return ret

class SpidersEverywhereBrawl(Game):
	"""
	Spiders, Spiders EVERYWHERE!

	Spiders have overrun everything, including your deck!
	Whatever class you play, your deck will be TEEMING with
	Webspinners.
	"""

	def __init__(self, players):
		from .. import cards
		super().__init__(players)
		for player in players:
			hero = player.starting_hero
			player_class = getattr(cards, hero).card_class
			spells = cards.filter(card_class=player_class, type=CardType.SPELL)
			deck = ["FP1_011"] * 23
			for i in range(7):
				deck.append(random.choice(spells))
			player.prepare_deck(deck, hero)

class GreatSummonerBrawl(Game):
	"""
	The Great Summoner Competition

	Summoners from across the world have come to compete. When
	you cast a spell, a random minion of the same cost is summoned
	for you!
	"""

	def _play(self, card, *args):
		if card.type == CardType.SPELL:
			action = Summon(card.controller, RandomMinion(cost=card.cost))
			self.queue_actions(card.controller, [action])
		return super()._play(card, *args)

class CrossroadsEncounterBrawl(Game):
	"""
	Encounter at the Crossroads

	Encounter at the crossroads! Pick a class.
	Let's see what's in your deck this time!
	"""

	def __init__(self, players):
		from .. import cards
		super().__init__(players)
		for player in players:
			hero = player.starting_hero
			player_class = getattr(cards, hero).card_class
			pool = cards.filter(card_class=player_class, collectible=True)
			deck = [random.choice(pool) for i in range(15)]
			pool = cards.filter(card_class=CardClass.INVALID, collectible=True)
			deck += [random.choice(pool) for i in range(15)]
			player.prepare_deck(deck, hero)

class HeartOfTheSunwellBrawl(Game):
	"""
	Heart of the Sunwell

	In the Sunwell lies unlimited power, and that power is
	yours! Start each game with 10 mana and see what you
	can do with it!
	"""

	def prepare(self):
		super().prepare()
		for player in self.players:
			player.max_mana = 10

class TooManyPortalsBrawl(Game):
	"""
	Too Many Portals!

	The master mages of Dalaran have gone too far this time,
	opening up hundreds of portals!  Choose a class and use a
	few spells and a WHOLE lot of portals to defeat your rivals!
	"""
	UNSTABLE_PORTAL = "GVG_003"

	def __init__(self, players):
		from .. import cards
		super().__init__(players)
		for player in players:
			hero = player.starting_hero
			player_class = getattr(cards, hero).card_class
			spells = cards.filter(card_class=player_class, type=CardType.SPELL)
			deck = [self.UNSTABLE_PORTAL] * 23
			for i in range(7):
				deck.append(random.choice(spells))
			player.prepare_deck(deck, hero)

class MaskedBallBrawl(Game):
	"""
	The Masked Ball

	At the SI:7 mansion in Stormwind they have a grand masked ball every year.
	Everyone is in disguise! When a minion dies, its disguise is revealed, showing the
	minion to actually be a different random minion that costs (2) less and ready for
	another fight!
	"""

	def _play(self, card, *args):
		if card.type == CardType.MINION and card.cost >= 2:
			action = Buff(card, "TB_Pilot1")
			self.queue_actions(card, [action])
		return super()._play(card, *args)

class GrandTournamentBrawl(Game):
	"""
	Grand Tournament Match

	On the road to The Grand Tournament, two knights face off!
	One relies on jousting skill, while the other inspires troops to
	great deeds. Who will win?
	"""

	ALLERIA_DECK = ([
		"AT_010",
		"AT_058", "AT_058",
		"AT_060",
		"AT_061", "AT_061",
		"AT_063",
		"AT_102", "AT_102",
		"AT_103"
		"AT_108", "AT_108",
		"AT_111",
		"AT_112", "AT_112",
		"AT_128",
		"CS2_084", "CS2_084",
		"DS1_178",
		"DS1_184", "DS1_184",
		"DS1_185", "DS1_185",
		"EX1_538", "EX1_538",
		"EX1_609",
		"EX1_611",
		"GVG_073",
		"NEW1_031", "NEW1_031",
	], "HERO_05a")

	MEDIVH_DECK = ([
		"AT_001", "AT_001",
		"AT_003", "AT_003",
		"AT_005", "AT_005",
		"AT_007", "AT_007",
		"AT_008", "AT_008",
		"AT_009",
		"AT_080", "AT_080",
		"AT_082", "AT_082",
		"AT_083",
		"AT_085", "AT_085",
		"AT_087",
		"AT_089", "AT_089",
		"AT_097",
		"AT_099",
		"AT_114", "AT_114",
		"AT_119",
		"AT_127",
		"AT_120", "AT_120",
		"CS2_028",
	], "HERO_08a")

	@classmethod
	def new_game(cls, *players):
		decks = random.sample((cls.ALLERIA_DECK, cls.MEDIVH_DECK), 2)
		for player, deck in zip(players, decks):
			player.prepare_deck(deck[0], hero=deck[1])
		return cls(players)

class RainingManaBrawl(Game):
	"""
	It's Raining Mana!

	Dalaran floats overhead, it's soaking your deck with mana! On
	your first turn you get 1 like normal, but then you get 3, then
	5, then 7, etc
	"""
	def begin_turn(self, player):
		super().begin_turn(player)
		if self.turn > 2:
			player.max_mana += 1
<|endoftext|>