<commit_before>#!usr/bin/env python
""" UpdateDevices.py

This program controls several other Python scripts that each update the inventory
information for a network device. Currently supported devices are access points,
switches, APC or Liebert UPSes, and phones using VOIP.

Uses a configuration file for information about device logins, OIDs, IP addresses,
and so on.

Usage: UpdateDevices.py [-h] [-d] [-a] [-s] [-u] [-p]

Optional arguments:
  -h, --help         show a help message and exit
  -d, --debug        enable logging on update failure
  -a, --accesspoint  update access points
  -e, --apstatus     update access point status and send notification email
  -s, --switch       update switches
  -u, --ups          update UPSes
  -p, --phone        update phones

Running with no arguments will attempt to update all devices.
"""

import sys
import os
import re
import argparse
import logging
import datetime

path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
configPath = os.path.join(path, "config")
logPath = os.path.join(path, "log")
logging.basicConfig(filename=logPath)
debug = False;

def updateAP():
    """ This function reads necessary information from a config file, then
    calls the updateAccessPoints function in apInv.py.
    """
    try:
        from ap import apInv
        print "Updating access points..."

        with open(configPath) as f:
            config = f.read()
            ap_mac = re.findall(r'AP_MAC_OID = (.+?)\n', config)[0]
            ap_name = re.findall(r'AP_Name_OID = (.+?)\n', config)[0]
            ap_ip = re.findall(r'AP_IP_OID = (.+?)\n', config)[0]
            ap_serial = re.findall(r'AP_Serial_OID = (.+?)\n', config)[0]
            ap_model = re.findall(r'AP_Model_OID = (.+?)\n', config)[0]
            ap_status = re.findall(r'AP_Status_OID = (.+?)\n', config)[0]
            ap_oids = [ap_mac, ap_name, ap_ip, ap_serial, ap_model, ap_status]
            ap_controllers = re.findall(r'AP_ControllerIPs = \[(.+?)\]', config)[0].split(",")
        apInv.updateAccessPoints(path, ap_oids, ap_controllers)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update access points:\n\n"
            logging.exception(msg)
            print "Could not update access points. See", logPath, "for details."
        else:
            print "Could not update access points."

def updateAPStatus():
    """ This function reads necessary information from a config file, then
    calls the updateStatus function in apInv.py.
    """
    try:
        from ap import apInv
        print "Updating access point status..."

        with open(configPath) as f:
            config = f.read()
            ap_status = re.findall(r'AP_Status_OID = (.+?)\n', config)[0]
            ap_controllers = re.findall(r'AP_ControllerIPs = \[(.+?)\]', config)[0].split(",")
            ap_emailFrom = re.findall(r'AP_Email_From = (.+?)\n', config)[0]
            ap_emailTo = re.findall(r'AP_Email_To = (.+?)\n', config)
            ap_emailServer = re.findall(r'AP_Email_Server = (.+?)\n', config)[0]
            ap_email = {"from":ap_emailFrom, "to":ap_emailTo, "server":ap_emailServer}
        apInv.updateStatus(ap_controllers, ap_status, ap_email)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update access points:\n\n"
            logging.exception(msg)
            print "Could not update access point status. See", logPath, "for details."
        else:
            print "Could not update access point status."

def updateSwitch(switch_IPs = []):
    """ This function reads necessary information from a config file, then
    calls the updateSwitches function in switchInv.py.
    """
    try:
        from switch import switchInv
        print "Updating switches..."

        with open(configPath) as f:
            config = f.read()
            switch_login = re.findall(r'Switch_Login = \[(.+?)\]', config)[0].split(",")
            if len(switch_IPs) == 0:
                switch_IPs = []
                try:
                    switch_IPs = re.findall(r'Switch_IPs = \[(.+?)\]', config)[0].split(",")
                except:
                    pass
        switchInv.updateSwitches(switch_login, switch_IPs)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update switches:\n\n"
            logging.exception(msg)
            print "Could not update switches. See", logPath, "for details."
        else:
            print "Could not update switches."

def updateUPS():
    """ This function reads necessary information from a config file, then
    calls the updateUPS function in upsInv.py.
    """
    try:
        from ups import upsInv
        print "Updating UPSes..."

        with open(configPath) as f:
            config = f.read()
            apc_serial = re.findall(r'APC_Serial_OID = (.+?)\n', config)[0]
            apc_model = re.findall(r'APC_Model_OID = (.+?)\n', config)[0]
            apc_mac = re.findall(r'APC_MAC_OID = (.+?)\n', config)[0]
            apc_name = re.findall(r'APC_Name_OID = (.+?)\n', config)[0]
            apc_oids = [apc_serial, apc_model, apc_mac, apc_name]
            lie_serial = re.findall(r'Liebert_Serial_OID = (.+?)\n', config)[0]
            lie_model = re.findall(r'Liebert_Model_OID = (.+?)\n', config)[0]
            lie_mac = re.findall(r'Liebert_MAC_OID = (.+?)\n', config)[0]
            lie_mfdate = re.findall(r'Liebert_MfDate_OID = (.+?)\n', config)[0]
            liebert_oids = [lie_serial, lie_model, lie_mac, lie_mfdate]
        upsInv.updateUPS(apc_oids, liebert_oids)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update UPSes:\n\n"
            logging.exception(msg)
            print "Could not update UPSes. See", logPath, "for details."
        else:
            print "Could not update UPSes."

def updatePhone():
    """ This function reads necessary information from a config file, then
    calls the updatePhones function in phoneInv.py.
    """
    try:
        from phone import phoneInv
        print "Updating phones..."

        with open(configPath) as f:
            config = f.read()
            phone_login = re.findall(r'Phone_Login = (.+?)\n', config)[0]
        phoneInv.updatePhones(path, phone_login)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update phones:\n\n"
            logging.exception(msg)
            print "Could not update phones. See", logPath, "for details."
        else:
            print "Could not update phones."

if __name__ == "__main__":
    import django
    sys.path.append(os.path.abspath(path))
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "network_inventory.settings")
    django.setup()

    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--debug", help="enable logging on update failure", action="store_true")
    parser.add_argument("-a", "--accesspoint", help="update access points", action="store_true")
    parser.add_argument("-e", "--apstatus", help="check AP status and send email alerts", action="store_true")
    parser.add_argument("-s", "--switch", help="update switches", action="store_true")
    parser.add_argument("-u", "--ups", help="update UPSes", action="store_true")
    parser.add_argument("-p", "--phone", help="update phones", action="store_true")
    args = parser.parse_args()
    if args.debug:
        debug = True
    if args.accesspoint:
        updateAP()
        updateAPStatus()
    if args.apstatus:
        updateAPStatus()
    if args.switch:
        updateSwitch()
    if args.ups:
        updateUPS()
    if args.phone:
        updatePhone()

    if not args.apstatus and not args.accesspoint and not args.switch and not args.ups and not args.phone:
        updateAP()
        updateAPStatus()
        updateSwitch()
        updateUPS()
        updatePhone()
<commit_msg>Allowed commenting out lines in config file with a '#'<commit_after>#!usr/bin/env python
""" UpdateDevices.py

This program controls several other Python scripts that each update the inventory
information for a network device. Currently supported devices are access points,
switches, APC or Liebert UPSes, and phones using VOIP.

Uses a configuration file for information about device logins, OIDs, IP addresses,
and so on.

Usage: UpdateDevices.py [-h] [-d] [-a] [-s] [-u] [-p]

Optional arguments:
  -h, --help         show a help message and exit
  -d, --debug        enable logging on update failure
  -a, --accesspoint  update access points
  -e, --apstatus     update access point status and send notification email
  -s, --switch       update switches
  -u, --ups          update UPSes
  -p, --phone        update phones

Running with no arguments will attempt to update all devices.
"""

import sys
import os
import re
import argparse
import logging
import datetime

path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
configPath = os.path.join(path, "config")
logPath = os.path.join(path, "log")
logging.basicConfig(filename=logPath)
debug = False;

def updateAP():
    """ This function reads necessary information from a config file, then
    calls the updateAccessPoints function in apInv.py.
    """
    try:
        from ap import apInv
        print "Updating access points..."

        with open(configPath) as f:
            config = f.read()
            ap_mac = re.findall(r'(?<!#)AP_MAC_OID = (.+?)\n', config)[0]
            ap_name = re.findall(r'(?<!#)AP_Name_OID = (.+?)\n', config)[0]
            ap_ip = re.findall(r'(?<!#)AP_IP_OID = (.+?)\n', config)[0]
            ap_serial = re.findall(r'(?<!#)AP_Serial_OID = (.+?)\n', config)[0]
            ap_model = re.findall(r'(?<!#)AP_Model_OID = (.+?)\n', config)[0]
            ap_status = re.findall(r'(?<!#)AP_Status_OID = (.+?)\n', config)[0]
            ap_oids = [ap_mac, ap_name, ap_ip, ap_serial, ap_model, ap_status]
            ap_controllers = re.findall(r'(?<!#)AP_ControllerIPs = \[(.+?)\]', config)[0].split(",")
        apInv.updateAccessPoints(path, ap_oids, ap_controllers)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update access points:\n\n"
            logging.exception(msg)
            print "Could not update access points. See", logPath, "for details."
        else:
            print "Could not update access points."

def updateAPStatus():
    """ This function reads necessary information from a config file, then
    calls the updateStatus function in apInv.py.
    """
    try:
        from ap import apInv
        print "Updating access point status..."

        with open(configPath) as f:
            config = f.read()
            ap_status = re.findall(r'(?<!#)AP_Status_OID = (.+?)\n', config)[0]
            ap_controllers = re.findall(r'(?<!#)AP_ControllerIPs = \[(.+?)\]', config)[0].split(",")
            ap_emailFrom = re.findall(r'(?<!#)AP_Email_From = (.+?)\n', config)[0]
            ap_emailTo = re.findall(r'(?<!#)AP_Email_To = (.+?)\n', config)
            ap_emailServer = re.findall(r'(?<!#)AP_Email_Server = (.+?)\n', config)[0]
            ap_email = {"from":ap_emailFrom, "to":ap_emailTo, "server":ap_emailServer}
        apInv.updateStatus(ap_controllers, ap_status, ap_email)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update access points:\n\n"
            logging.exception(msg)
            print "Could not update access point status. See", logPath, "for details."
        else:
            print "Could not update access point status."

def updateSwitch(switch_IPs = []):
    """ This function reads necessary information from a config file, then
    calls the updateSwitches function in switchInv.py.
    """
    try:
        from switch import switchInv
        print "Updating switches..."

        with open(configPath) as f:
            config = f.read()
            switch_login = re.findall(r'(?<!#)Switch_Login = \[(.+?)\]', config)[0].split(",")
            if len(switch_IPs) == 0:
                switch_IPs = []
                try:
                    switch_IPs = re.findall(r'(?<!#)Switch_IPs = \[(.+?)\]', config)[0].split(",")
                except:
                    pass
        switchInv.updateSwitches(switch_login, switch_IPs)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update switches:\n\n"
            logging.exception(msg)
            print "Could not update switches. See", logPath, "for details."
        else:
            print "Could not update switches."

def updateUPS():
    """ This function reads necessary information from a config file, then
    calls the updateUPS function in upsInv.py.
    """
    try:
        from ups import upsInv
        print "Updating UPSes..."

        with open(configPath) as f:
            config = f.read()
            apc_serial = re.findall(r'(?<!#)APC_Serial_OID = (.+?)\n', config)[0]
            apc_model = re.findall(r'(?<!#)APC_Model_OID = (.+?)\n', config)[0]
            apc_mac = re.findall(r'(?<!#)APC_MAC_OID = (.+?)\n', config)[0]
            apc_name = re.findall(r'(?<!#)APC_Name_OID = (.+?)\n', config)[0]
            apc_oids = [apc_serial, apc_model, apc_mac, apc_name]
            lie_serial = re.findall(r'(?<!#)Liebert_Serial_OID = (.+?)\n', config)[0]
            lie_model = re.findall(r'(?<!#)Liebert_Model_OID = (.+?)\n', config)[0]
            lie_mac = re.findall(r'(?<!#)Liebert_MAC_OID = (.+?)\n', config)[0]
            lie_mfdate = re.findall(r'(?<!#)Liebert_MfDate_OID = (.+?)\n', config)[0]
            liebert_oids = [lie_serial, lie_model, lie_mac, lie_mfdate]
        upsInv.updateUPS(apc_oids, liebert_oids)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update UPSes:\n\n"
            logging.exception(msg)
            print "Could not update UPSes. See", logPath, "for details."
        else:
            print "Could not update UPSes."

def updatePhone():
    """ This function reads necessary information from a config file, then
    calls the updatePhones function in phoneInv.py.
    """
    try:
        from phone import phoneInv
        print "Updating phones..."

        with open(configPath) as f:
            config = f.read()
            phone_login = re.findall(r'(?<!#)Phone_Login = (.+?)\n', config)[0]
        phoneInv.updatePhones(path, phone_login)

    except:
        if debug:
            msg = "========================================================\n"\
                + str(datetime.datetime.today()) + "\n"\
                + "Could not update phones:\n\n"
            logging.exception(msg)
            print "Could not update phones. See", logPath, "for details."
        else:
            print "Could not update phones."

if __name__ == "__main__":
    import django
    sys.path.append(os.path.abspath(path))
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "network_inventory.settings")
    django.setup()

    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--debug", help="enable logging on update failure", action="store_true")
    parser.add_argument("-a", "--accesspoint", help="update access points", action="store_true")
    parser.add_argument("-e", "--apstatus", help="check AP status and send email alerts", action="store_true")
    parser.add_argument("-s", "--switch", help="update switches", action="store_true")
    parser.add_argument("-u", "--ups", help="update UPSes", action="store_true")
    parser.add_argument("-p", "--phone", help="update phones", action="store_true")
    args = parser.parse_args()
    if args.debug:
        debug = True
    if args.accesspoint:
        updateAP()
        updateAPStatus()
    if args.apstatus:
        updateAPStatus()
    if args.switch:
        updateSwitch()
    if args.ups:
        updateUPS()
    if args.phone:
        updatePhone()

    if not args.apstatus and not args.accesspoint and not args.switch and not args.ups and not args.phone:
        updateAP()
        updateAPStatus()
        updateSwitch()
        updateUPS()
        updatePhone()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from compas.files import URDFParser
from compas.geometry import Vector
from compas.geometry import transform_vectors
from compas.geometry import Rotation
from compas.geometry import Transformation
from compas.geometry import Translation

from compas.robots.model.geometry import Origin
from compas.robots.model.geometry import _parse_floats

__all__ = [
    'Joint',
    'ParentLink',
    'ChildLink',
    'Calibration',
    'Dynamics',
    'Limit',
    'Axis',
    'Mimic',
    'SafetyController'
]

class ParentLink(object):
    """Describes a parent relation between a joint its parent link."""

    def __init__(self, link):
        self.link = link

    def __str__(self):
        return str(self.link)

class ChildLink(object):
    """Describes a child relation between a joint and its child link."""

    def __init__(self, link):
        self.link = link

    def __str__(self):
        return str(self.link)

class Calibration(object):
    """Reference positions of the joint, used to calibrate the absolute position."""

    def __init__(self, rising=0.0, falling=0.0, reference_position=0.0):
        self.rising = float(rising)
        self.falling = float(falling)
        self.reference_position = float(reference_position)

class Dynamics(object):
    """Physical properties of the joint used for simulation of dynamics."""

    def __init__(self, damping=0.0, friction=0.0):
        self.damping = float(damping)
        self.friction = float(friction)

class Limit(object):
    """Joint limit properties.

    Attributes:
        effort: Maximum joint effort.
        velocity: Maximum joint velocity.
        lower: Lower joint limit (radians for revolute joints, meter for
            prismatic joints).
        upper: Upper joint limit (radians for revolute joints, meter for
            prismatic joints).
    """

    def __init__(self, effort=0.0, velocity=0.0, lower=0.0, upper=0.0):
        self.effort = float(effort)
        self.velocity = float(velocity)
        self.lower = float(lower)
        self.upper = float(upper)

    def scale(self, factor):
        self.lower *= factor
        self.upper *= factor

class Mimic(object):
    """Description of joint mimic."""

    def __init__(self, joint, multiplier=1.0, offset=0):
        self.joint = joint
        self.multiplier = multiplier
        self.offset = offset

class SafetyController(object):
    """Safety controller properties."""

    def __init__(self, k_velocity, k_position=0.0, soft_lower_limit=0.0, soft_upper_limit=0.0):
        self.k_velocity = float(k_velocity)
        self.k_position = float(k_position)
        self.soft_lower_limit = float(soft_lower_limit)
        self.soft_upper_limit = float(soft_upper_limit)

class Axis(object):
    """Representation of an axis or vector."""

    def __init__(self, xyz='0 0 0', **kwargs):
        # We are not using Vector here because we
        # cannot attach _urdf_source to it due to __slots__
        xyz = _parse_floats(xyz)
        self.x = xyz[0]
        self.y = xyz[1]
        self.z = xyz[2]
        self.attr = kwargs

    def copy(self):
        cls = type(self)
        return cls("%f %f %f" % (self.x, self.y, self.z))

    def transform(self, transformation):
        xyz = transform_vectors(
            [[self.x, self.y, self.z]], transformation.matrix)
        self.x = xyz[0][0]
        self.y = xyz[0][1]
        self.z = xyz[0][2]

    @property
    def vector(self):
        return Vector(self.x, self.y, self.z)

    def __str__(self):
        return "[%.3f, %.3f, %.3f]" % (self.x, self.y, self.z)

class Joint(object):
    """Representation of the kinematics and dynamics of a joint and its safety
    limits.

    Attributes:
        name: Unique name for the joint.
        type: Joint type.
        origin: Transformation from the parent link to the child link frame.
        parent: Name of the parent link.
        child: Name of the child link.
        axis: Joint axis specified in the joint frame. Represents the axis of
            rotation for revolute joints, the axis of translation for prismatic
            joints, and the surface normal for planar joints. The axis is
            specified in the joint frame of reference.
        calibration: Reference positions of the joint, used to calibrate the
            absolute position of the joint.
        dynamics: Physical properties of the joint. These values are used to
            specify modeling properties of the joint, particularly useful for
            simulation.
        limit: Joint limit properties.
        safety_controller: Safety controller properties.
        mimic: Used to specify that the defined joint mimics another existing
            joint.
        attr: Non-standard attributes.
        child_link: the joint's child link
        position (float): The current position of the joint. This depends on the
            joint type, i.e. for revolute joints, it will be the rotation angle
            in radians, for prismatic joints the translation in meters.
    """

    REVOLUTE = 0
    CONTINUOUS = 1
    PRISMATIC = 2
    FIXED = 3
    FLOATING = 4
    PLANAR = 5

    SUPPORTED_TYPES = ('revolute', 'continuous', 'prismatic', 'fixed',
                       'floating', 'planar')

    def __init__(self, name, type, parent, child, origin=None, axis=None, calibration=None, dynamics=None, limit=None, safety_controller=None, mimic=None, **kwargs):
        if type not in (Joint.SUPPORTED_TYPES):
            raise ValueError('Unsupported joint type: %s' % type)

        self.name = name
        self.type = Joint.SUPPORTED_TYPES.index(type)
        self.parent = parent if isinstance(parent, ParentLink) else ParentLink(parent)
        self.child = child if isinstance(child, ChildLink) else ChildLink(child)
        self.origin = origin
        self.axis = axis
        self.calibration = calibration
        self.dynamics = dynamics
        self.limit = limit
        self.safety_controller = safety_controller
        self.mimic = mimic
        self.attr = kwargs
        self.child_link = None
        self.position = 0
        self.init_origin = origin.copy() if origin else None
        self.init_axis = axis.copy() if axis else None

        switcher = {
            Joint.REVOLUTE: self.calculate_revolute_transformation,
            Joint.CONTINUOUS: self.calculate_continous_transformation,
            Joint.PRISMATIC: self.calculate_prismatic_transformation,
            Joint.FIXED: self.calculate_fixed_transformation,
            Joint.FLOATING: self.calculate_floating_transformation,
            Joint.PLANAR: self.calculate_planar_transformation
        }
        # set the transformation function according to the type
        self.calculate_transformation = switcher.get(self.type)

    @property
    def current_transformation(self):
        if self.origin:
            return Transformation.from_frame(self.origin)
        else:
            return Transformation()

    @property
    def init_transformation(self):
        if self.init_origin:
            return Transformation.from_frame(self.init_origin)
        else:
            return Transformation()

    @property
    def reset_transformation(self):
        return self.init_transformation * self.current_transformation.inverse()

    def reset_transform(self):
        if self.init_origin:
            self.origin = self.init_origin.copy()
        if self.init_axis:
            self.axis = self.init_axis.copy()

    def transform(self, transformation):
        if self.origin:
            self.origin.transform(transformation)
        if self.axis:
            self.axis.transform(transformation)

    def create(self, transformation):
        if self.origin:
            self.origin.transform(transformation)
            self.init_origin = self.origin.copy()
        if self.axis:
            self.axis.transform(self.current_transformation)
            self.init_axis = self.axis.copy()

    def calculate_revolute_transformation(self, position):
        """Returns a transformation of a revolute joint.

        A revolute joint rotates about the axis and has a limited range
        specified by the upper and lower limits.

        Args:
            position (float): the angle in radians
        """
        position = max(min(position, self.limit.upper), self.limit.lower)
        return self.calculate_continous_transformation(position)

    def calculate_continous_transformation(self, position):
        """Returns a transformation of a continous joint.

        A continous joint rotates about the axis and has no upper and lower
        limits.

        Args:
            position (float): the angle in radians
        """
        return Rotation.from_axis_and_angle(self.axis.vector, position, self.origin.point)

    def calculate_prismatic_transformation(self, position):
        """Returns a transformation of a prismatic joint.

        A prismatic joint slides along the axis and has a limited range
        specified by the upper and lower limits.

        Args:
            position (float): the movement in meters.
        """
        position = max(min(position, self.limit.upper), self.limit.lower)
        return Translation(self.axis.vector * position)

    # does this ever happen?
    def calculate_fixed_transformation(self, position):
        """Returns an identity transformation.

        The fixed joint is is not really a joint because it cannot move. All
        degrees of freedom are locked.
        """
        return Transformation()

    def calculate_floating_transformation(self, position):
        """Returns a transformation of a floating joint.

        A floating joint allows motion for all 6 degrees of freedom.
        """
        raise NotImplementedError

    def calculate_planar_transformation(self, position):
        """Returns a transformation of a planar joint.

        A planar joint allows motion in a plane perpendicular to the axis.
        """
        raise NotImplementedError

    def calculate_transformation(self, position):
        """Returns the transformation of the joint.

        This function is overwitten in the init based on the joint type.

        Args:
            position (float): radians or meters.
        """
        pass

    def is_configurable(self):
        """Returns ``True`` if the joint can be configured, otherwise ``False``."""
        return self.type != Joint.FIXED

    def is_scalable(self):
        """Returns ``True`` if the joint can be scaled, otherwise ``False``."""
        return self.type in [Joint.PLANAR, Joint.PRISMATIC]

    def scale(self, factor):
        self.origin.scale(factor)
        self.init_origin.point *= factor
        if self.is_scalable():
            self.limit.scale(factor)

URDFParser.install_parser(Joint, 'robot/joint')
URDFParser.install_parser(ParentLink, 'robot/joint/parent')
URDFParser.install_parser(ChildLink, 'robot/joint/child')
URDFParser.install_parser(Calibration, 'robot/joint/calibration')
URDFParser.install_parser(Dynamics, 'robot/joint/dynamics')
URDFParser.install_parser(Limit, 'robot/joint/limit')
URDFParser.install_parser(Axis, 'robot/joint/axis')
URDFParser.install_parser(Mimic, 'robot/joint/mimic')
URDFParser.install_parser(SafetyController, 'robot/joint/safety_controller')

URDFParser.install_parser(Origin, 'robot/joint/origin')
<commit_msg>removing init and reset<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from compas.files import URDFParser
from compas.geometry import Vector
from compas.geometry import transform_vectors
from compas.geometry import Rotation
from compas.geometry import Transformation
from compas.geometry import Translation

from compas.robots.model.geometry import Origin
from compas.robots.model.geometry import _parse_floats

__all__ = [
    'Joint',
    'ParentLink',
    'ChildLink',
    'Calibration',
    'Dynamics',
    'Limit',
    'Axis',
    'Mimic',
    'SafetyController'
]

class ParentLink(object):
    """Describes a parent relation between a joint its parent link."""

    def __init__(self, link):
        self.link = link

    def __str__(self):
        return str(self.link)

class ChildLink(object):
    """Describes a child relation between a joint and its child link."""

    def __init__(self, link):
        self.link = link

    def __str__(self):
        return str(self.link)

class Calibration(object):
    """Reference positions of the joint, used to calibrate the absolute position."""

    def __init__(self, rising=0.0, falling=0.0, reference_position=0.0):
        self.rising = float(rising)
        self.falling = float(falling)
        self.reference_position = float(reference_position)

class Dynamics(object):
    """Physical properties of the joint used for simulation of dynamics."""

    def __init__(self, damping=0.0, friction=0.0):
        self.damping = float(damping)
        self.friction = float(friction)

class Limit(object):
    """Joint limit properties.

    Attributes:
        effort: Maximum joint effort.
        velocity: Maximum joint velocity.
        lower: Lower joint limit (radians for revolute joints, meter for
            prismatic joints).
        upper: Upper joint limit (radians for revolute joints, meter for
            prismatic joints).
    """

    def __init__(self, effort=0.0, velocity=0.0, lower=0.0, upper=0.0):
        self.effort = float(effort)
        self.velocity = float(velocity)
        self.lower = float(lower)
        self.upper = float(upper)

    def scale(self, factor):
        self.lower *= factor
        self.upper *= factor

class Mimic(object):
    """Description of joint mimic."""

    def __init__(self, joint, multiplier=1.0, offset=0):
        self.joint = joint
        self.multiplier = multiplier
        self.offset = offset

class SafetyController(object):
    """Safety controller properties."""

    def __init__(self, k_velocity, k_position=0.0, soft_lower_limit=0.0, soft_upper_limit=0.0):
        self.k_velocity = float(k_velocity)
        self.k_position = float(k_position)
        self.soft_lower_limit = float(soft_lower_limit)
        self.soft_upper_limit = float(soft_upper_limit)

class Axis(object):
    """Representation of an axis or vector."""

    def __init__(self, xyz='0 0 0', **kwargs):
        # We are not using Vector here because we
        # cannot attach _urdf_source to it due to __slots__
        xyz = _parse_floats(xyz)
        self.x = xyz[0]
        self.y = xyz[1]
        self.z = xyz[2]
        self.attr = kwargs

    def copy(self):
        cls = type(self)
        return cls("%f %f %f" % (self.x, self.y, self.z))

    def transform(self, transformation):
        xyz = transform_vectors(
            [[self.x, self.y, self.z]], transformation.matrix)
        self.x = xyz[0][0]
        self.y = xyz[0][1]
        self.z = xyz[0][2]

    @property
    def vector(self):
        return Vector(self.x, self.y, self.z)

    def __str__(self):
        return "[%.3f, %.3f, %.3f]" % (self.x, self.y, self.z)

class Joint(object):
    """Representation of the kinematics and dynamics of a joint and its safety
    limits.

    Attributes:
        name: Unique name for the joint.
        type: Joint type.
        origin: Transformation from the parent link to the child link frame.
        parent: Name of the parent link.
        child: Name of the child link.
        axis: Joint axis specified in the joint frame. Represents the axis of
            rotation for revolute joints, the axis of translation for prismatic
            joints, and the surface normal for planar joints. The axis is
            specified in the joint frame of reference.
        calibration: Reference positions of the joint, used to calibrate the
            absolute position of the joint.
        dynamics: Physical properties of the joint. These values are used to
            specify modeling properties of the joint, particularly useful for
            simulation.
        limit: Joint limit properties.
        safety_controller: Safety controller properties.
        mimic: Used to specify that the defined joint mimics another existing
            joint.
        attr: Non-standard attributes.
        child_link: the joint's child link
        position (float): The current position of the joint. This depends on the
            joint type, i.e. for revolute joints, it will be the rotation angle
            in radians, for prismatic joints the translation in meters.
    """

    REVOLUTE = 0
    CONTINUOUS = 1
    PRISMATIC = 2
    FIXED = 3
    FLOATING = 4
    PLANAR = 5

    SUPPORTED_TYPES = ('revolute', 'continuous', 'prismatic', 'fixed',
                       'floating', 'planar')

    def __init__(self, name, type, parent, child, origin=None, axis=None, calibration=None, dynamics=None, limit=None, safety_controller=None, mimic=None, **kwargs):
        if type not in (Joint.SUPPORTED_TYPES):
            raise ValueError('Unsupported joint type: %s' % type)

        self.name = name
        self.type = Joint.SUPPORTED_TYPES.index(type)
        self.parent = parent if isinstance(parent, ParentLink) else ParentLink(parent)
        self.child = child if isinstance(child, ChildLink) else ChildLink(child)
        self.origin = origin
        self.axis = axis
        self.calibration = calibration
        self.dynamics = dynamics
        self.limit = limit
        self.safety_controller = safety_controller
        self.mimic = mimic
        self.attr = kwargs
        self.child_link = None
        self.position = 0

        switcher = {
            Joint.REVOLUTE: self.calculate_revolute_transformation,
            Joint.CONTINUOUS: self.calculate_continous_transformation,
            Joint.PRISMATIC: self.calculate_prismatic_transformation,
            Joint.FIXED: self.calculate_fixed_transformation,
            Joint.FLOATING: self.calculate_floating_transformation,
            Joint.PLANAR: self.calculate_planar_transformation
        }
        # set the transformation function according to the type
        self.calculate_transformation = switcher.get(self.type)

    @property
    def current_transformation(self):
        if self.origin:
            return Transformation.from_frame(self.origin)
        else:
            return Transformation()

    def transform(self, transformation):
        if self.origin:
            self.origin.transform(transformation)
        if self.axis:
            self.axis.transform(transformation)

    def create(self, transformation):
        if self.origin:
            self.origin.transform(transformation)
        if self.axis:
            self.axis.transform(self.current_transformation)

    def calculate_revolute_transformation(self, position):
        """Returns a transformation of a revolute joint.

        A revolute joint rotates about the axis and has a limited range
        specified by the upper and lower limits.

        Args:
            position (float): the angle in radians
        """
        position = max(min(position, self.limit.upper), self.limit.lower)
        return self.calculate_continous_transformation(position)

    def calculate_continous_transformation(self, position):
        """Returns a transformation of a continous joint.

        A continous joint rotates about the axis and has no upper and lower
        limits.

        Args:
            position (float): the angle in radians
        """
        return Rotation.from_axis_and_angle(self.axis.vector, position, self.origin.point)

    def calculate_prismatic_transformation(self, position):
        """Returns a transformation of a prismatic joint.

        A prismatic joint slides along the axis and has a limited range
        specified by the upper and lower limits.

        Args:
            position (float): the movement in meters.
        """
        position = max(min(position, self.limit.upper), self.limit.lower)
        return Translation(self.axis.vector * position)

    # does this ever happen?
    def calculate_fixed_transformation(self, position):
        """Returns an identity transformation.

        The fixed joint is is not really a joint because it cannot move. All
        degrees of freedom are locked.
        """
        return Transformation()

    def calculate_floating_transformation(self, position):
        """Returns a transformation of a floating joint.

        A floating joint allows motion for all 6 degrees of freedom.
        """
        raise NotImplementedError

    def calculate_planar_transformation(self, position):
        """Returns a transformation of a planar joint.

        A planar joint allows motion in a plane perpendicular to the axis.
        """
        raise NotImplementedError

    def calculate_transformation(self, position):
        """Returns the transformation of the joint.

        This function is overwitten in the init based on the joint type.

        Args:
            position (float): radians or meters.
        """
        pass

    def is_configurable(self):
        """Returns ``True`` if the joint can be configured, otherwise ``False``."""
        return self.type != Joint.FIXED

    def is_scalable(self):
        """Returns ``True`` if the joint can be scaled, otherwise ``False``."""
        return self.type in [Joint.PLANAR, Joint.PRISMATIC]

    def scale(self, factor):
        self.origin.scale(factor)
        if self.is_scalable():
            self.limit.scale(factor)

URDFParser.install_parser(Joint, 'robot/joint')
URDFParser.install_parser(ParentLink, 'robot/joint/parent')
URDFParser.install_parser(ChildLink, 'robot/joint/child')
URDFParser.install_parser(Calibration, 'robot/joint/calibration')
URDFParser.install_parser(Dynamics, 'robot/joint/dynamics')
URDFParser.install_parser(Limit, 'robot/joint/limit')
URDFParser.install_parser(Axis, 'robot/joint/axis')
URDFParser.install_parser(Mimic, 'robot/joint/mimic')
URDFParser.install_parser(SafetyController, 'robot/joint/safety_controller')

URDFParser.install_parser(Origin, 'robot/joint/origin')
<|endoftext|>


<commit_before>import numpy as np
import glob
import time
import sys
import os

import scipy.io as sio

import librosa
import kl_nmf

def train(nmf, updateW=True, criterion=0.0005, maxiter=1000):
    score = nmf.bound()
    objs = []
    for i in xrange(maxiter):
        start_t = time.time()
        nmf.update(updateW=updateW, disp=1)
        t = time.time() - start_t

        lastscore = score
        score = nmf.bound()
        objs.append(score)
        improvement = (score - lastscore) / abs(lastscore)
        print ('iteration {}: bound = {:.2f} ({:.5f} improvement) '
               'time = {:.2f}'.format(i, score, improvement, t))
        if i > 10 and improvement < criterion:
            break
    return objs

def learn_dictionary(prior_mat, K, d, n_fft=1024, hop_length=512, seed=None):
    prior = sio.loadmat(prior_mat)
    U = prior['U'].T
    gamma = prior['gamma']
    alpha = prior['alpha'].ravel()

    paths = sorted(glob.glob('denoise/TIMIT_speech/*.wav'))

    n = len(paths)
    W_nu = np.zeros((n, n_fft/2 + 1, K))
    W_rho = np.zeros_like(W_nu)

    for (i, path) in enumerate(paths):
        print('Learn dictionary for {}...'.format(path))
        x, sr = librosa.load(path, sr=None)
        X = np.abs(librosa.stft(x, n_fft=n_fft, hop_length=hop_length))

        nmf_sf = kl_nmf.KL_NMF(X, K=K, d=d, seed=seed,
                               U=U, gamma=gamma, alpha=alpha)
        train(nmf_sf)
        W_nu[i], W_rho[i] = nmf_sf.nuw, nmf_sf.rhow

    sio.savemat('SF_TIMIT60_dict_{}_K{}_d{}.mat'.format(prior_mat, K, d),
                {'W_nu': W_nu, 'W_rho': W_rho})
    return

if __name__ == '__main__':
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
    prior_mat = sys.argv[1]
    K = int(sys.argv[2])
    d = int(sys.argv[3])
    learn_dictionary(prior_mat, K, d, seed=98765)
    pass
<commit_msg>added seed parameter in the script<commit_after>import numpy as np
import glob
import time
import sys
import os

import scipy.io as sio

import librosa
import kl_nmf

def train(nmf, updateW=True, criterion=0.0005, maxiter=1000):
    score = nmf.bound()
    objs = []
    for i in xrange(maxiter):
        start_t = time.time()
        nmf.update(updateW=updateW, disp=1)
        t = time.time() - start_t

        lastscore = score
        score = nmf.bound()
        objs.append(score)
        improvement = (score - lastscore) / abs(lastscore)
        print ('iteration {}: bound = {:.2f} ({:.5f} improvement) '
               'time = {:.2f}'.format(i, score, improvement, t))
        if i > 10 and improvement < criterion:
            break
    return objs

def learn_dictionary(prior_mat, K, d, seed, n_fft=1024, hop_length=512):
    prior = sio.loadmat(prior_mat)
    U = prior['U'].T
    gamma = prior['gamma']
    alpha = prior['alpha'].ravel()

    paths = sorted(glob.glob('denoise/TIMIT_speech/*.wav'))

    n = len(paths)
    W_nu = np.zeros((n, n_fft/2 + 1, K))
    W_rho = np.zeros_like(W_nu)

    for (i, path) in enumerate(paths):
        print('Learn dictionary for {}...'.format(path))
        x, sr = librosa.load(path, sr=None)
        X = np.abs(librosa.stft(x, n_fft=n_fft, hop_length=hop_length))

        nmf_sf = kl_nmf.KL_NMF(X, K=K, d=d, seed=seed,
                               U=U, gamma=gamma, alpha=alpha)
        train(nmf_sf)
        W_nu[i], W_rho[i] = nmf_sf.nuw, nmf_sf.rhow

    sio.savemat('SF_TIMIT60_dict_{}_K{}_d{}_seed{}.mat'.format(prior_mat, K, d, seed),
                {'W_nu': W_nu, 'W_rho': W_rho})
    return

if __name__ == '__main__':
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
    prior_mat = sys.argv[1]
    K = int(sys.argv[2])
    d = int(sys.argv[3])
    seed = int(sys.argv[4])
    learn_dictionary(prior_mat, K, d, seed)
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Measure performance for a run file, an intervalType (day,week, all), and a judgmentLevel (1 or 2).

Creates the output next to the input file, appending "-eval-week.tsv" (for weeks)

 example usage
 python measure-slice.py --intervalType week --judgmentLevel 1 -f ~/kba-evaluation/taia/data/umass-runs/UMass_CIIR-PC_RM20_1500.gz
"""

import os.path
import numpy as np
from numpy.lib import recfunctions
import gzip
from metrics import *
import string
from utils import *
from truthutil import *
from argparse import ArgumentParser
from targetentities import *
import csv
import tempfile

DEBUG = False

metrics = {
    'nDCG@R': ndcgREval, 'Prec@R': precREval, 'MAP': mapEval}

runFile = '~/kba-evaluation/taia/data/umass-runs/UMass_CIIR-PC_RM20_1500.gz'

parser = ArgumentParser()
parser.add_argument('--intervalType', help='slice segmentation: day, week, or all', default='all')
parser.add_argument('--judgmentLevel', type=int, help='Judgement level', default=1)
parser.add_argument('--trec_eval', action='store_true', help='Also create output for evaluation with trec_eval',
                    default=False)
parser.add_argument('-f', '--runFile', metavar='FILE', default=runFile)
args = parser.parse_args()

judgmentLevel = args.judgmentLevel
intervalType = args.intervalType
runFile = args.runFile
DUMP_TREC_EVAL = args.trec_eval  # also create output for trec_eval.

entry_dtype = np.dtype([('runname', '50a'), ('docid', '50a'), ('query', '50a'),
                        ('confidence', 'd4')])
judg_dtype = np.dtype([('docid', '50a'), ('query', '50a'),
                       ('label', 'd4')])
eval_dtype = np.dtype(
    [('team', '50a'), ('runname', '50a'), ('query', '50a'), ('intervalLow', 'd4'), ('intervalUp', 'd4'),
     ('unjudged', '50a'), ('judgmentLevel', 'd4'), ('metric', '50a'), ('value', 'f4')])

class Annotations(object):
    """ represents a set of annotations that have been sorted by entity.
        We do this on disk to keep the memory footprint down. """

    def __init__(self, fname):
        #print 'generating prediction cache for', fname.filename
        self.entity_files = {}

        annotation_file = csv.reader(fname, delimiter='\t')
        for line in annotation_file:
            if len(line) < 4: continue
            entity = line[3]
            if entity not in self.entity_files:
                self.entity_files[entity] = tempfile.NamedTemporaryFile(prefix='kba', delete=True)
            self.entity_files[entity].write('\t'.join(line) + '\n')

        print 'indexed predictions for %d entities' % (len(self.entity_files))

    def get_predictions(self, entity):
        if entity not in self.entity_files:
            return None
        f = self.entity_files[entity]
        f.seek(0)
        a = np.genfromtxt(f, dtype=entry_dtype, usecols=[1, 2, 3, 4])
        print 'read %d predictions' % len(a)
        times = [int(t) for t in np.core.defchararray.partition(a['docid'], '-')[:, 0]]
        return recfunctions.append_fields(a, 'time', times)

def read_judgments(fname):
    a = np.genfromtxt(fname, dtype=judg_dtype, usecols=[2, 3, 5])
    times = [int(t) for t in np.core.defchararray.partition(a['docid'], '-')[:, 0]]
    return recfunctions.append_fields(a, 'time', times)

def read_zipped_predictions(fname):
    with gzip.open(fname) as f:
        return Annotations(f)

def readPredictionsHeader(fname):
    f = gzip.open(fname, mode='r')
    firstline = f.readline()
    secondline = f.readline()
    team, runname, rest = string.split(secondline, maxsplit=2)
    f.close()
    return (team, runname)

team, runname = readPredictionsHeader(os.path.expanduser(runFile))
print team, runname

testEntityList = ['Boris_Berezovsky_(businessman)', 'Boris_Berezovsky_(pianist)', 'Alex_Kapranos', 'James_McCartney']

entityList = loadEntities()

if DEBUG: entityList = testEntityList

records = []

print runFile

def createEvalRecord(entity, intervalLow, intervalUp, unjudgedAs, judgmentLevel, metricname, score):
    return np.array([(team, runname, entity, intervalLow, intervalUp, unjudgedAs, judgmentLevel, metricname, score)],
                    dtype=eval_dtype)

runOutputFile = None
if DUMP_TREC_EVAL:
    runOutputFilename = "%s-%s.run" % (os.path.expanduser(runFile), intervalType)
    runOutputFile = open(runOutputFilename, 'w')

print 'measuring slices for ', os.path.expanduser(runFile)
annotations = read_zipped_predictions(os.path.expanduser(runFile))

for entity in entityList:
    #print 'fetching annotations for entity',entity

    a = annotations.get_predictions(entity)
    intervalList = intervalBounds[judgmentLevel][intervalType]

    for (i, (intervalLow, intervalUp)) in enumerate(intervalList):
        posGroundTruth = trueDocs(judgmentLevel, entity, intervalLow, intervalUp)

        numPos = len(posGroundTruth)

        # a is None if there are no judgments for this entity in this run
        if a is None and numPos > 0:
            records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numPos', numPos))
            records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numNeg', 0))
            records.append(
                createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numPredictions', 0))
            records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel,
                                            'numPosPredictions', 0))
            records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'posTruthsInterval',
                                            posTruthsInterval(judgmentLevel, entity, intervalLow, intervalUp)))
            records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel,
                                            'posTruthsTotal', posTruths(judgmentLevel, entity)))
            records.append(createEvalRecord(entity,
                                            intervalLow,
                                            intervalUp, '',
                                            judgmentLevel,
                                            'numPosIntervals',
                                            numPosIntervals(judgmentLevel, entity, intervalType)))

            for metricname in metrics:
                records.append(
                    createEvalRecord(entity, intervalLow, intervalUp, 'neg', judgmentLevel, metricname, 0.0))

        else:
            # segment data
            slice = a[np.logical_and(a['time'] >= intervalLow, a['time'] < intervalUp)]
            if (len(slice) > 0):
            # sort by confidence and revert (highest first)
                slice = np.sort(slice, order=['confidence'])[::-1]

            if DUMP_TREC_EVAL:
                for i, row in enumerate(slice):
                    score = float(row['confidence'])
                    rank = i + 1
                    runOutputFile.write(
                        "%s\t%s\t%s\t%d\t%f\t%s\n" % (entity, intervalType, row['docid'], rank, score, row['runname']))

            judgedPosSlice = np.array([np.count_nonzero(posGroundTruth['docid'] == elem['docid']) > 0 for elem in slice])
            unjudgedAsNegSlice = judgedPosSlice
            numNeg = len(slice) - numPos
            if DEBUG: print entity, i, judgmentLevel, 'pos:', numPos, 'neg:', numNeg, 'data:', len(slice)
            if numPos > 0:

                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numPos', numPos))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numNeg', numNeg))
                records.append(
                    createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numPredictions', len(slice)))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel,
                                                'numPosPredictions', sum(judgedPosSlice)))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'posTruthsInterval',
                                                posTruthsInterval(judgmentLevel, entity, intervalLow, intervalUp)))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel,
                                                'posTruthsTotal', posTruths(judgmentLevel, entity)))
                records.append(createEvalRecord(entity,
                                                intervalLow,
                                                intervalUp, '',
                                                judgmentLevel,
                                                'numPosIntervals',
                                                numPosIntervals(judgmentLevel, entity, intervalType)))

                if len(slice) > 0:
                    for metricname in metrics:
                        metric = metricsMap[metricname]
                        score = metric(unjudgedAsNegSlice, numPos, numNeg)
                        records.append(
                            createEvalRecord(entity, intervalLow, intervalUp, 'neg', judgmentLevel, metricname, score))
                else:
                    for metricname in metrics:
                        records.append(
                            createEvalRecord(entity, intervalLow, intervalUp, 'neg', judgmentLevel, metricname, 0.0))

print runFile

if DUMP_TREC_EVAL: runOutputFile.close()

df = np.hstack(records)

evalFile = "%s-eval-%s.tsv" % (os.path.expanduser(runFile), intervalType)
np.savetxt(evalFile, df, fmt='%s\t%s\t%s\t%d\t%d\t%s\t%d\t%s\t%f')

print runFile

for entity in entityList:
    for judgmentLevel in [1]:
        ndcgR = df[np.logical_and(df['query'] == entity,
                                  np.logical_and(df['unjudged'] == 'neg',
                                                 np.logical_and(df['judgmentLevel'] == judgmentLevel,
                                                                df['metric'] == 'MAP')))]['value']
        print entity, judgmentLevel
        print ndcgR
        if len(ndcgR) > 0:
            print np.mean(ndcgR)

for judgmentLevel in [1]:
    for metric in metrics:
        print 'all', 'judgmentLevel =', judgmentLevel
        ndcgR = df[np.logical_and(df['unjudged'] == 'neg',
                                  np.logical_and(df['judgmentLevel'] == judgmentLevel, df['metric'] == metric))][
            'value']
        print ndcgR
        if len(ndcgR) > 0:
            print metric, np.mean(ndcgR)
<commit_msg>handle case of no predictions for one particular entity<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Measure performance for a run file, an intervalType (day,week, all), and a judgmentLevel (1 or 2).

Creates the output next to the input file, appending "-eval-week.tsv" (for weeks)

 example usage
 python measure-slice.py --intervalType week --judgmentLevel 1 -f ~/kba-evaluation/taia/data/umass-runs/UMass_CIIR-PC_RM20_1500.gz
"""

import os.path
import numpy as np
from numpy.lib import recfunctions
import gzip
from metrics import *
import string
from utils import *
from truthutil import *
from argparse import ArgumentParser
from targetentities import *
import csv
import tempfile

DEBUG = False

metrics = {
    'nDCG@R': ndcgREval, 'Prec@R': precREval, 'MAP': mapEval}

runFile = '~/kba-evaluation/taia/data/umass-runs/UMass_CIIR-PC_RM20_1500.gz'

parser = ArgumentParser()
parser.add_argument('--intervalType', help='slice segmentation: day, week, or all', default='all')
parser.add_argument('--judgmentLevel', type=int, help='Judgement level', default=1)
parser.add_argument('--trec_eval', action='store_true', help='Also create output for evaluation with trec_eval',
                    default=False)
parser.add_argument('-f', '--runFile', metavar='FILE', default=runFile)
args = parser.parse_args()

judgmentLevel = args.judgmentLevel
intervalType = args.intervalType
runFile = args.runFile
DUMP_TREC_EVAL = args.trec_eval  # also create output for trec_eval.

entry_dtype = np.dtype([('runname', '50a'), ('docid', '50a'), ('query', '50a'),
                        ('confidence', 'd4')])
judg_dtype = np.dtype([('docid', '50a'), ('query', '50a'),
                       ('label', 'd4')])
eval_dtype = np.dtype(
    [('team', '50a'), ('runname', '50a'), ('query', '50a'), ('intervalLow', 'd4'), ('intervalUp', 'd4'),
     ('unjudged', '50a'), ('judgmentLevel', 'd4'), ('metric', '50a'), ('value', 'f4')])

class Annotations(object):
    """ represents a set of annotations that have been sorted by entity.
        We do this on disk to keep the memory footprint down. """

    def __init__(self, fname):
        #print 'generating prediction cache for', fname.filename
        self.entity_files = {}

        annotation_file = csv.reader(fname, delimiter='\t')
        for line in annotation_file:
            if len(line) < 4: continue
            entity = line[3]
            if entity not in self.entity_files:
                self.entity_files[entity] = tempfile.NamedTemporaryFile(prefix='kba', delete=True)
            self.entity_files[entity].write('\t'.join(line) + '\n')

        print 'indexed predictions for %d entities' % (len(self.entity_files))

    def get_predictions(self, entity):
        if entity not in self.entity_files:
            return None
        f = self.entity_files[entity]
        f.seek(0)
        a = np.genfromtxt(f, dtype=entry_dtype, usecols=[1, 2, 3, 4])
        print 'read %d predictions' % len(a)
        times = [int(t) for t in np.core.defchararray.partition(a['docid'], '-')[:, 0]]
        return recfunctions.append_fields(a, 'time', times)

def read_judgments(fname):
    a = np.genfromtxt(fname, dtype=judg_dtype, usecols=[2, 3, 5])
    times = [int(t) for t in np.core.defchararray.partition(a['docid'], '-')[:, 0]]
    return recfunctions.append_fields(a, 'time', times)

def read_zipped_predictions(fname):
    with gzip.open(fname) as f:
        return Annotations(f)

def readPredictionsHeader(fname):
    f = gzip.open(fname, mode='r')
    firstline = f.readline()
    secondline = f.readline()
    team, runname, rest = string.split(secondline, maxsplit=2)
    f.close()
    return (team, runname)

team, runname = readPredictionsHeader(os.path.expanduser(runFile))
print team, runname

testEntityList = ['Boris_Berezovsky_(businessman)', 'Boris_Berezovsky_(pianist)', 'Alex_Kapranos', 'James_McCartney']

entityList = loadEntities()

if DEBUG: entityList = testEntityList

records = []

print runFile

def createEvalRecord(entity, intervalLow, intervalUp, unjudgedAs, judgmentLevel, metricname, score):
    return np.array([(team, runname, entity, intervalLow, intervalUp, unjudgedAs, judgmentLevel, metricname, score)],
                    dtype=eval_dtype)

runOutputFile = None
if DUMP_TREC_EVAL:
    runOutputFilename = "%s-%s.run" % (os.path.expanduser(runFile), intervalType)
    runOutputFile = open(runOutputFilename, 'w')

print 'measuring slices for ', os.path.expanduser(runFile)
annotations = read_zipped_predictions(os.path.expanduser(runFile))

for entity in entityList:
    #print 'fetching annotations for entity',entity

    a = annotations.get_predictions(entity)
    intervalList = intervalBounds[judgmentLevel][intervalType]

    for (i, (intervalLow, intervalUp)) in enumerate(intervalList):
        posGroundTruth = trueDocs(judgmentLevel, entity, intervalLow, intervalUp)

        numPos = len(posGroundTruth)

        if numPos > 0:
            # a is None if there are no judgments for this entity in this run
            if a is None:
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numPos', numPos))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numNeg', 0))
                records.append(
                    createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numPredictions', 0))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel,
                                                'numPosPredictions', 0))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'posTruthsInterval',
                                                posTruthsInterval(judgmentLevel, entity, intervalLow, intervalUp)))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel,
                                                'posTruthsTotal', posTruths(judgmentLevel, entity)))
                records.append(createEvalRecord(entity,
                                                intervalLow,
                                                intervalUp, '',
                                                judgmentLevel,
                                                'numPosIntervals',
                                                numPosIntervals(judgmentLevel, entity, intervalType)))

                for metricname in metrics:
                    records.append(
                        createEvalRecord(entity, intervalLow, intervalUp, 'neg', judgmentLevel, metricname, 0.0))

            else: # a is not None
                # segment data
                slice = a[np.logical_and(a['time'] >= intervalLow, a['time'] < intervalUp)]
                if (len(slice) > 0):
                # sort by confidence and revert (highest first)
                    slice = np.sort(slice, order=['confidence'])[::-1]

                if DUMP_TREC_EVAL:
                    for i, row in enumerate(slice):
                        score = float(row['confidence'])
                        rank = i + 1
                        runOutputFile.write(
                            "%s\t%s\t%s\t%d\t%f\t%s\n" % (entity, intervalType, row['docid'], rank, score, row['runname']))

                judgedPosSlice = np.array([np.count_nonzero(posGroundTruth['docid'] == elem['docid']) > 0 for elem in slice])
                unjudgedAsNegSlice = judgedPosSlice
                numNeg = len(slice) - numPos
                if DEBUG: print entity, i, judgmentLevel, 'pos:', numPos, 'neg:', numNeg, 'data:', len(slice)

                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numPos', numPos))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numNeg', numNeg))
                records.append(
                    createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'numPredictions', len(slice)))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel,
                                                'numPosPredictions', sum(judgedPosSlice)))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel, 'posTruthsInterval',
                                                posTruthsInterval(judgmentLevel, entity, intervalLow, intervalUp)))
                records.append(createEvalRecord(entity, intervalLow, intervalUp, '', judgmentLevel,
                                                'posTruthsTotal', posTruths(judgmentLevel, entity)))
                records.append(createEvalRecord(entity,
                                                intervalLow,
                                                intervalUp, '',
                                                judgmentLevel,
                                                'numPosIntervals',
                                                numPosIntervals(judgmentLevel, entity, intervalType)))

                if len(slice) > 0:
                    for metricname in metrics:
                        metric = metricsMap[metricname]
                        score = metric(unjudgedAsNegSlice, numPos, numNeg)
                        records.append(
                            createEvalRecord(entity, intervalLow, intervalUp, 'neg', judgmentLevel, metricname, score))
                else:
                    for metricname in metrics:
                        records.append(
                            createEvalRecord(entity, intervalLow, intervalUp, 'neg', judgmentLevel, metricname, 0.0))

print runFile

if DUMP_TREC_EVAL:
    runOutputFile.close()

df = np.hstack(records)

evalFile = "%s-eval-%s.tsv" % (os.path.expanduser(runFile), intervalType)
np.savetxt(evalFile, df, fmt='%s\t%s\t%s\t%d\t%d\t%s\t%d\t%s\t%f')

print runFile

for entity in entityList:
    for judgmentLevel in [1]:
        ndcgR = df[np.logical_and(df['query'] == entity,
                                  np.logical_and(df['unjudged'] == 'neg',
                                                 np.logical_and(df['judgmentLevel'] == judgmentLevel,
                                                                df['metric'] == 'MAP')))]['value']
        print entity, judgmentLevel
        print ndcgR
        if len(ndcgR) > 0:
            print np.mean(ndcgR)

for judgmentLevel in [1]:
    for metric in metrics:
        print 'all', 'judgmentLevel =', judgmentLevel
        ndcgR = df[np.logical_and(df['unjudged'] == 'neg',
                                  np.logical_and(df['judgmentLevel'] == judgmentLevel, df['metric'] == metric))][
            'value']
        print ndcgR
        if len(ndcgR) > 0:
            print metric, np.mean(ndcgR)
<|endoftext|>


<commit_before>from basic import *
import time

with Parser(locals()) as p:
    p.flag('from_pair_seqs').described_as('Restart from the nucleotide sequences (default is to start from parsed clones file)')
    p.flag('multicore').described_as('Allow the pipeline to start multiple processes for motif finding; also run mouse and human analyses simultaneously')
    p.set_help_prefix( "\nThis script will re-run the analysis from the primary citation; it will take a few hours. Use --multicore to speed things up if you have multiple cores to burn\n" )

## These are the distance parameters used in the original publication in Nature.
##
## Since then, we've seen evidence that a gap penalty in the CDR3 region of 12 performs slightly better.
## So that is the new default; here the scale_factor term accounts for the difference in TCRdist values
## (lower overall w/ classic gap penalty of 8)
##
distance_params='gap_penalty_v_region:4,gap_penalty_cdr3_region:8,weight_cdr3_region:3,align_cdr3s:False,distance_matrix:bsd4,trim_cdr3s:True,scale_factor:1.0439137134052388'

for organism in ['mouse','human']:

    if from_pair_seqs:

        pair_seqs_file = 'datasets/{}_pairseqs_v1.tsv'.format(organism)
        assert exists( pair_seqs_file )

        extra_args = ' --find_cdr3_motifs_in_parallel ' if multicore else ''
        cmd_suffix = ' &' if multicore else ''

        cmd = 'nice python run_basic_analysis.py {} --distance_params {} --organism {} --pair_seqs_file {} > {}.log 2> {}.err {}'\
            .format( extra_args, distance_params, organism, pair_seqs_file, pair_seqs_file, pair_seqs_file, cmd_suffix )
        print cmd
        system(cmd)
        time.sleep(1.0) ## short pause
    else:

        clones_file = 'datasets/{}_pairseqs_v1_parsed_seqs_probs_mq20_clones.tsv'.format(organism)
        assert exists( clones_file )

        extra_args = ' --find_cdr3_motifs_in_parallel ' if multicore else ''
        cmd_suffix = ' &' if multicore else ''

        cmd = 'nice python run_basic_analysis.py {} --distance_params {} --organism {} --clones_file {} > {}.log 2> {}.err {}'\
            .format( extra_args, distance_params, organism, clones_file, clones_file, clones_file, cmd_suffix )
        print cmd
        system(cmd)
        time.sleep(1.0) ## short pause

<commit_msg>remove legacy param in distance_params string<commit_after>from basic import *
import time

with Parser(locals()) as p:
    p.flag('from_pair_seqs').described_as('Restart from the nucleotide sequences (default is to start from parsed clones file)')
    p.flag('multicore').described_as('Allow the pipeline to start multiple processes for motif finding; also run mouse and human analyses simultaneously')
    p.set_help_prefix( "\nThis script will re-run the analysis from the primary citation; it will take a few hours. Use --multicore to speed things up if you have multiple cores to burn\n" )

## These are the distance parameters used in the original publication in Nature.
##
## Since then, we've seen evidence that a gap penalty in the CDR3 region of 12 performs slightly better.
## So that is the new default; here the scale_factor term accounts for the difference in TCRdist values
## (lower overall w/ classic gap penalty of 8)
##
distance_params='gap_penalty_v_region:4,gap_penalty_cdr3_region:8,weight_cdr3_region:3,align_cdr3s:False,trim_cdr3s:True,scale_factor:1.0439137134052388'

for organism in ['mouse','human']:

    if from_pair_seqs:

        pair_seqs_file = 'datasets/{}_pairseqs_v1.tsv'.format(organism)
        assert exists( pair_seqs_file )

        extra_args = ' --find_cdr3_motifs_in_parallel ' if multicore else ''
        cmd_suffix = ' &' if multicore else ''

        cmd = 'nice python run_basic_analysis.py {} --distance_params {} --organism {} --pair_seqs_file {} > {}.log 2> {}.err {}'\
            .format( extra_args, distance_params, organism, pair_seqs_file, pair_seqs_file, pair_seqs_file, cmd_suffix )
        print cmd
        system(cmd)
        time.sleep(1.0) ## short pause
    else:

        clones_file = 'datasets/{}_pairseqs_v1_parsed_seqs_probs_mq20_clones.tsv'.format(organism)
        assert exists( clones_file )

        extra_args = ' --find_cdr3_motifs_in_parallel ' if multicore else ''
        cmd_suffix = ' &' if multicore else ''

        cmd = 'nice python run_basic_analysis.py {} --distance_params {} --organism {} --clones_file {} > {}.log 2> {}.err {}'\
            .format( extra_args, distance_params, organism, clones_file, clones_file, clones_file, cmd_suffix )
        print cmd
        system(cmd)
        time.sleep(1.0) ## short pause

<|endoftext|>


<commit_before>import os
import distutils
import sys

from ..platforms.xcode import XcodePlatform
from .. import project

class BoostBuildProject(project.Project):

    @staticmethod
    def identifier():
        return 'boostbuild'

    @staticmethod
    def is_valid_project(definition, needy):
        for name in BoostBuildProject.__valid_jamroot_filenames():
            if os.path.isfile(name):
                return True, 'Jamroot file {} found'.format(name)
        return False, 'no Jamroot file matching {} found'.format(BoostBuildProject.__valid_jamroot_filenames())

    @staticmethod
    def __valid_jamroot_filenames():
        return ['Jamroot', 'Jamroot.jam']

    @staticmethod
    def missing_prerequisites(definition, needy):
        return ['b2'] if not os.path.isfile('bootstrap.sh') and distutils.spawn.find_executable('b2') is None else []

    @staticmethod
    def configuration_keys():
        return project.Project.configuration_keys() | {'b2-args', 'bootstrap-args'}

    def get_build_concurrency_args(self):
        concurrency = self.build_concurrency()

        if concurrency > 1:
            return ['-j', str(concurrency)]
        elif concurrency == 0:
            return ['-j']
        return []

    def configure(self, build_directory):
        bootstrap_args = self.evaluate(self.configuration('bootstrap-args'))
        if not os.path.isfile('bootstrap.sh'):
            if len(bootstrap_args) > 0:
                raise RuntimeError('bootstrap-args was given, but no bootstrap script is present')
            return

        self.command(['./bootstrap.sh'] + bootstrap_args, use_target_overrides=False)

    def build(self, output_directory):
        b2 = './b2' if os.path.isfile('b2') else 'b2'
        b2_args = self.evaluate(self.configuration('b2-args'))
        b2_args.extend(self.get_build_concurrency_args())

        b2_args.append('architecture={}'.format('arm' if 'arm' in self.target().architecture else 'x86'))
        b2_args.append('address-model={}'.format('64' if '64' in self.target().architecture else '32'))

        if self.target().platform.identifier() in ['ios', 'iossimulator', 'tvos', 'tvossimulator']:
            b2_args.append('target-os=iphone')
        elif self.target().platform.identifier() == 'android':
            b2_args.append('target-os=android')

        toolset = 'gcc'
        if sys.platform == 'darwin' and isinstance(self.target().platform, XcodePlatform):
            toolset = 'darwin'
        elif distutils.spawn.find_executable('clang') is not None:
            toolset = 'clang'
        b2_args.append('toolset={}-needy'.format(toolset))

        project_config = ''
        if os.path.exists('project-config.jam'):
            with open('project-config.jam', 'r') as f:
                project_config = f.read()

        if ' : needy : [ os.environ CC ] ;' not in project_config:
            project_config = "import os ;\nusing {} : needy : [ os.environ CC ] ;\n".format(toolset) + project_config
            with open('project-config.jam', 'w') as f:
                f.write(project_config)

        if 'CFLAGS' in os.environ:
            b2_args.append('cflags={}'.format(os.environ['CFLAGS']))
        if 'CXXFLAGS' in os.environ:
            b2_args.append('cxxflags={}'.format(os.environ['CXXFLAGS']))
        if 'LDFLAGS' in os.environ:
            b2_args.append('linkflags={}'.format(os.environ['LDFLAGS']))

        self.command([b2, 'install', '--prefix={}'.format(output_directory)] + b2_args)
<commit_msg>more robust boost build building<commit_after>import os
import distutils
import textwrap
import sys

from ..platforms.xcode import XcodePlatform
from .. import project

class BoostBuildProject(project.Project):

    @staticmethod
    def identifier():
        return 'boostbuild'

    @staticmethod
    def is_valid_project(definition, needy):
        for name in BoostBuildProject.__valid_jamroot_filenames():
            if os.path.isfile(name):
                return True, 'Jamroot file {} found'.format(name)
        return False, 'no Jamroot file matching {} found'.format(BoostBuildProject.__valid_jamroot_filenames())

    @staticmethod
    def __valid_jamroot_filenames():
        return ['Jamroot', 'Jamroot.jam']

    @staticmethod
    def missing_prerequisites(definition, needy):
        return ['b2'] if not os.path.isfile('bootstrap.sh') and distutils.spawn.find_executable('b2') is None else []

    @staticmethod
    def configuration_keys():
        return project.Project.configuration_keys() | {'b2-args', 'bootstrap-args'}

    def get_build_concurrency_args(self):
        concurrency = self.build_concurrency()

        if concurrency > 1:
            return ['-j', str(concurrency)]
        elif concurrency == 0:
            return ['-j']
        return []

    def configure(self, build_directory):
        bootstrap_args = self.evaluate(self.configuration('bootstrap-args'))
        if not os.path.isfile('bootstrap.sh'):
            if len(bootstrap_args) > 0:
                raise RuntimeError('bootstrap-args was given, but no bootstrap script is present')
            return

        self.command(['./bootstrap.sh'] + bootstrap_args, use_target_overrides=False)

    def build(self, output_directory):
        b2 = './b2' if os.path.isfile('b2') else 'b2'
        b2_args = self.evaluate(self.configuration('b2-args'))
        b2_args.extend(self.get_build_concurrency_args())

        b2_args.append('architecture={}'.format('arm' if 'arm' in self.target().architecture else 'x86'))
        b2_args.append('address-model={}'.format('64' if '64' in self.target().architecture else '32'))

        toolset = 'gcc'
        if sys.platform == 'darwin' and isinstance(self.target().platform, XcodePlatform):
            toolset = 'darwin'
        elif distutils.spawn.find_executable('clang') is not None:
            toolset = 'clang'
        b2_args.append('toolset={}-needy'.format(toolset))

        project_config = ''
        if os.path.exists('project-config.jam'):
            with open('project-config.jam', 'r') as f:
                project_config = f.read()

        new_project_config = textwrap.dedent("""\
            ### GENERATED BY NEEDY ###
            import feature ;
            import os ;
        """)

        if toolset == 'darwin':
            new_project_config += "feature.extend instruction-set : {} ;\n".format(self.target().architecture)
            b2_args.append('instruction-set={}'.format(self.target().architecture))

        if self.target().platform.identifier() in ['ios', 'iossimulator']:
            b2_args.append('target-os=iphone')
        elif self.target().platform.identifier() in ['tvos', 'tvossimulator']:
            b2_args.append('target-os=appletv')
            new_project_config += "feature.extend target-os : appletv ;\n"
        elif self.target().platform.identifier() == 'android':
            b2_args.append('target-os=android')

        new_project_config += textwrap.dedent("""\
            using {toolset} : needy : [ os.environ CC ] ;
            ### NO LONGER GENERATED BY NEEDY ###
        """).format(
            toolset=toolset
        )

        skip_lines = False
        for line in project_config.splitlines(True):
            if 'GENERATED BY NEEDY' in line:
                skip_lines = not skip_lines
            if not skip_lines:
                new_project_config += line

        with open('project-config.jam', 'w') as f:
            f.write(new_project_config)

        if 'CFLAGS' in os.environ:
            b2_args.append('cflags={}'.format(os.environ['CFLAGS']))
        if 'CXXFLAGS' in os.environ:
            b2_args.append('cxxflags={}'.format(os.environ['CXXFLAGS']))
        if 'LDFLAGS' in os.environ:
            b2_args.append('linkflags={}'.format(os.environ['LDFLAGS']))

        self.command([b2, 'install', '--prefix={}'.format(output_directory)] + b2_args)
<|endoftext|>


<commit_before>from ..model import Entity, Name

import logging

log = logging.getLogger()

class BuildDbpediaLexicalisations(object):
    "Build entity and name models from DBpedia Spotlight lexicalisations file."
    ENTITY_SCORE = u'http://dbpedia.org/spotlight/score#uriProbability'
    ENTITY_PRE = u'http://dbpedia.org/resource/'
    NAME_SCORE = u'http://dbpedia.org/spotlight/score#uriGivenSf'
    NAME_PRE = u'http://dbepdia.org/spotlight/id/'

    def __init__(self, inpath, redirect_model_path, name_outpath, entity_outpath):
        self.in_path = inpath
        self.redirect_model_path = redirect_model_path
        self.name_out_path = name_outpath
        self.entity_out_path = entity_outpath

        log.info('Loading redirect model: %s ...', self.redirect_model_path)
        self.redirect_model = marshal.load(open(self.redirect_model_path, 'rb'))

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('inpath', metavar='INFILE')
        p.add_argument('redirect_model_path', metavar='REDIRECT_MODEL_PATH')
        p.add_argument('name_outpath', metavar='NAME_MODEL_OUTFILE')
        p.add_argument('entity_outpath', metavar='ENTITY_MODEL_OUTFILE')
        p.set_defaults(cls=cls)
        return p

    def __call__(self):
        from .util.nqstream import NqStream
        from .util.ntstream import NtStream
        entity = Entity()
        names = Name()

        entity_count = 0

        with open(self.in_path, 'rb') as fh:
            stream = NqStream()
            for subject, predicate, obj, _ in stream.iter(fh):
                if predicate.toPython() == self.ENTITY_SCORE:
                    #print 'Entity subject: %s' % subject.toPython().encode('utf8')
                    title = self._to_title(subject.toPython())
                    score = obj.toPython()
                    entity.update(title, score)

                    entity_count += 1
                    if entity_count % 100000 == 0:
                        log.debug('Processed %i entities...' % entity_count)

                elif predicate.toPython() == self.NAME_SCORE:
                    #print 'Name subject: %s' % subject.toPython().encode('utf8')
                    title, name = self._to_name(subject.toPython())
                    score = obj.toPython()

                    if title  == 'Pisco' or title == 'Carnoustie' or title == 'Bengkulu':
                        log.debug('%s|#|%s|#|%s', score, title, name)
                    names.update(name, title, score)

        entity.write(self.entity_out_path)
        names.write(self.name_out_path)

    def _trim(self, uri, prefix):
        assert uri.startswith(prefix), 'not valid subject'
        return unquote(uri.encode('utf8')[len(prefix):]).decode('utf8')

    def _to_title(self, uri):
        "Return final component of uri path."
        title = self._trim(uri, self.ENTITY_PRE)
        return self.redirect_model.get(title, title)

    def _normalise(self, name):
        "Return normalised name."
        return ' '.join(name.split('_'))

    def _to_name(self, uri):
        "Return split and normalised title and name."
        title, name = self._trim(uri, self.NAME_PRE).split('---', 1)
        return self.redirect_model.get(title, title), self._normalise(name)

class BuildDbpediaLinks(object):
    "Build cooccurrence and outlink models from DBpedia pagelinks nt."
    ENTITY_PRE = u'http://dbpedia.org/resource/'

    def __init__(self, fname, redirect_model_path, outpath):
        self.in_path = fname
        self.out_path = outpath
        self.namespace_re = re.compile(NAMESPACE_RE, re.I)
        self.pseudonamespace_re = re.compile(PSEUDONAMESPACE_RE)

        log.info('Loading redirect model: %s ...', redirect_model_path)
        self.redirect_model = marshal.load(open(redirect_model_path,'rb'))

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('fname', metavar='INFILE')
        p.add_argument('redirect_model_path', metavar='REDIRECT_MODEL_PATH')
        p.add_argument('outpath', metavar='OUTFILE')
        p.set_defaults(cls=cls)
        return p

    def __call__(self):
        self.build().write(self.out_path)

    def build(self):
        log.info('Building DBpedia links model from file: %s' % self.in_path)
        links = Links()
        for i, (source, target) in enumerate(self.read()):
            if i % 500000 == 0: log.debug('Processed %i links...' % i)

            source = self.redirect_model.get(source, source)
            target = self.redirect_model.get(target, target)

            links.update(source, target)
        return links

    def read(self):
        o = bz2.BZ2File if self.in_path.endswith('bz2') else open
        fh = o(self.in_path)
        stream = NtStream()
        for subject, predicate, obj in stream.iter(fh):
            source = self._to_title(subject.toPython())
            if not self.is_article(source):
                continue
            target = self._to_title(obj.toPython())
            if not self.is_article(target):
                continue
            yield source, target

    def _to_title(self, uri):
        "Return final component of uri path."
        return self._trim(uri, self.ENTITY_PRE)

    def _trim(self, uri, prefix):
        assert uri.startswith(prefix), 'not valid subject'
        return unquote(uri.encode('utf8')[len(prefix):]).decode('utf8')

    def is_article(self, title):
        "Return true if page is an article."
        if title == '':
            return False # empty name
        if title.endswith(DISAMBIGUATION_CLOSE):
            return False # disambiguation page
        if self.namespace_re.match(title):
            return False # not main article namespace
        if self.pseudonamespace_re.match(title):
            return False # not main article  namespace
        return True

class BuildDbpediaRedirects(object):
    "Build redirect model from DBpedia (transitive) redirects nt."
    ENTITY_PRE = u'http://dbpedia.org/resource/'

    def __init__(self, fname, outpath):
        self.in_path = fname
        self.out_path = outpath
        self.namespace_re = re.compile(NAMESPACE_RE, re.I)
        self.pseudonamespace_re = re.compile(PSEUDONAMESPACE_RE)

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('fname', metavar='FILE')
        p.add_argument('outpath', metavar='OUTFILE')
        p.set_defaults(cls=cls)
        return p

    def __call__(self):
        redirects = Redirects()
        for source, target in self.read():
            # TODO check whether dbpedia titles need to be mapped to wikipedia
            # TODO applies to BuildLexicalisations as well
            redirects.update(source, target)

        redirects.write(self.out_path)

    @property
    def fh(self):
        if self.in_path.endswith('bz2'):
            #return bz2.BZ2File(self.in_path)
            # TODO workaround until bz2 multistream update
            return os.popen('bzip2 -cd {}'.format(self.in_path))
        else:
            return open(self.in_path, 'rb')

    def read(self):
        stream = NtStream()
        for subject, _, obj in stream.iter(self.fh):
            source = self._to_title(subject.toPython())
            target = self._to_title(obj.toPython())
            if not self.is_article(target):
                continue
            yield source, target

    def _to_title(self, uri):
        "Return final component of uri path."
        return self._trim(uri, self.ENTITY_PRE)

    def _trim(self, uri, prefix):
        assert uri.startswith(prefix), 'not valid subject'
        return unquote(uri.encode('utf8')[len(prefix):]).decode('utf8')

    def is_article(self, title):
        "Return true if page is an article."
        if title == '':
            return False # empty name
        if title.endswith(DISAMBIGUATION_CLOSE):
            return False # disambiguation page
        if self.namespace_re.match(title):
            return False # not main article namespace
        if self.pseudonamespace_re.match(title):
            return False # not main article  namespace
        return True
<commit_msg>Fixed import error for dbpedia derived models<commit_after>from ...model import model

import logging
log = logging.getLogger()

class BuildDbpediaLexicalisations(object):
    "Build entity and name models from DBpedia Spotlight lexicalisations file."
    ENTITY_SCORE = u'http://dbpedia.org/spotlight/score#uriProbability'
    ENTITY_PRE = u'http://dbpedia.org/resource/'
    NAME_SCORE = u'http://dbpedia.org/spotlight/score#uriGivenSf'
    NAME_PRE = u'http://dbepdia.org/spotlight/id/'

    def __init__(self, inpath, redirect_model_path, name_outpath, entity_outpath):
        self.in_path = inpath
        self.redirect_model_path = redirect_model_path
        self.name_out_path = name_outpath
        self.entity_out_path = entity_outpath

        log.info('Loading redirect model: %s ...', self.redirect_model_path)
        self.redirect_model = marshal.load(open(self.redirect_model_path, 'rb'))

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('inpath', metavar='INFILE')
        p.add_argument('redirect_model_path', metavar='REDIRECT_MODEL_PATH')
        p.add_argument('name_outpath', metavar='NAME_MODEL_OUTFILE')
        p.add_argument('entity_outpath', metavar='ENTITY_MODEL_OUTFILE')
        p.set_defaults(cls=cls)
        return p

    def __call__(self):
        from .util.nqstream import NqStream
        from .util.ntstream import NtStream
        entity = Entity()
        names = Name()

        entity_count = 0

        with open(self.in_path, 'rb') as fh:
            stream = NqStream()
            for subject, predicate, obj, _ in stream.iter(fh):
                if predicate.toPython() == self.ENTITY_SCORE:
                    #print 'Entity subject: %s' % subject.toPython().encode('utf8')
                    title = self._to_title(subject.toPython())
                    score = obj.toPython()
                    entity.update(title, score)

                    entity_count += 1
                    if entity_count % 100000 == 0:
                        log.debug('Processed %i entities...' % entity_count)

                elif predicate.toPython() == self.NAME_SCORE:
                    #print 'Name subject: %s' % subject.toPython().encode('utf8')
                    title, name = self._to_name(subject.toPython())
                    score = obj.toPython()

                    if title  == 'Pisco' or title == 'Carnoustie' or title == 'Bengkulu':
                        log.debug('%s|#|%s|#|%s', score, title, name)
                    names.update(name, title, score)

        entity.write(self.entity_out_path)
        names.write(self.name_out_path)

    def _trim(self, uri, prefix):
        assert uri.startswith(prefix), 'not valid subject'
        return unquote(uri.encode('utf8')[len(prefix):]).decode('utf8')

    def _to_title(self, uri):
        "Return final component of uri path."
        title = self._trim(uri, self.ENTITY_PRE)
        return self.redirect_model.get(title, title)

    def _normalise(self, name):
        "Return normalised name."
        return ' '.join(name.split('_'))

    def _to_name(self, uri):
        "Return split and normalised title and name."
        title, name = self._trim(uri, self.NAME_PRE).split('---', 1)
        return self.redirect_model.get(title, title), self._normalise(name)

class BuildDbpediaLinks(object):
    "Build cooccurrence and outlink models from DBpedia pagelinks nt."
    ENTITY_PRE = u'http://dbpedia.org/resource/'

    def __init__(self, fname, redirect_model_path, outpath):
        self.in_path = fname
        self.out_path = outpath
        self.namespace_re = re.compile(NAMESPACE_RE, re.I)
        self.pseudonamespace_re = re.compile(PSEUDONAMESPACE_RE)

        log.info('Loading redirect model: %s ...', redirect_model_path)
        self.redirect_model = marshal.load(open(redirect_model_path,'rb'))

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('fname', metavar='INFILE')
        p.add_argument('redirect_model_path', metavar='REDIRECT_MODEL_PATH')
        p.add_argument('outpath', metavar='OUTFILE')
        p.set_defaults(cls=cls)
        return p

    def __call__(self):
        self.build().write(self.out_path)

    def build(self):
        log.info('Building DBpedia links model from file: %s' % self.in_path)
        links = Links()
        for i, (source, target) in enumerate(self.read()):
            if i % 500000 == 0: log.debug('Processed %i links...' % i)

            source = self.redirect_model.get(source, source)
            target = self.redirect_model.get(target, target)

            links.update(source, target)
        return links

    def read(self):
        o = bz2.BZ2File if self.in_path.endswith('bz2') else open
        fh = o(self.in_path)
        stream = NtStream()
        for subject, predicate, obj in stream.iter(fh):
            source = self._to_title(subject.toPython())
            if not self.is_article(source):
                continue
            target = self._to_title(obj.toPython())
            if not self.is_article(target):
                continue
            yield source, target

    def _to_title(self, uri):
        "Return final component of uri path."
        return self._trim(uri, self.ENTITY_PRE)

    def _trim(self, uri, prefix):
        assert uri.startswith(prefix), 'not valid subject'
        return unquote(uri.encode('utf8')[len(prefix):]).decode('utf8')

    def is_article(self, title):
        "Return true if page is an article."
        if title == '':
            return False # empty name
        if title.endswith(DISAMBIGUATION_CLOSE):
            return False # disambiguation page
        if self.namespace_re.match(title):
            return False # not main article namespace
        if self.pseudonamespace_re.match(title):
            return False # not main article  namespace
        return True

class BuildDbpediaRedirects(object):
    "Build redirect model from DBpedia (transitive) redirects nt."
    ENTITY_PRE = u'http://dbpedia.org/resource/'

    def __init__(self, fname, outpath):
        self.in_path = fname
        self.out_path = outpath
        self.namespace_re = re.compile(NAMESPACE_RE, re.I)
        self.pseudonamespace_re = re.compile(PSEUDONAMESPACE_RE)

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('fname', metavar='FILE')
        p.add_argument('outpath', metavar='OUTFILE')
        p.set_defaults(cls=cls)
        return p

    def __call__(self):
        redirects = Redirects()
        for source, target in self.read():
            # TODO check whether dbpedia titles need to be mapped to wikipedia
            # TODO applies to BuildLexicalisations as well
            redirects.update(source, target)

        redirects.write(self.out_path)

    @property
    def fh(self):
        if self.in_path.endswith('bz2'):
            #return bz2.BZ2File(self.in_path)
            # TODO workaround until bz2 multistream update
            return os.popen('bzip2 -cd {}'.format(self.in_path))
        else:
            return open(self.in_path, 'rb')

    def read(self):
        stream = NtStream()
        for subject, _, obj in stream.iter(self.fh):
            source = self._to_title(subject.toPython())
            target = self._to_title(obj.toPython())
            if not self.is_article(target):
                continue
            yield source, target

    def _to_title(self, uri):
        "Return final component of uri path."
        return self._trim(uri, self.ENTITY_PRE)

    def _trim(self, uri, prefix):
        assert uri.startswith(prefix), 'not valid subject'
        return unquote(uri.encode('utf8')[len(prefix):]).decode('utf8')

    def is_article(self, title):
        "Return true if page is an article."
        if title == '':
            return False # empty name
        if title.endswith(DISAMBIGUATION_CLOSE):
            return False # disambiguation page
        if self.namespace_re.match(title):
            return False # not main article namespace
        if self.pseudonamespace_re.match(title):
            return False # not main article  namespace
        return True
<|endoftext|>


<commit_before>__version__ = "0.4.12-dev"
<commit_msg>Release v0.4.12<commit_after>__version__ = "0.4.12"
<|endoftext|>