<commit_before>import numpy as np
import scipy.special as sps
from scipy.misc import comb
from scipy.special import gammaln
from random import sample
from itertools import combinations_with_replacement as CwR

from gameanalysis import rsgame
import sys

_TINY = float(np.finfo(np.float64).tiny)

class Sym_AGG_FNA(rsgame.BaseGame):
    """Action Graph Game with Function Nodes.

    Represented games are symmetric. Action node utilities have additive structure.
    Function nodes are contribution-independent. Function nodes have in-edges only
    from action nodes.
    """
    def __init__(self, num_players, num_strategies, action_weights, function_inputs=[],
                 node_functions=[]):
        """
        Parameters
        ----------
            num_players
            num_strategies
            action_weights: floating point matrix with num_strategies rows and
                    (num_strategies + |node_functions|) columns. Each entry specifies the
                    incoming weight in the action graph for the action node (row).
            function_inputs: boolean matrix with |node_functions| rows and num_strategies
                    columns. Each entry specifies whether the action node (column) is an
                    input to the function node (row).
            node_functions: Activation functions for each function node.
                    This can either be a list of python functions that work
                    correctly when applied to a vector of inputs, or an array
                    already storing the correct tabular representation.
        Class Variables
        ---------------
            self.action_weights
            self.function_inputs
            self.configs
            self.log_dev_reps
            self.func_table
        """
        super().__init__(num_players, num_strategies)
        self.action_weights = np.array(action_weights, dtype=float)
        self.function_inputs = np.array(function_inputs, dtype=bool)
        self.configs = np.arange(num_players+1)[:,None]
        self.log_dev_reps = gammaln(num_players) - gammaln(self.configs[1:]) - \
                            gammaln(num_players - self.configs[:-1])
        if isinstance(node_functions, np.ndarray): # already a table
            self.func_table = node_functions
        else: # list of python functions, need to be applied to configurations
            self.func_table = np.array([f(self.configs[:,0]) for f in
                                        node_functions], dtype=float).T
        self.num_funcs = self.func_table.shape[1]
        self.num_nodes = self.num_funcs + self.num_strategies[0]
        self._min_payoffs = None
        self._max_payoffs = None

    @staticmethod
    def from_json(json_):
        """
        Build a game from the information stored in a dictionary in the json
        format
        """
        num_players = json_["num_players"]
        strategy_names = sorted(json_["strategy_names"])
        num_strats = len(strategy_names)
        function_names = sorted(json_["function_names"])
        num_funcs = len(function_names)
        functions = np.array([json_["function_tables"][f] for
                              f in function_names]).T

        action_weights = np.empty([num_strats, num_strats + num_funcs],
                                  dtype=float)
        function_inputs = np.zeros([num_strats, num_funcs], dtype=bool)

        for s,strat in enumerate(strategy_names):
            for n,node in enumerate(strategy_names + function_names):
                action_weights[s][n] = json_["action_weights"][strat].get(node,0)
            for f,func in enumerate(function_names):
                if strat in json_["function_inputs"][func]:
                    function_inputs[s][f] = True
        return Sym_AGG_FNA(num_players, num_strats, action_weights,
                           function_inputs, functions)

    def min_payoffs(self):
        """Returns a lower bound on the payoffs."""
        if self._min_payoffs is None:
            minima = np.zeros([self.num_strategies[0], self.num_nodes])
            minima[:,-self.num_funcs:] = self.func_table.min(0)
            minima[self.action_weights <= 0] = 0

            maxima = np.zeros(minima.shape)
            maxima[:,-self.num_funcs:] = self.func_table.max(0)
            maxima[self.action_weights >= 0] = 0

            self._min_payoffs = ((minima + maxima) *
                                 self.action_weights).sum(1).min(keepdims=True)
            self._min_payoffs.setflags(write=False)
        return self._min_payoffs.view()

    def max_payoffs(self):
        """Returns an upper bound on the payoffs."""
        if self._max_payoffs is None:
            minima = np.zeros([self.num_strategies[0], self.num_nodes])
            minima[:,-self.num_funcs:] = self.func_table.min(0)
            minima[self.action_weights >= 0] = 0

            maxima = np.zeros(minima.shape)
            maxima[:,-self.num_funcs:] = self.func_table.max(0)
            maxima[self.action_weights <= 0] = 0

            self._max_payoffs = ((minima + maxima) *
                                 self.action_weights).sum(1).min(keepdims=True)
            self._max_payoffs.setflags(write=False)
        return self._max_payoffs.view()

    def deviation_payoffs(self, mix, assume_complete=True, jacobian=False):
        # TODO To add jacobian support.
        assert not jacobian, "Sym_AGG_FNA doesn't support jacobian"
        func_node_probs = mix[:,None].repeat(self.num_funcs, 1)
        func_node_probs[np.logical_not(self.function_inputs)] = 0
        func_node_probs = func_node_probs.sum(0)

        act_conf_probs = np.exp(np.log(mix + _TINY) * self.configs[:-1] +
                  np.log(1 - mix + _TINY) * (self.num_players -
                  self.configs[1:]) + self.log_dev_reps)
        func_conf_probs = np.exp(np.log(func_node_probs + _TINY) *
                  self.configs[:-1] + np.log(1 - func_node_probs + _TINY) *
                  (self.num_players - self.configs[1:]))

        EVs = np.empty(self.num_strategies)
        for s in range(self.num_strategies[0]):
            action_outputs = self.configs[:-1].repeat(self.num_strategies, 1)
            action_outputs[:,s] += 1
            function_outputs = np.array(self.func_table[:-1])
            function_outputs[:,self.function_inputs[s]] = \
                            self.func_table[1:,self.function_inputs[s]]
            node_EVs = np.append((act_conf_probs * action_outputs).sum(0),
                                 (func_conf_probs * function_outputs).sum(0))
            EVs[s] = np.dot(node_EVs, self.action_weights[s])
        return EVs

    def to_json(self):
        """
        Creates a json format of the game for storage
        """
        raise NotImplementedError("not yet compatible with array implementation.")
        json = {}
        json['players'] = self.players['All']
        json['strategies'] = list(self.strategies['All'])
        json['function_nodes'] = self.function_nodes
        action_graph = {s:list(self.neighbors[s].keys()) \
                for s in self.neighbors}
        json['action_graph'] = action_graph
        json['utilities'] = self.utilities
        json['functions'] = self.functions
        return json

    @staticmethod
    def randomAGG(num_players, num_strats, num_FNs, D_min=0, D_max=-1,
                  w_mean=0, w_var=3):
        """
        D_min: minimum in-degree for an action node
        D_max: maximum in-degree for an action node
        w_mean: mean of weights
        w_var: variance of weights
        """
        raise NotImplementedError("not yet compatible with array implementation.")
        if D_min < 0 or D_min >= num_strats:
            D_min = 0
        if D_max < 0 or D_max >= num_strats:
            D_max = num_strats / 2

        # This maps a function to the (mean, var) tuple for params
        func = {
                'quadratic': ([0,0,0],[tiny,2,1]),
                'linear': ([0,0],[2,2])
        }

        strategies = ["s"+str(i) for i in range(num_strats)]
        FNs = ["p"+str(i) for i in range(num_FNs)]
        nodes = strategies + FNs
        action_graph = {}
        utilities = {}
        functions = {}

        # Connect the function nodes first
        for fn in FNs:
            num_neighbors = np.random.randint(num_strats/2, num_strats)
            neighbors = sorted(sample(strategies, num_neighbors))
            action_graph[fn] = neighbors
            func_type = sample(func.keys(),1)[0]
            param = [np.random.normal(m,v) for m,v in zip(*func[func_type])]
            functions[fn] = [func_type, tuple(param)]

        for s, strat in enumerate(strategies):
            num_neighbors = np.random.randint(D_min,D_max+1)
            neighbors = sorted(sample(strategies[:s] + strategies[s+1:] + FNs,\
                                      num_neighbors) + [strat])
            action_graph[strat] = neighbors
            u = [np.random.normal(w_mean,w_var) for neighbor in neighbors]
            utilities[strat] = np.array(u)

        return Sym_AGG_FNA(num_players, strategies, FNs, action_graph,
                           utilities, functions)

    def pure_payoff(self, strat, profile):
        """
        Returns the payoff to the given pure strategy profile
        Input:
            strat: strategy
            prof: profile
        Output:
            The payoff
        """
        raise NotImplementedError("not yet compatible with array implementation.")
        payoff = 0
        for s,i in self.neighbors[strat].items():
            if s in self.strategies['All']:
                count = profile[sorted(self.strategies['All']).index(s)]
                payoff += self.utilities[strat][i] * count
            if s in self.function_nodes:
                count = 0
                for n in self.neighbors[s]:
                    count += profile[ sorted(self.strategies['All']).index(n) ]
                payoff += self.utilities[strat][i] * self.func_table[s][count]

        return payoff

    def to_rsgame(self):
        """
        This method builds an rsgame object that represent the same
        game. There will only be one role "All" in the constructed
        rsgame
        """
        raise NotImplementedError("not yet compatible with array implementation.")
        aprofiles = []
        apayoffs = []
        for profile in CwR(self.strategies['All'],self.players['All']):
            count = np.array(
                    [profile.count(n) for n in sorted(self.strategies['All'])])
            u = np.array(
                [self.pure_payoff(n, count) for n in sorted(self.strategies['All'])])
            nz = np.nonzero(count)
            mask = np.zeros(count.shape)
            mask[nz] = 1
            u = u * mask
            aprofiles.append(count)
            apayoffs.append(u)
        aprofiles = np.array(aprofiles)
        apayoffs = np.array(apayoffs)
        return rsgame.Game(self.players, self.strategies,
                           aprofiles, apayoffs)
<commit_msg>max_payoffs computes a max not a min<commit_after>import numpy as np
import scipy.special as sps
from scipy.misc import comb
from scipy.special import gammaln
from random import sample
from itertools import combinations_with_replacement as CwR

from gameanalysis import rsgame
import sys

_TINY = float(np.finfo(np.float64).tiny)

class Sym_AGG_FNA(rsgame.BaseGame):
    """Action Graph Game with Function Nodes.

    Represented games are symmetric. Action node utilities have additive structure.
    Function nodes are contribution-independent. Function nodes have in-edges only
    from action nodes.
    """
    def __init__(self, num_players, num_strategies, action_weights, function_inputs=[],
                 node_functions=[]):
        """
        Parameters
        ----------
            num_players
            num_strategies
            action_weights: floating point matrix with num_strategies rows and
                    (num_strategies + |node_functions|) columns. Each entry specifies the
                    incoming weight in the action graph for the action node (row).
            function_inputs: boolean matrix with |node_functions| rows and num_strategies
                    columns. Each entry specifies whether the action node (column) is an
                    input to the function node (row).
            node_functions: Activation functions for each function node.
                    This can either be a list of python functions that work
                    correctly when applied to a vector of inputs, or an array
                    already storing the correct tabular representation.
        Class Variables
        ---------------
            self.action_weights
            self.function_inputs
            self.configs
            self.log_dev_reps
            self.func_table
        """
        super().__init__(num_players, num_strategies)
        self.action_weights = np.array(action_weights, dtype=float)
        self.function_inputs = np.array(function_inputs, dtype=bool)
        self.configs = np.arange(num_players+1)[:,None]
        self.log_dev_reps = gammaln(num_players) - gammaln(self.configs[1:]) - \
                            gammaln(num_players - self.configs[:-1])
        if isinstance(node_functions, np.ndarray): # already a table
            self.func_table = node_functions
        else: # list of python functions, need to be applied to configurations
            self.func_table = np.array([f(self.configs[:,0]) for f in
                                        node_functions], dtype=float).T
        self.num_funcs = self.func_table.shape[1]
        self.num_nodes = self.num_funcs + self.num_strategies[0]
        self._min_payoffs = None
        self._max_payoffs = None

    @staticmethod
    def from_json(json_):
        """
        Build a game from the information stored in a dictionary in the json
        format
        """
        num_players = json_["num_players"]
        strategy_names = sorted(json_["strategy_names"])
        num_strats = len(strategy_names)
        function_names = sorted(json_["function_names"])
        num_funcs = len(function_names)
        functions = np.array([json_["function_tables"][f] for
                              f in function_names]).T

        action_weights = np.empty([num_strats, num_strats + num_funcs],
                                  dtype=float)
        function_inputs = np.zeros([num_strats, num_funcs], dtype=bool)

        for s,strat in enumerate(strategy_names):
            for n,node in enumerate(strategy_names + function_names):
                action_weights[s][n] = json_["action_weights"][strat].get(node,0)
            for f,func in enumerate(function_names):
                if strat in json_["function_inputs"][func]:
                    function_inputs[s][f] = True
        return Sym_AGG_FNA(num_players, num_strats, action_weights,
                           function_inputs, functions)

    def min_payoffs(self):
        """Returns a lower bound on the payoffs."""
        if self._min_payoffs is None:
            minima = np.zeros([self.num_strategies[0], self.num_nodes])
            minima[:,-self.num_funcs:] = self.func_table.min(0)
            minima[self.action_weights <= 0] = 0

            maxima = np.zeros(minima.shape)
            maxima[:,-self.num_funcs:] = self.func_table.max(0)
            maxima[self.action_weights >= 0] = 0

            self._min_payoffs = ((minima + maxima) *
                                 self.action_weights).sum(1).min(keepdims=True)
            self._min_payoffs.setflags(write=False)
        return self._min_payoffs.view()

    def max_payoffs(self):
        """Returns an upper bound on the payoffs."""
        if self._max_payoffs is None:
            minima = np.zeros([self.num_strategies[0], self.num_nodes])
            minima[:,-self.num_funcs:] = self.func_table.min(0)
            minima[self.action_weights >= 0] = 0

            maxima = np.zeros(minima.shape)
            maxima[:,-self.num_funcs:] = self.func_table.max(0)
            maxima[self.action_weights <= 0] = 0

            self._max_payoffs = ((minima + maxima) *
                                 self.action_weights).sum(1).max(keepdims=True)
            self._max_payoffs.setflags(write=False)
        return self._max_payoffs.view()

    def deviation_payoffs(self, mix, assume_complete=True, jacobian=False):
        # TODO To add jacobian support.
        assert not jacobian, "Sym_AGG_FNA doesn't support jacobian"
        func_node_probs = mix[:,None].repeat(self.num_funcs, 1)
        func_node_probs[np.logical_not(self.function_inputs)] = 0
        func_node_probs = func_node_probs.sum(0)

        act_conf_probs = np.exp(np.log(mix + _TINY) * self.configs[:-1] +
                  np.log(1 - mix + _TINY) * (self.num_players -
                  self.configs[1:]) + self.log_dev_reps)
        func_conf_probs = np.exp(np.log(func_node_probs + _TINY) *
                  self.configs[:-1] + np.log(1 - func_node_probs + _TINY) *
                  (self.num_players - self.configs[1:]))

        EVs = np.empty(self.num_strategies)
        for s in range(self.num_strategies[0]):
            action_outputs = self.configs[:-1].repeat(self.num_strategies, 1)
            action_outputs[:,s] += 1
            function_outputs = np.array(self.func_table[:-1])
            function_outputs[:,self.function_inputs[s]] = \
                            self.func_table[1:,self.function_inputs[s]]
            node_EVs = np.append((act_conf_probs * action_outputs).sum(0),
                                 (func_conf_probs * function_outputs).sum(0))
            EVs[s] = np.dot(node_EVs, self.action_weights[s])
        return EVs

    def to_json(self):
        """
        Creates a json format of the game for storage
        """
        raise NotImplementedError("not yet compatible with array implementation.")
        json = {}
        json['players'] = self.players['All']
        json['strategies'] = list(self.strategies['All'])
        json['function_nodes'] = self.function_nodes
        action_graph = {s:list(self.neighbors[s].keys()) \
                for s in self.neighbors}
        json['action_graph'] = action_graph
        json['utilities'] = self.utilities
        json['functions'] = self.functions
        return json

    @staticmethod
    def randomAGG(num_players, num_strats, num_FNs, D_min=0, D_max=-1,
                  w_mean=0, w_var=3):
        """
        D_min: minimum in-degree for an action node
        D_max: maximum in-degree for an action node
        w_mean: mean of weights
        w_var: variance of weights
        """
        raise NotImplementedError("not yet compatible with array implementation.")
        if D_min < 0 or D_min >= num_strats:
            D_min = 0
        if D_max < 0 or D_max >= num_strats:
            D_max = num_strats / 2

        # This maps a function to the (mean, var) tuple for params
        func = {
                'quadratic': ([0,0,0],[tiny,2,1]),
                'linear': ([0,0],[2,2])
        }

        strategies = ["s"+str(i) for i in range(num_strats)]
        FNs = ["p"+str(i) for i in range(num_FNs)]
        nodes = strategies + FNs
        action_graph = {}
        utilities = {}
        functions = {}

        # Connect the function nodes first
        for fn in FNs:
            num_neighbors = np.random.randint(num_strats/2, num_strats)
            neighbors = sorted(sample(strategies, num_neighbors))
            action_graph[fn] = neighbors
            func_type = sample(func.keys(),1)[0]
            param = [np.random.normal(m,v) for m,v in zip(*func[func_type])]
            functions[fn] = [func_type, tuple(param)]

        for s, strat in enumerate(strategies):
            num_neighbors = np.random.randint(D_min,D_max+1)
            neighbors = sorted(sample(strategies[:s] + strategies[s+1:] + FNs,\
                                      num_neighbors) + [strat])
            action_graph[strat] = neighbors
            u = [np.random.normal(w_mean,w_var) for neighbor in neighbors]
            utilities[strat] = np.array(u)

        return Sym_AGG_FNA(num_players, strategies, FNs, action_graph,
                           utilities, functions)

    def pure_payoff(self, strat, profile):
        """
        Returns the payoff to the given pure strategy profile
        Input:
            strat: strategy
            prof: profile
        Output:
            The payoff
        """
        raise NotImplementedError("not yet compatible with array implementation.")
        payoff = 0
        for s,i in self.neighbors[strat].items():
            if s in self.strategies['All']:
                count = profile[sorted(self.strategies['All']).index(s)]
                payoff += self.utilities[strat][i] * count
            if s in self.function_nodes:
                count = 0
                for n in self.neighbors[s]:
                    count += profile[ sorted(self.strategies['All']).index(n) ]
                payoff += self.utilities[strat][i] * self.func_table[s][count]

        return payoff

    def to_rsgame(self):
        """
        This method builds an rsgame object that represent the same
        game. There will only be one role "All" in the constructed
        rsgame
        """
        raise NotImplementedError("not yet compatible with array implementation.")
        aprofiles = []
        apayoffs = []
        for profile in CwR(self.strategies['All'],self.players['All']):
            count = np.array(
                    [profile.count(n) for n in sorted(self.strategies['All'])])
            u = np.array(
                [self.pure_payoff(n, count) for n in sorted(self.strategies['All'])])
            nz = np.nonzero(count)
            mask = np.zeros(count.shape)
            mask[nz] = 1
            u = u * mask
            aprofiles.append(count)
            apayoffs.append(u)
        aprofiles = np.array(aprofiles)
        apayoffs = np.array(apayoffs)
        return rsgame.Game(self.players, self.strategies,
                           aprofiles, apayoffs)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals

import logging
import shutil

import six
import texttable

from compose.cli import colors

if hasattr(shutil, "get_terminal_size"):
    from shutil import get_terminal_size
else:
    from backports.shutil_get_terminal_size import get_terminal_size

def get_tty_width():
    try:
        width, _ = get_terminal_size()
        return int(width)
    except OSError:
        return 0

class Formatter:
    """Format tabular data for printing."""

    @staticmethod
    def table(headers, rows):
        table = texttable.Texttable(max_width=get_tty_width())
        table.set_cols_dtype(['t' for h in headers])
        table.add_rows([headers] + rows)
        table.set_deco(table.HEADER)
        table.set_chars(['-', '|', '+', '-'])

        return table.draw()

class ConsoleWarningFormatter(logging.Formatter):
    """A logging.Formatter which prints WARNING and ERROR messages with
    a prefix of the log level colored appropriate for the log level.
    """

    def get_level_message(self, record):
        separator = ': '
        if record.levelno == logging.WARNING:
            return colors.yellow(record.levelname) + separator
        if record.levelno == logging.ERROR:
            return colors.red(record.levelname) + separator

        return ''

    def format(self, record):
        if isinstance(record.msg, six.binary_type):
            record.msg = record.msg.decode('utf-8')
        message = super(ConsoleWarningFormatter, self).format(record)
        return '{0}{1}'.format(self.get_level_message(record), message)
<commit_msg>Assume infinite terminal width when not running in a terminal<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

import logging
import shutil

import six
import texttable

from compose.cli import colors

if hasattr(shutil, "get_terminal_size"):
    from shutil import get_terminal_size
else:
    from backports.shutil_get_terminal_size import get_terminal_size

def get_tty_width():
    try:
        # get_terminal_size can't determine the size if compose is piped
        # to another command. But in such case it doesn't make sense to
        # try format the output by terminal size as this output is consumed
        # by another command. So let's pretend we have a huge terminal so
        # output is single-lined
        width, _ = get_terminal_size(fallback=(999, 0))
        return int(width)
    except OSError:
        return 0

class Formatter:
    """Format tabular data for printing."""

    @staticmethod
    def table(headers, rows):
        table = texttable.Texttable(max_width=get_tty_width())
        table.set_cols_dtype(['t' for h in headers])
        table.add_rows([headers] + rows)
        table.set_deco(table.HEADER)
        table.set_chars(['-', '|', '+', '-'])

        return table.draw()

class ConsoleWarningFormatter(logging.Formatter):
    """A logging.Formatter which prints WARNING and ERROR messages with
    a prefix of the log level colored appropriate for the log level.
    """

    def get_level_message(self, record):
        separator = ': '
        if record.levelno == logging.WARNING:
            return colors.yellow(record.levelname) + separator
        if record.levelno == logging.ERROR:
            return colors.red(record.levelname) + separator

        return ''

    def format(self, record):
        if isinstance(record.msg, six.binary_type):
            record.msg = record.msg.decode('utf-8')
        message = super(ConsoleWarningFormatter, self).format(record)
        return '{0}{1}'.format(self.get_level_message(record), message)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import cv2

import rospy
import rospkg
from std_msgs.msg      import String, Header
from sensor_msgs.msg   import Image
from geometry_msgs.msg import Point, Pose
from sensor_msgs.msg   import PointCloud
from cv_bridge         import CvBridge, CvBridgeError

from RecursiveStereoPackage.RecursiveStereo import RecursiveStereo

class RecursiveStereoNode:
    def __init__(self):
        self.verbose          = rospy.get_param('/recursivestereo/parameters/verbose', False)
        self.rospack          = rospkg.RosPack() # get an instance of RosPack with the default search paths
        self.subscriber_left  = rospy.Subscriber('/airsim/left/image_raw',  Image, self.CallbackLeft,  queue_size = 1)
        self.subscriber_right = rospy.Subscriber('/airsim/right/image_raw', Image, self.CallbackRight, queue_size = 1)
        self.subscriber_pose  = rospy.Subscriber('/airsim/pose',            Pose,  self.CallbackPose,  queue_size = 1)
        self.publisher        = rospy.Publisher('/airsim/pointcloud', PointCloud, queue_size = 1)
        self.cv_bridge        = CvBridge()
        self.timestamp_left   = None
        self.timestamp_right  = None

        # Recursive Stereo
        self.algorithm = RecursiveStereo()

        # Set parameters
        self.algorithm.c_u = rospy.get_param('/recursivestereo/parameters/c_u', 609.5593)
        self.algorithm.c_v = rospy.get_param('/recursivestereo/parameters/c_v', 172.8840)
        self.algorithm.b   = rospy.get_param('/recursivestereo/parameters/b',     0.5572)
        self.algorithm.f   = rospy.get_param('/recursivestereo/parameters/f',   721.5577)
        self.algorithm.blockmatching_blocksize           = rospy.get_param('/recursivestereo/parameters/blockmatching_blocksize',            10)
        self.algorithm.blockmatching_window_size         = rospy.get_param('/recursivestereo/parameters/blockmatching_window_size',           9)
        self.algorithm.blockmatching_minimum_disparities = rospy.get_param('/recursivestereo/parameters/blockmatching_minimum_disparities',   1)
        self.algorithm.blockmatching_maximum_disparities = rospy.get_param('/recursivestereo/parameters/blockmatching_maximum_disparities',  65)
        self.algorithm.export_pcl       = False
        self.algorithm.enable_recursive = True

    def VerbosePrint(self, string):
        if self.verbose == True:
            print(string)

    def CallbackLeft(self, image):
        cv_image_left = None
        try:
            cv_image_left = self.cv_bridge.imgmsg_to_cv2(image, "bgr8")
        except CvBridgeError as e:
            self.VerbosePrint(e)
        self.algorithm.left_image  = cv_image_left
        self.algorithm.color_image = cv_image_left
        self.timestamp_left        = image.header.stamp.sec*1e9 + image.header.stamp.nsec # ns
        self.CallbackCalculate()

    def CallbackRight(self, image):
        cv_image_right = None
        try:
            cv_image_right = self.cv_bridge.imgmsg_to_cv2(image, "bgr8")
        except CvBridgeError as e:
            self.VerbosePrint(e)
        self.algorithm.right_image  = cv_image_right
        self.timestamp_right        = image.header.stamp.sec*1e9 + image.header.stamp.nsec # ns
        self.CallbackCalculate()

    def CallbackPose(self, new_pose):
        # Save last pose
        self.algorithm.last_pose = self.algorithm.pose
        # Set new pose
        self.algorithm.pose['position']['x']     = new_pose.position.x
        self.algorithm.pose['position']['y']     = new_pose.position.y
        self.algorithm.pose['position']['z']     = new_pose.position.z
        self.algorithm.pose['orientation']['x']  = new_pose.orientation.x
        self.algorithm.pose['orientation']['y']  = new_pose.orientation.y
        self.algorithm.pose['orientation']['z']  = new_pose.orientation.z
        self.algorithm.pose['orientation']['w']  = new_pose.orientation.w

    def CallbackCalculate(self):
        if ((self.timestamp_left is None) or (self.timestamp_right is None)):
            rospy.loginfo("Timestamp(s) missing")
            return
        if self.timestamp_left == self.timestamp_right:
            self.algorithm.pcl_filename = 'airsim_pcl_time_{}.ply'.format(self.timestamp_left)
            self.algorithm.Step()
            pointcloud = PointCloud()
            self.VerbosePrint('Calculating')
            for i in range(len(self.algorithm.pcl)):
                 pointcloud.points.append(Point(self.algorithm.pcl[i,0],
                                                self.algorithm.pcl[i,1],
                                                self.algorithm.pcl[i,2]))
            self.publisher.publish(pointcloud)
            rospy.loginfo("Sending PointCloud")
        else:
            self.VerbosePrint('Timestamp of left image and rigt image do not match (yet)')
            self.VerbosePrint('   T(Left)  = {}'.format(self.timestamp_left))
            self.VerbosePrint('   T(Right) = {}'.format(self.timestamp_right))

if __name__ == '__main__':
    rospy.init_node('recursive_stereo_node', anonymous=True)
    node = RecursiveStereoNode()
    try:
        rospy.spin()
    except KeyboardInterrupt:
        self.VerbosePrint("Shutting down")
    cv2.destroyAllWindows()
<commit_msg>corrected rospy timestamp names to secs and nsecs<commit_after>#!/usr/bin/env python
import cv2

import rospy
import rospkg
from std_msgs.msg      import String, Header
from sensor_msgs.msg   import Image
from geometry_msgs.msg import Point, Pose
from sensor_msgs.msg   import PointCloud
from cv_bridge         import CvBridge, CvBridgeError

from RecursiveStereoPackage.RecursiveStereo import RecursiveStereo

class RecursiveStereoNode:
    def __init__(self):
        self.verbose          = rospy.get_param('/recursivestereo/parameters/verbose', False)
        self.rospack          = rospkg.RosPack() # get an instance of RosPack with the default search paths
        self.subscriber_left  = rospy.Subscriber('/airsim/left/image_raw',  Image, self.CallbackLeft,  queue_size = 1)
        self.subscriber_right = rospy.Subscriber('/airsim/right/image_raw', Image, self.CallbackRight, queue_size = 1)
        self.subscriber_pose  = rospy.Subscriber('/airsim/pose',            Pose,  self.CallbackPose,  queue_size = 1)
        self.publisher        = rospy.Publisher('/airsim/pointcloud', PointCloud, queue_size = 1)
        self.cv_bridge        = CvBridge()
        self.timestamp_left   = None
        self.timestamp_right  = None

        # Recursive Stereo
        self.algorithm = RecursiveStereo()

        # Set parameters
        self.algorithm.c_u = rospy.get_param('/recursivestereo/parameters/c_u', 609.5593)
        self.algorithm.c_v = rospy.get_param('/recursivestereo/parameters/c_v', 172.8840)
        self.algorithm.b   = rospy.get_param('/recursivestereo/parameters/b',     0.5572)
        self.algorithm.f   = rospy.get_param('/recursivestereo/parameters/f',   721.5577)
        self.algorithm.blockmatching_blocksize           = rospy.get_param('/recursivestereo/parameters/blockmatching_blocksize',            10)
        self.algorithm.blockmatching_window_size         = rospy.get_param('/recursivestereo/parameters/blockmatching_window_size',           9)
        self.algorithm.blockmatching_minimum_disparities = rospy.get_param('/recursivestereo/parameters/blockmatching_minimum_disparities',   1)
        self.algorithm.blockmatching_maximum_disparities = rospy.get_param('/recursivestereo/parameters/blockmatching_maximum_disparities',  65)
        self.algorithm.export_pcl       = False
        self.algorithm.enable_recursive = True

    def VerbosePrint(self, string):
        if self.verbose == True:
            print(string)

    def CallbackLeft(self, image):
        cv_image_left = None
        try:
            cv_image_left = self.cv_bridge.imgmsg_to_cv2(image, "bgr8")
        except CvBridgeError as e:
            self.VerbosePrint(e)
        self.algorithm.left_image  = cv_image_left
        self.algorithm.color_image = cv_image_left
        self.timestamp_left        = image.header.stamp.secs*1e9 + image.header.stamp.nsecs # ns
        self.CallbackCalculate()

    def CallbackRight(self, image):
        cv_image_right = None
        try:
            cv_image_right = self.cv_bridge.imgmsg_to_cv2(image, "bgr8")
        except CvBridgeError as e:
            self.VerbosePrint(e)
        self.algorithm.right_image  = cv_image_right
        self.timestamp_right        = image.header.stamp.secs*1e9 + image.header.stamp.nsecs # ns
        self.CallbackCalculate()

    def CallbackPose(self, new_pose):
        # Save last pose
        self.algorithm.last_pose = self.algorithm.pose
        # Set new pose
        self.algorithm.pose['position']['x']     = new_pose.position.x
        self.algorithm.pose['position']['y']     = new_pose.position.y
        self.algorithm.pose['position']['z']     = new_pose.position.z
        self.algorithm.pose['orientation']['x']  = new_pose.orientation.x
        self.algorithm.pose['orientation']['y']  = new_pose.orientation.y
        self.algorithm.pose['orientation']['z']  = new_pose.orientation.z
        self.algorithm.pose['orientation']['w']  = new_pose.orientation.w

    def CallbackCalculate(self):
        if ((self.timestamp_left is None) or (self.timestamp_right is None)):
            rospy.loginfo("Timestamp(s) missing")
            return
        if self.timestamp_left == self.timestamp_right:
            self.algorithm.pcl_filename = 'airsim_pcl_time_{}.ply'.format(self.timestamp_left)
            self.algorithm.Step()
            pointcloud = PointCloud()
            self.VerbosePrint('Calculating')
            for i in range(len(self.algorithm.pcl)):
                 pointcloud.points.append(Point(self.algorithm.pcl[i,0],
                                                self.algorithm.pcl[i,1],
                                                self.algorithm.pcl[i,2]))
            self.publisher.publish(pointcloud)
            rospy.loginfo("Sending PointCloud")
        else:
            self.VerbosePrint('Timestamp of left image and rigt image do not match (yet)')
            self.VerbosePrint('   T(Left)  = {}'.format(self.timestamp_left))
            self.VerbosePrint('   T(Right) = {}'.format(self.timestamp_right))

if __name__ == '__main__':
    rospy.init_node('recursive_stereo_node', anonymous=True)
    node = RecursiveStereoNode()
    try:
        rospy.spin()
    except KeyboardInterrupt:
        self.VerbosePrint("Shutting down")
    cv2.destroyAllWindows()
<|endoftext|>


<commit_before># coding: utf-8
"""A simple configuration system.

Authors
-------
* Brian Granger
* Fernando Perez
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2009  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import __builtin__
import os
import sys

from IPython.external import argparse
from IPython.utils.path import filefind

#-----------------------------------------------------------------------------
# Exceptions
#-----------------------------------------------------------------------------

class ConfigError(Exception):
    pass

class ConfigLoaderError(ConfigError):
    pass

#-----------------------------------------------------------------------------
# Argparse fix
#-----------------------------------------------------------------------------

# Unfortunately argparse by default prints help messages to stderr instead of
# stdout.  This makes it annoying to capture long help screens at the command
# line, since one must know how to pipe stderr, which many users don't know how
# to do.  So we override the print_help method with one that defaults to
# stdout and use our class instead.

class ArgumentParser(argparse.ArgumentParser):
    """Simple argparse subclass that prints help to stdout by default."""

    def print_help(self, file=None):
        if file is None:
            file = sys.stdout
        return super(ArgumentParser, self).print_help(file)

    print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__

#-----------------------------------------------------------------------------
# Config class for holding config information
#-----------------------------------------------------------------------------

class Config(dict):
    """An attribute based dict that can do smart merges."""

    def __init__(self, *args, **kwds):
        dict.__init__(self, *args, **kwds)
        # This sets self.__dict__ = self, but it has to be done this way
        # because we are also overriding __setattr__.
        dict.__setattr__(self, '__dict__', self)

    def _merge(self, other):
        to_update = {}
        for k, v in other.items():
            if not self.has_key(k):
                to_update[k] = v
            else: # I have this key
                if isinstance(v, Config):
                    # Recursively merge common sub Configs
                    self[k]._merge(v)
                else:
                    # Plain updates for non-Configs
                    to_update[k] = v

        self.update(to_update)

    def _is_section_key(self, key):
        if key[0].upper()==key[0] and not key.startswith('_'):
            return True
        else:
            return False

    # .has_key is deprecated for dictionaries.
    def has_key(self, key):
        if self._is_section_key(key):
            return True
        else:
            return key in self

    def _has_section(self, key):
        if self._is_section_key(key):
            if key in self:
                return True
        return False

    def copy(self):
        return type(self)(dict.copy(self))

    def __copy__(self):
        return self.copy()

    def __deepcopy__(self, memo):
        import copy
        return type(self)(copy.deepcopy(self.items()))

    def __getitem__(self, key):
        # Because we use this for an exec namespace, we need to delegate
        # the lookup of names in __builtin__ to itself.  This means
        # that you can't have section or attribute names that are
        # builtins.
        try:
            return getattr(__builtin__, key)
        except AttributeError:
            pass
        if self._is_section_key(key):
            try:
                return dict.__getitem__(self, key)
            except KeyError:
                c = Config()
                dict.__setitem__(self, key, c)
                return c
        else:
            return dict.__getitem__(self, key)

    def __setitem__(self, key, value):
        # Don't allow names in __builtin__ to be modified.
        if hasattr(__builtin__, key):
            raise ConfigError('Config variable names cannot have the same name '
                              'as a Python builtin: %s' % key)
        if self._is_section_key(key):
            if not isinstance(value, Config):
                raise ValueError('values whose keys begin with an uppercase '
                                 'char must be Config instances: %r, %r' % (key, value))
        else:
            dict.__setitem__(self, key, value)

    def __getattr__(self, key):
        try:
            return self.__getitem__(key)
        except KeyError, e:
            raise AttributeError(e)

    def __setattr__(self, key, value):
        try:
            self.__setitem__(key, value)
        except KeyError, e:
            raise AttributeError(e)

    def __delattr__(self, key):
        try:
            dict.__delitem__(self, key)
        except KeyError, e:
            raise AttributeError(e)

#-----------------------------------------------------------------------------
# Config loading classes
#-----------------------------------------------------------------------------

class ConfigLoader(object):
    """A object for loading configurations from just about anywhere.

    The resulting configuration is packaged as a :class:`Struct`.

    Notes
    -----
    A :class:`ConfigLoader` does one thing: load a config from a source
    (file, command line arguments) and returns the data as a :class:`Struct`.
    There are lots of things that :class:`ConfigLoader` does not do.  It does
    not implement complex logic for finding config files.  It does not handle
    default values or merge multiple configs.  These things need to be
    handled elsewhere.
    """

    def __init__(self):
        """A base class for config loaders.

        Examples
        --------

        >>> cl = ConfigLoader()
        >>> config = cl.load_config()
        >>> config
        {}
        """
        self.clear()

    def clear(self):
        self.config = Config()

    def load_config(self):
        """Load a config from somewhere, return a :class:`Config` instance.

        Usually, this will cause self.config to be set and then returned.
        However, in most cases, :meth:`ConfigLoader.clear` should be called
        to erase any previous state.
        """
        self.clear()
        return self.config

class FileConfigLoader(ConfigLoader):
    """A base class for file based configurations.

    As we add more file based config loaders, the common logic should go
    here.
    """
    pass

class PyFileConfigLoader(FileConfigLoader):
    """A config loader for pure python files.

    This calls execfile on a plain python file and looks for attributes
    that are all caps.  These attribute are added to the config Struct.
    """

    def __init__(self, filename, path=None):
        """Build a config loader for a filename and path.

        Parameters
        ----------
        filename : str
            The file name of the config file.
        path : str, list, tuple
            The path to search for the config file on, or a sequence of
            paths to try in order.
        """
        super(PyFileConfigLoader, self).__init__()
        self.filename = filename
        self.path = path
        self.full_filename = ''
        self.data = None

    def load_config(self):
        """Load the config from a file and return it as a Struct."""
        self.clear()
        self._find_file()
        self._read_file_as_dict()
        self._convert_to_config()
        return self.config

    def _find_file(self):
        """Try to find the file by searching the paths."""
        self.full_filename = filefind(self.filename, self.path)

    def _read_file_as_dict(self):
        """Load the config file into self.config, with recursive loading."""
        # This closure is made available in the namespace that is used
        # to exec the config file.  This allows users to call
        # load_subconfig('myconfig.py') to load config files recursively.
        # It needs to be a closure because it has references to self.path
        # and self.config.  The sub-config is loaded with the same path
        # as the parent, but it uses an empty config which is then merged
        # with the parents.
        def load_subconfig(fname):
            loader = PyFileConfigLoader(fname, self.path)
            try:
                sub_config = loader.load_config()
            except IOError:
                # Pass silently if the sub config is not there. This happens
                # when a user us using a profile, but not the default config.
                pass
            else:
                self.config._merge(sub_config)

        # Again, this needs to be a closure and should be used in config
        # files to get the config being loaded.
        def get_config():
            return self.config

        namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
        execfile(self.full_filename, namespace)

    def _convert_to_config(self):
        if self.data is None:
            ConfigLoaderError('self.data does not exist')

class CommandLineConfigLoader(ConfigLoader):
    """A config loader for command line arguments.

    As we add more command line based loaders, the common logic should go
    here.
    """

class ArgParseConfigLoader(CommandLineConfigLoader):

    def __init__(self, argv=None, *parser_args, **parser_kw):
        """Create a config loader for use with argparse.

        Parameters
        ----------

        argv : optional, list
          If given, used to read command-line arguments from, otherwise
          sys.argv[1:] is used.

        parser_args : tuple
          A tuple of positional arguments that will be passed to the
          constructor of :class:`argparse.ArgumentParser`.

        parser_kw : dict
          A tuple of keyword arguments that will be passed to the
          constructor of :class:`argparse.ArgumentParser`.
        """
        super(CommandLineConfigLoader, self).__init__()
        if argv == None:
            argv = sys.argv[1:]
        self.argv = argv
        self.parser_args = parser_args
        kwargs = dict(argument_default=argparse.SUPPRESS)
        kwargs.update(parser_kw)
        self.parser_kw = kwargs

    def load_config(self, args=None):
        """Parse command line arguments and return as a Struct.

        Parameters
        ----------

        args : optional, list
          If given, a list with the structure of sys.argv[1:] to parse
          arguments from. If not given, the instance's self.argv attribute
          (given at construction time) is used."""
        self.clear()
        if args is None:
            args = self.argv
        self._create_parser()
        self._parse_args(args)
        self._convert_to_config()
        return self.config

    def get_extra_args(self):
        if hasattr(self, 'extra_args'):
            return self.extra_args
        else:
            return []

    def _create_parser(self):
        self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
        self._add_arguments()

    def _add_arguments(self):
        raise NotImplementedError("subclasses must implement _add_arguments")

    def _parse_args(self, args):
        """self.parser->self.parsed_data"""
        self.parsed_data, self.extra_args = self.parser.parse_known_args(args)

    def _convert_to_config(self):
        """self.parsed_data->self.config"""
        for k, v in vars(self.parsed_data).items():
            exec_str = 'self.config.' + k + '= v'
            exec exec_str in locals(), globals()

<commit_msg>Tidy up dictionary-style methods of Config (2to3 converts 'has_key' calls to use the 'in' operator, which will be that of the parent dict unless we define it).<commit_after># coding: utf-8
"""A simple configuration system.

Authors
-------
* Brian Granger
* Fernando Perez
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2009  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import __builtin__
import os
import sys

from IPython.external import argparse
from IPython.utils.path import filefind

#-----------------------------------------------------------------------------
# Exceptions
#-----------------------------------------------------------------------------

class ConfigError(Exception):
    pass

class ConfigLoaderError(ConfigError):
    pass

#-----------------------------------------------------------------------------
# Argparse fix
#-----------------------------------------------------------------------------

# Unfortunately argparse by default prints help messages to stderr instead of
# stdout.  This makes it annoying to capture long help screens at the command
# line, since one must know how to pipe stderr, which many users don't know how
# to do.  So we override the print_help method with one that defaults to
# stdout and use our class instead.

class ArgumentParser(argparse.ArgumentParser):
    """Simple argparse subclass that prints help to stdout by default."""

    def print_help(self, file=None):
        if file is None:
            file = sys.stdout
        return super(ArgumentParser, self).print_help(file)

    print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__

#-----------------------------------------------------------------------------
# Config class for holding config information
#-----------------------------------------------------------------------------

class Config(dict):
    """An attribute based dict that can do smart merges."""

    def __init__(self, *args, **kwds):
        dict.__init__(self, *args, **kwds)
        # This sets self.__dict__ = self, but it has to be done this way
        # because we are also overriding __setattr__.
        dict.__setattr__(self, '__dict__', self)

    def _merge(self, other):
        to_update = {}
        for k, v in other.items():
            if not self.has_key(k):
                to_update[k] = v
            else: # I have this key
                if isinstance(v, Config):
                    # Recursively merge common sub Configs
                    self[k]._merge(v)
                else:
                    # Plain updates for non-Configs
                    to_update[k] = v

        self.update(to_update)

    def _is_section_key(self, key):
        if key[0].upper()==key[0] and not key.startswith('_'):
            return True
        else:
            return False

    def __contains__(self, key):
        if self._is_section_key(key):
            return True
        else:
            return super(Config, self).__contains__(key)
    # .has_key is deprecated for dictionaries.
    has_key = __contains__

    def _has_section(self, key):
        if self._is_section_key(key):
            if super(Config, self).__contains__(key):
                return True
        return False

    def copy(self):
        return type(self)(dict.copy(self))

    def __copy__(self):
        return self.copy()

    def __deepcopy__(self, memo):
        import copy
        return type(self)(copy.deepcopy(self.items()))

    def __getitem__(self, key):
        # Because we use this for an exec namespace, we need to delegate
        # the lookup of names in __builtin__ to itself.  This means
        # that you can't have section or attribute names that are
        # builtins.
        try:
            return getattr(__builtin__, key)
        except AttributeError:
            pass
        if self._is_section_key(key):
            try:
                return dict.__getitem__(self, key)
            except KeyError:
                c = Config()
                dict.__setitem__(self, key, c)
                return c
        else:
            return dict.__getitem__(self, key)

    def __setitem__(self, key, value):
        # Don't allow names in __builtin__ to be modified.
        if hasattr(__builtin__, key):
            raise ConfigError('Config variable names cannot have the same name '
                              'as a Python builtin: %s' % key)
        if self._is_section_key(key):
            if not isinstance(value, Config):
                raise ValueError('values whose keys begin with an uppercase '
                                 'char must be Config instances: %r, %r' % (key, value))
        else:
            dict.__setitem__(self, key, value)

    def __getattr__(self, key):
        try:
            return self.__getitem__(key)
        except KeyError, e:
            raise AttributeError(e)

    def __setattr__(self, key, value):
        try:
            self.__setitem__(key, value)
        except KeyError, e:
            raise AttributeError(e)

    def __delattr__(self, key):
        try:
            dict.__delitem__(self, key)
        except KeyError, e:
            raise AttributeError(e)

#-----------------------------------------------------------------------------
# Config loading classes
#-----------------------------------------------------------------------------

class ConfigLoader(object):
    """A object for loading configurations from just about anywhere.

    The resulting configuration is packaged as a :class:`Struct`.

    Notes
    -----
    A :class:`ConfigLoader` does one thing: load a config from a source
    (file, command line arguments) and returns the data as a :class:`Struct`.
    There are lots of things that :class:`ConfigLoader` does not do.  It does
    not implement complex logic for finding config files.  It does not handle
    default values or merge multiple configs.  These things need to be
    handled elsewhere.
    """

    def __init__(self):
        """A base class for config loaders.

        Examples
        --------

        >>> cl = ConfigLoader()
        >>> config = cl.load_config()
        >>> config
        {}
        """
        self.clear()

    def clear(self):
        self.config = Config()

    def load_config(self):
        """Load a config from somewhere, return a :class:`Config` instance.

        Usually, this will cause self.config to be set and then returned.
        However, in most cases, :meth:`ConfigLoader.clear` should be called
        to erase any previous state.
        """
        self.clear()
        return self.config

class FileConfigLoader(ConfigLoader):
    """A base class for file based configurations.

    As we add more file based config loaders, the common logic should go
    here.
    """
    pass

class PyFileConfigLoader(FileConfigLoader):
    """A config loader for pure python files.

    This calls execfile on a plain python file and looks for attributes
    that are all caps.  These attribute are added to the config Struct.
    """

    def __init__(self, filename, path=None):
        """Build a config loader for a filename and path.

        Parameters
        ----------
        filename : str
            The file name of the config file.
        path : str, list, tuple
            The path to search for the config file on, or a sequence of
            paths to try in order.
        """
        super(PyFileConfigLoader, self).__init__()
        self.filename = filename
        self.path = path
        self.full_filename = ''
        self.data = None

    def load_config(self):
        """Load the config from a file and return it as a Struct."""
        self.clear()
        self._find_file()
        self._read_file_as_dict()
        self._convert_to_config()
        return self.config

    def _find_file(self):
        """Try to find the file by searching the paths."""
        self.full_filename = filefind(self.filename, self.path)

    def _read_file_as_dict(self):
        """Load the config file into self.config, with recursive loading."""
        # This closure is made available in the namespace that is used
        # to exec the config file.  This allows users to call
        # load_subconfig('myconfig.py') to load config files recursively.
        # It needs to be a closure because it has references to self.path
        # and self.config.  The sub-config is loaded with the same path
        # as the parent, but it uses an empty config which is then merged
        # with the parents.
        def load_subconfig(fname):
            loader = PyFileConfigLoader(fname, self.path)
            try:
                sub_config = loader.load_config()
            except IOError:
                # Pass silently if the sub config is not there. This happens
                # when a user us using a profile, but not the default config.
                pass
            else:
                self.config._merge(sub_config)

        # Again, this needs to be a closure and should be used in config
        # files to get the config being loaded.
        def get_config():
            return self.config

        namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
        execfile(self.full_filename, namespace)

    def _convert_to_config(self):
        if self.data is None:
            ConfigLoaderError('self.data does not exist')

class CommandLineConfigLoader(ConfigLoader):
    """A config loader for command line arguments.

    As we add more command line based loaders, the common logic should go
    here.
    """

class ArgParseConfigLoader(CommandLineConfigLoader):

    def __init__(self, argv=None, *parser_args, **parser_kw):
        """Create a config loader for use with argparse.

        Parameters
        ----------

        argv : optional, list
          If given, used to read command-line arguments from, otherwise
          sys.argv[1:] is used.

        parser_args : tuple
          A tuple of positional arguments that will be passed to the
          constructor of :class:`argparse.ArgumentParser`.

        parser_kw : dict
          A tuple of keyword arguments that will be passed to the
          constructor of :class:`argparse.ArgumentParser`.
        """
        super(CommandLineConfigLoader, self).__init__()
        if argv == None:
            argv = sys.argv[1:]
        self.argv = argv
        self.parser_args = parser_args
        kwargs = dict(argument_default=argparse.SUPPRESS)
        kwargs.update(parser_kw)
        self.parser_kw = kwargs

    def load_config(self, args=None):
        """Parse command line arguments and return as a Struct.

        Parameters
        ----------

        args : optional, list
          If given, a list with the structure of sys.argv[1:] to parse
          arguments from. If not given, the instance's self.argv attribute
          (given at construction time) is used."""
        self.clear()
        if args is None:
            args = self.argv
        self._create_parser()
        self._parse_args(args)
        self._convert_to_config()
        return self.config

    def get_extra_args(self):
        if hasattr(self, 'extra_args'):
            return self.extra_args
        else:
            return []

    def _create_parser(self):
        self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
        self._add_arguments()

    def _add_arguments(self):
        raise NotImplementedError("subclasses must implement _add_arguments")

    def _parse_args(self, args):
        """self.parser->self.parsed_data"""
        self.parsed_data, self.extra_args = self.parser.parse_known_args(args)

    def _convert_to_config(self):
        """self.parsed_data->self.config"""
        for k, v in vars(self.parsed_data).items():
            exec_str = 'self.config.' + k + '= v'
            exec exec_str in locals(), globals()

<|endoftext|>


<commit_before>import os

from progressbar import ProgressBar

from cnntools.descriptor_aggregator import DescriptorAggregator
from cnntools.descstore import DescriptorStoreHdf5
from cnntools.models import CaffeCNNSnapshot
from cnntools.redis_aggregator import patch_interrupt_signal, detach_patch_interrupt_signal
from cnntools.tasks import (compute_cnn_features_gpu_task,
                            compute_crf_features_task)
from cnntools.common_utils import iter_batch, progress_bar_widgets

def get_snapshot_id(caffe_cnn, snapshot_id):
    if snapshot_id:
        snapshot = CaffeCNNSnapshot.objects.get(id=snapshot_id)
        if snapshot.training_run.net.id != caffe_cnn.id:
            raise ValueError(
                'Snapshot\'s corresponding network id doesn\'t match given'
                ' network id!'
            )
        return snapshot_id

    # Get last snapshot
    snapshot = CaffeCNNSnapshot.objects.\
        filter(training_run__net__id=caffe_cnn.id).\
        order_by('-id')[0]

    return snapshot.id

def get_slug(caffe_cnn, snapshot_id, slug_extra):
    snapshot_id = get_snapshot_id(caffe_cnn, snapshot_id)
    slug = 'netid-{}:snapshot-{}'.format(caffe_cnn.netid, snapshot_id)
    if slug_extra:
        slug += slug_extra

    return slug, snapshot_id

def get_redis_key_core(slug, feature_name_list):
    return 'desc:slug-%s:fetname%s:batchid#' % (
        slug,
        '-'.join(feature_name_list),
    )

def get_redis_key(slug, feature_name_list, batchid_counter):
    return '%s%s' % (
        get_redis_key_core(slug, feature_name_list),
        batchid_counter,
    )

def get_task_id(slug, feature_name_list):
    return '%stask' % (
        get_redis_key_core(slug, feature_name_list),
    )

def get_descstore_dirname(netid, feature_name):
    dirname = '{}-{}'.format(netid, feature_name)
    # clean up
    invalid_chars = ['/', '(', ')', '[', ']', '{', '}', ' ', ':']

    for c in invalid_chars:
        dirname = dirname.replace(c, '-')

    return dirname

def get_descstore_filename(netid, feature_name):
    dirname = get_descstore_dirname(netid, feature_name)
    return '{}.hdf5'.format(dirname)

def dispatch_feature_comp(
    desc_rootpath,
    item_type,
    item_ids,
    node_batchsize,
    feature_name_list,
    num_dims_list,
    fetcomp_func,
    fetcomp_kwargs,
    slug,
):
    print 'Dispatching feature computation for incomplete {} models'.format(item_type)

    # TODO: Use DescriptorAggregator instead, so we don't have to assemble the filename...
    completed_ids = set()
    for feature_name in feature_name_list:
        filename = get_descstore_filename(slug, feature_name)
        if os.path.exists(os.path.join(desc_rootpath, filename)):
            src_store = DescriptorStoreHdf5(
                path=os.path.join(desc_rootpath, filename),
                readonly=True,
            )
            print 'Loaded completed descriptors for {}.'.format(filename)
            # Get already completed ids
            current_ids = set(src_store.ids[...])
            if completed_ids:
                completed_ids.intersection_update(current_ids)
            else:
                completed_ids = current_ids

    print 'Getting ids for incomplete items...'
    todo_ids = item_ids.difference(completed_ids)
    print '{} ids completed already'.format(len(completed_ids))
    print '{} ids to do...'.format(len(todo_ids))

    task_id = get_task_id(slug, feature_name_list)

    pbar = ProgressBar(widgets=progress_bar_widgets(), maxval=len(todo_ids))
    pbar_counter = 0
    pbar.start()
    batchid_counter = 0
    batch = []

    for batch in iter_batch(todo_ids, node_batchsize):
        # Dispatch job
        batch_id = get_redis_key(
            slug,
            feature_name_list,
            batchid_counter,
        )
        fetcomp_func.delay(
            item_type,
            task_id,
            batch_id,
            batch,
            feature_name_list,
            fetcomp_kwargs,
        )

        batchid_counter += 1
        pbar_counter += len(batch)
        pbar.update(pbar_counter)

    pbar.finish()

def aggregate_feature_comp(
    desc_rootpath,
    item_type,
    item_ids,
    feature_name_list,
    num_dims_list,
    aggr_batchsize,
    slug,
):
    print 'Aggregating feature computation for {} models'.format(item_type)
    patch_interrupt_signal()
    dirname_list = [
        get_descstore_dirname(slug, feature_name)
        for feature_name in feature_name_list
    ]

    aggregator = DescriptorAggregator(feature_name_list, dirname_list, num_dims_list)
    aggregator.load(desc_rootpath, readonly=False)
    task_id = get_task_id(slug, feature_name_list)
    ret = aggregator.run(item_ids, task_id, aggr_batchsize=aggr_batchsize)
    detach_patch_interrupt_signal()

    return ret

def retrieve_features(
    desc_rootpath,
    item_ids,
    feature_name_list,
    slug,
):
    # TODO: Use DescriptorAggregator instead, so we don't have to assemble the filename...
    features = {}
    for feature_name in feature_name_list:
        filename = get_descstore_filename(slug, feature_name)
        src_store = DescriptorStoreHdf5(
            path=os.path.join(desc_rootpath, filename),
            readonly=True,
        )
        if item_ids is None:
            item_ids = src_store.ids[...]

        features[feature_name] = src_store.block_get(item_ids, show_progress=True)
        del src_store

    return features

def compute_features(
    desc_rootpath,
    item_type,
    item_ids,
    node_batchsize,
    aggr_batchsize,
    feature_name_list,
    num_dims_list,
    fetcomp_func,
    fetcomp_kwargs,
    slug,
):
    """
    Extracts the specified feature for the specified items using a trained CNN.

    :param desc_rootpath: The root path of the directory where the computed
    features will be stored in.

    :param item_type: The type of the model class for the item which are
    classified (e.g. FgPhoto). This class should have 'title', 'photo'
    attributes/properties. The photo attribute should have most of the Photo
    model's fields. It is advised to use an actual Photo instance here.

    :param item_ids: List (or numpy array) of ids into the :ref:`item_type`
    table. The length of this list is the same as the length of :ref:`y_true`
    list and they have the same order.

    :param node_batchsize: The number of feature computations to put in one
    task executed on a worker node.

    :param aggr_batchsize: The number of batches to wait for before forcing the
    aggregator to download and remove those batches from redis.

    :param feature_name_list: The features' names in the network which will be
    extracted.

    :param num_dims_list: Dimensions of the computed features.

    :param fetcomp_func: The function which will be executed to compute
    features.

    :param fetcomp_kwargs: The parameters to pass to the feature computer
    function.

    :param slug: The unique humanly readable name associated with the feature
    computation task.
    """
    item_ids_set = set(item_ids)
    if isinstance(feature_name_list, (str, unicode)):
        if isinstance(num_dims_list, list):
            raise ValueError(
                'If "feature_name_list" is not specified as a list, '
                '"num_dims_list" has to a single number!'
            )
        feature_name_list = [feature_name_list]
        num_dims_list = [num_dims_list]

    dispatch_feature_comp(
        desc_rootpath,
        item_type,
        item_ids_set,
        node_batchsize,
        feature_name_list,
        num_dims_list,
        fetcomp_func,
        fetcomp_kwargs,
        slug,
    )
    was_interrupted = not aggregate_feature_comp(
        desc_rootpath,
        item_type,
        item_ids_set,
        feature_name_list,
        num_dims_list,
        aggr_batchsize,
        slug,
    )
    if was_interrupted:
        return None

    # The order of the item ids is important!
    return retrieve_features(
        desc_rootpath,
        item_ids,
        feature_name_list,
        slug,
    )

def compute_cnn_features(
    desc_rootpath,
    item_type,
    item_ids,
    node_batchsize,
    aggr_batchsize,
    caffe_cnn,
    feature_name_list,
    num_dims_list,
    snapshot_id,
    image_dims,
    mean,
    grayscale=False,
    auto_reshape=False,
    transfer_weights=False,
    slug_extra='',
    image_trafo_type_id=None,
    image_trafo_kwargs=None,
    fet_trafo_type_id=None,
    fet_trafo_kwargs=None,
):
    """
    Extracts the specified feature for the specified items using a trained CNN.

    :param desc_rootpath: The root path of the directory where the computed
    features will be stored in.

    :param item_type: The type of the model class for the item which are
    classified (e.g. FgPhoto). This class should have 'title', 'photo'
    attributes/properties. The photo attribute should have most of the Photo
    model's fields. It is advised to use an actual Photo instance here.

    :param item_ids: List (or numpy array) of ids into the :ref:`item_type`
    table. The length of this list is the same as the length of :ref:`y_true`
    list and they have the same order.

    :param node_batchsize: The number of feature computations to put in one
    task executed on a worker node.

    :param aggr_batchsize: The number of batches to wait for before forcing the
    aggregator to download and remove those batches from redis.

    :param caffe_cnn: The CaffeCNN instance which represents the CNN which is
    used for feature computation.

    :param feature_name_list: The features' names in the network which will be
    extracted.

    :param num_dims_list: Dimensions of the computed features.

    :param snapshot_id: The ID of the snapshot to use.

    :param image_dims: Tuple with two elements which defines the image
    dimensions (width, height). All input images will be resized to this size.
    This should be the same as the one was used for training.

    :param mean: Tuple with 3 elements which defines the mean which will be
    subtracted from all images. This should be the same as the one was used for
    training.

    :param grayscale: If true we convert all images to grayscale.

    :param auto_reshape: Automatically reshapes the network to the size of the
    input image. The user should still produce the same feature dimension for
    all images after the feature transformation.

    :param transfer_weights: Automatically convert the net weights to fully
    convolutional form. The deploy file of the model has to be fully
    convolutional!

    :param slug_extra: Additional string to add to the slug.

    :param image_trafo_type_id: Image transformation primitive. Currently the
    user can select from:
        ['MINC-padding']

    :param image_trafo_kwargs: Keyword arguments for the chosen image
    transformation primitive.

    :param fet_trafo_type_id: Feature transformation primitive. Currently the
    user can select from:
        ['MINC-spatial-avg', 'MINC-gram', 'MINC-mean-std']

    :param fet_trafo_kwargs: Keyword arguments for the chosen feature
    transformation primitive.
    """
    slug, snapshot_id = get_slug(caffe_cnn, snapshot_id, slug_extra)

    fetcomp_kwargs = {
        'snapshot_id': snapshot_id,
        'image_dims': image_dims,
        'mean': mean,
        'grayscale': grayscale,
        'auto_reshape': auto_reshape,
        'transfer_weights': transfer_weights,
        'image_trafo_type_id': image_trafo_type_id,
        'image_trafo_kwargs': image_trafo_kwargs,
        'fet_trafo_type_id': fet_trafo_type_id,
        'fet_trafo_kwargs': fet_trafo_kwargs,
    }

    return compute_features(
        desc_rootpath,
        item_type,
        item_ids,
        node_batchsize,
        aggr_batchsize,
        feature_name_list,
        num_dims_list,
        compute_cnn_features_gpu_task,
        fetcomp_kwargs,
        slug,
    )

def compute_crf_features(
    desc_rootpath,
    item_type,
    item_ids,
    node_batchsize,
    aggr_batchsize,
    feature_name_list,
    num_dims_list,
    crf_params,
    label_count,
    desc_dir,
    slug,
    image_trafo_type_id=None,
    image_trafo_kwargs=None,
    fet_trafo_type_id=None,
    fet_trafo_kwargs=None,
):
    """
    Runs the dense CRF on the specified features and images.

    :param desc_rootpath: The root path of the directory where the computed
    features will be stored in.

    :param item_type: The type of the model class for the item which are
    classified (e.g. FgPhoto). This class should have 'title', 'photo'
    attributes/properties. The photo attribute should have most of the Photo
    model's fields. It is advised to use an actual Photo instance here.

    :param item_ids: List (or numpy array) of ids into the :ref:`item_type`
    table. The length of this list is the same as the length of :ref:`y_true`
    list and they have the same order.

    :param node_batchsize: The number of feature computations to put in one
    task executed on a worker node.

    :param aggr_batchsize: The number of batches to wait for before forcing the
    aggregator to download and remove those batches from redis.

    :param feature_name_list: The features' names in the network which will be
    extracted.

    :param num_dims_list: Dimensions of the computed features.

    :param crf_params: Parameters used by the CRF algorithm. TODO: More
    documentation here.

    :param label_count: The number of classification labels.

    :param desc_dir: The directory where the descriptors which will be used by
    the CRF are

    :param slug: The unique humanly readable name associated with the feature
    computation task.

    :param image_trafo_type_id: Image transformation primitive. Currently the
    user can select from:
        ['MINC-padding']

    :param image_trafo_kwargs: Keyword arguments for the chosen image
    transformation primitive.

    :param fet_trafo_type_id: Feature transformation primitive. Currently the
    user can select from:
        ['MINC-spatial-avg', 'MINC-gram', 'MINC-mean-std']

    :param fet_trafo_kwargs: Keyword arguments for the chosen feature
    transformation primitive.
    """
    fetcomp_kwargs = {
        'crf_params': crf_params,
        'label_count': label_count,
        'desc_dir': desc_dir,
        'image_trafo_type_id': image_trafo_type_id,
        'image_trafo_kwargs': image_trafo_kwargs,
        'fet_trafo_type_id': fet_trafo_type_id,
        'fet_trafo_kwargs': fet_trafo_kwargs,
    }

    return compute_features(
        desc_rootpath,
        item_type,
        item_ids,
        node_batchsize,
        aggr_batchsize,
        feature_name_list,
        num_dims_list,
        compute_crf_features_task,
        fetcomp_kwargs,
        slug,
    )

<commit_msg>Removed unused compute_crf_features<commit_after>import os

from progressbar import ProgressBar

from cnntools.descriptor_aggregator import DescriptorAggregator
from cnntools.descstore import DescriptorStoreHdf5
from cnntools.models import CaffeCNNSnapshot
from cnntools.redis_aggregator import patch_interrupt_signal, detach_patch_interrupt_signal
from cnntools.tasks import compute_cnn_features_gpu_task
from cnntools.common_utils import iter_batch, progress_bar_widgets

def get_snapshot_id(caffe_cnn, snapshot_id):
    if snapshot_id:
        snapshot = CaffeCNNSnapshot.objects.get(id=snapshot_id)
        if snapshot.training_run.net.id != caffe_cnn.id:
            raise ValueError(
                'Snapshot\'s corresponding network id doesn\'t match given'
                ' network id!'
            )
        return snapshot_id

    # Get last snapshot
    snapshot = CaffeCNNSnapshot.objects.\
        filter(training_run__net__id=caffe_cnn.id).\
        order_by('-id')[0]

    return snapshot.id

def get_slug(caffe_cnn, snapshot_id, slug_extra):
    snapshot_id = get_snapshot_id(caffe_cnn, snapshot_id)
    slug = 'netid-{}:snapshot-{}'.format(caffe_cnn.netid, snapshot_id)
    if slug_extra:
        slug += slug_extra

    return slug, snapshot_id

def get_redis_key_core(slug, feature_name_list):
    return 'desc:slug-%s:fetname%s:batchid#' % (
        slug,
        '-'.join(feature_name_list),
    )

def get_redis_key(slug, feature_name_list, batchid_counter):
    return '%s%s' % (
        get_redis_key_core(slug, feature_name_list),
        batchid_counter,
    )

def get_task_id(slug, feature_name_list):
    return '%stask' % (
        get_redis_key_core(slug, feature_name_list),
    )

def get_descstore_dirname(netid, feature_name):
    dirname = '{}-{}'.format(netid, feature_name)
    # clean up
    invalid_chars = ['/', '(', ')', '[', ']', '{', '}', ' ', ':']

    for c in invalid_chars:
        dirname = dirname.replace(c, '-')

    return dirname

def get_descstore_filename(netid, feature_name):
    dirname = get_descstore_dirname(netid, feature_name)
    return '{}.hdf5'.format(dirname)

def dispatch_feature_comp(
    desc_rootpath,
    item_type,
    item_ids,
    node_batchsize,
    feature_name_list,
    num_dims_list,
    fetcomp_func,
    fetcomp_kwargs,
    slug,
):
    print 'Dispatching feature computation for incomplete {} models'.format(item_type)

    # TODO: Use DescriptorAggregator instead, so we don't have to assemble the filename...
    completed_ids = set()
    for feature_name in feature_name_list:
        filename = get_descstore_filename(slug, feature_name)
        if os.path.exists(os.path.join(desc_rootpath, filename)):
            src_store = DescriptorStoreHdf5(
                path=os.path.join(desc_rootpath, filename),
                readonly=True,
            )
            print 'Loaded completed descriptors for {}.'.format(filename)
            # Get already completed ids
            current_ids = set(src_store.ids[...])
            if completed_ids:
                completed_ids.intersection_update(current_ids)
            else:
                completed_ids = current_ids

    print 'Getting ids for incomplete items...'
    todo_ids = item_ids.difference(completed_ids)
    print '{} ids completed already'.format(len(completed_ids))
    print '{} ids to do...'.format(len(todo_ids))

    task_id = get_task_id(slug, feature_name_list)

    pbar = ProgressBar(widgets=progress_bar_widgets(), maxval=len(todo_ids))
    pbar_counter = 0
    pbar.start()
    batchid_counter = 0
    batch = []

    for batch in iter_batch(todo_ids, node_batchsize):
        # Dispatch job
        batch_id = get_redis_key(
            slug,
            feature_name_list,
            batchid_counter,
        )
        fetcomp_func.delay(
            item_type,
            task_id,
            batch_id,
            batch,
            feature_name_list,
            fetcomp_kwargs,
        )

        batchid_counter += 1
        pbar_counter += len(batch)
        pbar.update(pbar_counter)

    pbar.finish()

def aggregate_feature_comp(
    desc_rootpath,
    item_type,
    item_ids,
    feature_name_list,
    num_dims_list,
    aggr_batchsize,
    slug,
):
    print 'Aggregating feature computation for {} models'.format(item_type)
    patch_interrupt_signal()
    dirname_list = [
        get_descstore_dirname(slug, feature_name)
        for feature_name in feature_name_list
    ]

    aggregator = DescriptorAggregator(feature_name_list, dirname_list, num_dims_list)
    aggregator.load(desc_rootpath, readonly=False)
    task_id = get_task_id(slug, feature_name_list)
    ret = aggregator.run(item_ids, task_id, aggr_batchsize=aggr_batchsize)
    detach_patch_interrupt_signal()

    return ret

def retrieve_features(
    desc_rootpath,
    item_ids,
    feature_name_list,
    slug,
):
    # TODO: Use DescriptorAggregator instead, so we don't have to assemble the filename...
    features = {}
    for feature_name in feature_name_list:
        filename = get_descstore_filename(slug, feature_name)
        src_store = DescriptorStoreHdf5(
            path=os.path.join(desc_rootpath, filename),
            readonly=True,
        )
        if item_ids is None:
            item_ids = src_store.ids[...]

        features[feature_name] = src_store.block_get(item_ids, show_progress=True)
        del src_store

    return features

def compute_features(
    desc_rootpath,
    item_type,
    item_ids,
    node_batchsize,
    aggr_batchsize,
    feature_name_list,
    num_dims_list,
    fetcomp_func,
    fetcomp_kwargs,
    slug,
):
    """
    Extracts the specified feature for the specified items using a trained CNN.

    :param desc_rootpath: The root path of the directory where the computed
    features will be stored in.

    :param item_type: The type of the model class for the item which are
    classified (e.g. FgPhoto). This class should have 'title', 'photo'
    attributes/properties. The photo attribute should have most of the Photo
    model's fields. It is advised to use an actual Photo instance here.

    :param item_ids: List (or numpy array) of ids into the :ref:`item_type`
    table. The length of this list is the same as the length of :ref:`y_true`
    list and they have the same order.

    :param node_batchsize: The number of feature computations to put in one
    task executed on a worker node.

    :param aggr_batchsize: The number of batches to wait for before forcing the
    aggregator to download and remove those batches from redis.

    :param feature_name_list: The features' names in the network which will be
    extracted.

    :param num_dims_list: Dimensions of the computed features.

    :param fetcomp_func: The function which will be executed to compute
    features.

    :param fetcomp_kwargs: The parameters to pass to the feature computer
    function.

    :param slug: The unique humanly readable name associated with the feature
    computation task.
    """
    item_ids_set = set(item_ids)
    if isinstance(feature_name_list, (str, unicode)):
        if isinstance(num_dims_list, list):
            raise ValueError(
                'If "feature_name_list" is not specified as a list, '
                '"num_dims_list" has to a single number!'
            )
        feature_name_list = [feature_name_list]
        num_dims_list = [num_dims_list]

    dispatch_feature_comp(
        desc_rootpath,
        item_type,
        item_ids_set,
        node_batchsize,
        feature_name_list,
        num_dims_list,
        fetcomp_func,
        fetcomp_kwargs,
        slug,
    )
    was_interrupted = not aggregate_feature_comp(
        desc_rootpath,
        item_type,
        item_ids_set,
        feature_name_list,
        num_dims_list,
        aggr_batchsize,
        slug,
    )
    if was_interrupted:
        return None

    # The order of the item ids is important!
    return retrieve_features(
        desc_rootpath,
        item_ids,
        feature_name_list,
        slug,
    )

def compute_cnn_features(
    desc_rootpath,
    item_type,
    item_ids,
    node_batchsize,
    aggr_batchsize,
    caffe_cnn,
    feature_name_list,
    num_dims_list,
    snapshot_id,
    image_dims,
    mean,
    grayscale=False,
    auto_reshape=False,
    transfer_weights=False,
    slug_extra='',
    image_trafo_type_id=None,
    image_trafo_kwargs=None,
    fet_trafo_type_id=None,
    fet_trafo_kwargs=None,
):
    """
    Extracts the specified feature for the specified items using a trained CNN.

    :param desc_rootpath: The root path of the directory where the computed
    features will be stored in.

    :param item_type: The type of the model class for the item which are
    classified (e.g. FgPhoto). This class should have 'title', 'photo'
    attributes/properties. The photo attribute should have most of the Photo
    model's fields. It is advised to use an actual Photo instance here.

    :param item_ids: List (or numpy array) of ids into the :ref:`item_type`
    table. The length of this list is the same as the length of :ref:`y_true`
    list and they have the same order.

    :param node_batchsize: The number of feature computations to put in one
    task executed on a worker node.

    :param aggr_batchsize: The number of batches to wait for before forcing the
    aggregator to download and remove those batches from redis.

    :param caffe_cnn: The CaffeCNN instance which represents the CNN which is
    used for feature computation.

    :param feature_name_list: The features' names in the network which will be
    extracted.

    :param num_dims_list: Dimensions of the computed features.

    :param snapshot_id: The ID of the snapshot to use.

    :param image_dims: Tuple with two elements which defines the image
    dimensions (width, height). All input images will be resized to this size.
    This should be the same as the one was used for training.

    :param mean: Tuple with 3 elements which defines the mean which will be
    subtracted from all images. This should be the same as the one was used for
    training.

    :param grayscale: If true we convert all images to grayscale.

    :param auto_reshape: Automatically reshapes the network to the size of the
    input image. The user should still produce the same feature dimension for
    all images after the feature transformation.

    :param transfer_weights: Automatically convert the net weights to fully
    convolutional form. The deploy file of the model has to be fully
    convolutional!

    :param slug_extra: Additional string to add to the slug.

    :param image_trafo_type_id: Image transformation primitive. Currently the
    user can select from:
        ['MINC-padding']

    :param image_trafo_kwargs: Keyword arguments for the chosen image
    transformation primitive.

    :param fet_trafo_type_id: Feature transformation primitive. Currently the
    user can select from:
        ['MINC-spatial-avg', 'MINC-gram', 'MINC-mean-std']

    :param fet_trafo_kwargs: Keyword arguments for the chosen feature
    transformation primitive.
    """
    slug, snapshot_id = get_slug(caffe_cnn, snapshot_id, slug_extra)

    fetcomp_kwargs = {
        'snapshot_id': snapshot_id,
        'image_dims': image_dims,
        'mean': mean,
        'grayscale': grayscale,
        'auto_reshape': auto_reshape,
        'transfer_weights': transfer_weights,
        'image_trafo_type_id': image_trafo_type_id,
        'image_trafo_kwargs': image_trafo_kwargs,
        'fet_trafo_type_id': fet_trafo_type_id,
        'fet_trafo_kwargs': fet_trafo_kwargs,
    }

    return compute_features(
        desc_rootpath,
        item_type,
        item_ids,
        node_batchsize,
        aggr_batchsize,
        feature_name_list,
        num_dims_list,
        compute_cnn_features_gpu_task,
        fetcomp_kwargs,
        slug,
    )
<|endoftext|>


<commit_before># encoding: utf-8
"""
Utilities for getting information about IPython and the system it's running in.
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import os
import platform
import pprint
import sys
import subprocess

from IPython.core import release
from IPython.utils import py3compat, _sysinfo, encoding

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

def pkg_commit_hash(pkg_path):
    """Get short form of commit hash given directory `pkg_path`

    We get the commit hash from (in order of preference):

    * IPython.utils._sysinfo.commit
    * git output, if we are in a git repository

    If these fail, we return a not-found placeholder tuple

    Parameters
    ----------
    pkg_path : str
       directory containing package
       only used for getting commit from active repo

    Returns
    -------
    hash_from : str
       Where we got the hash from - description
    hash_str : str
       short form of hash
    """
    # Try and get commit from written commit text file
    if _sysinfo.commit:
        return "installation", _sysinfo.commit

    # maybe we are in a repository
    proc = subprocess.Popen('git rev-parse --short HEAD',
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            cwd=pkg_path, shell=True)
    repo_commit, _ = proc.communicate()
    if repo_commit:
        return 'repository', repo_commit.strip()
    return '(none found)', '<not found>'

def pkg_info(pkg_path):
    """Return dict describing the context of this package

    Parameters
    ----------
    pkg_path : str
       path containing __init__.py for package

    Returns
    -------
    context : dict
       with named parameters of interest
    """
    src, hsh = pkg_commit_hash(pkg_path)
    return dict(
        ipython_version=release.version,
        ipython_path=pkg_path,
        commit_source=src,
        commit_hash=hsh,
        sys_version=sys.version,
        sys_executable=sys.executable,
        sys_platform=sys.platform,
        platform=platform.platform(),
        os_name=os.name,
        default_encoding=encoding.DEFAULT_ENCODING,
        )

def get_sys_info():
    """Return useful information about IPython and the system, as a dict."""
    p = os.path
    path = p.dirname(p.abspath(p.join(__file__, '..')))
    return pkg_info(path)

@py3compat.doctest_refactor_print
def sys_info():
    """Return useful information about IPython and the system, as a string.

    Examples
    --------
    ::

        In [2]: print sys_info()
        {'commit_hash': '144fdae',      # random
         'commit_source': 'repository',
         'ipython_path': '/home/fperez/usr/lib/python2.6/site-packages/IPython',
         'ipython_version': '0.11.dev',
         'os_name': 'posix',
         'platform': 'Linux-2.6.35-22-generic-i686-with-Ubuntu-10.10-maverick',
         'sys_executable': '/usr/bin/python',
         'sys_platform': 'linux2',
         'sys_version': '2.6.6 (r266:84292, Sep 15 2010, 15:52:39) \\n[GCC 4.4.5]'}
    """
    return pprint.pformat(get_sys_info())

def _num_cpus_unix():
    """Return the number of active CPUs on a Unix system."""
    return os.sysconf("SC_NPROCESSORS_ONLN")

def _num_cpus_darwin():
    """Return the number of active CPUs on a Darwin system."""
    p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
    return p.stdout.read()

def _num_cpus_windows():
    """Return the number of active CPUs on a Windows system."""
    return os.environ.get("NUMBER_OF_PROCESSORS")

def num_cpus():
   """Return the effective number of CPUs in the system as an integer.

   This cross-platform function makes an attempt at finding the total number of
   available CPUs in the system, as returned by various underlying system and
   python calls.

   If it can't find a sensible answer, it returns 1 (though an error *may* make
   it return a large positive number that's actually incorrect).
   """

   # Many thanks to the Parallel Python project (http://www.parallelpython.com)
   # for the names of the keys we needed to look up for this function.  This
   # code was inspired by their equivalent function.

   ncpufuncs = {'Linux':_num_cpus_unix,
                'Darwin':_num_cpus_darwin,
                'Windows':_num_cpus_windows,
                # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
                # See http://bugs.python.org/issue1082 for details.
                'Microsoft':_num_cpus_windows,
                }

   ncpufunc = ncpufuncs.get(platform.system(),
                            # default to unix version (Solaris, AIX, etc)
                            _num_cpus_unix)

   try:
       ncpus = max(1,int(ncpufunc()))
   except:
       ncpus = 1
   return ncpus

<commit_msg>Calculate real path for symlinked IPython package in sysinfo<commit_after># encoding: utf-8
"""
Utilities for getting information about IPython and the system it's running in.
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import os
import platform
import pprint
import sys
import subprocess

from IPython.core import release
from IPython.utils import py3compat, _sysinfo, encoding

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

def pkg_commit_hash(pkg_path):
    """Get short form of commit hash given directory `pkg_path`

    We get the commit hash from (in order of preference):

    * IPython.utils._sysinfo.commit
    * git output, if we are in a git repository

    If these fail, we return a not-found placeholder tuple

    Parameters
    ----------
    pkg_path : str
       directory containing package
       only used for getting commit from active repo

    Returns
    -------
    hash_from : str
       Where we got the hash from - description
    hash_str : str
       short form of hash
    """
    # Try and get commit from written commit text file
    if _sysinfo.commit:
        return "installation", _sysinfo.commit

    # maybe we are in a repository
    proc = subprocess.Popen('git rev-parse --short HEAD',
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            cwd=pkg_path, shell=True)
    repo_commit, _ = proc.communicate()
    if repo_commit:
        return 'repository', repo_commit.strip()
    return '(none found)', '<not found>'

def pkg_info(pkg_path):
    """Return dict describing the context of this package

    Parameters
    ----------
    pkg_path : str
       path containing __init__.py for package

    Returns
    -------
    context : dict
       with named parameters of interest
    """
    src, hsh = pkg_commit_hash(pkg_path)
    return dict(
        ipython_version=release.version,
        ipython_path=pkg_path,
        commit_source=src,
        commit_hash=hsh,
        sys_version=sys.version,
        sys_executable=sys.executable,
        sys_platform=sys.platform,
        platform=platform.platform(),
        os_name=os.name,
        default_encoding=encoding.DEFAULT_ENCODING,
        )

def get_sys_info():
    """Return useful information about IPython and the system, as a dict."""
    p = os.path
    path = p.realpath(p.dirname(p.abspath(p.join(__file__, '..'))))
    return pkg_info(path)

@py3compat.doctest_refactor_print
def sys_info():
    """Return useful information about IPython and the system, as a string.

    Examples
    --------
    ::

        In [2]: print sys_info()
        {'commit_hash': '144fdae',      # random
         'commit_source': 'repository',
         'ipython_path': '/home/fperez/usr/lib/python2.6/site-packages/IPython',
         'ipython_version': '0.11.dev',
         'os_name': 'posix',
         'platform': 'Linux-2.6.35-22-generic-i686-with-Ubuntu-10.10-maverick',
         'sys_executable': '/usr/bin/python',
         'sys_platform': 'linux2',
         'sys_version': '2.6.6 (r266:84292, Sep 15 2010, 15:52:39) \\n[GCC 4.4.5]'}
    """
    return pprint.pformat(get_sys_info())

def _num_cpus_unix():
    """Return the number of active CPUs on a Unix system."""
    return os.sysconf("SC_NPROCESSORS_ONLN")

def _num_cpus_darwin():
    """Return the number of active CPUs on a Darwin system."""
    p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
    return p.stdout.read()

def _num_cpus_windows():
    """Return the number of active CPUs on a Windows system."""
    return os.environ.get("NUMBER_OF_PROCESSORS")

def num_cpus():
   """Return the effective number of CPUs in the system as an integer.

   This cross-platform function makes an attempt at finding the total number of
   available CPUs in the system, as returned by various underlying system and
   python calls.

   If it can't find a sensible answer, it returns 1 (though an error *may* make
   it return a large positive number that's actually incorrect).
   """

   # Many thanks to the Parallel Python project (http://www.parallelpython.com)
   # for the names of the keys we needed to look up for this function.  This
   # code was inspired by their equivalent function.

   ncpufuncs = {'Linux':_num_cpus_unix,
                'Darwin':_num_cpus_darwin,
                'Windows':_num_cpus_windows,
                # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
                # See http://bugs.python.org/issue1082 for details.
                'Microsoft':_num_cpus_windows,
                }

   ncpufunc = ncpufuncs.get(platform.system(),
                            # default to unix version (Solaris, AIX, etc)
                            _num_cpus_unix)

   try:
       ncpus = max(1,int(ncpufunc()))
   except:
       ncpus = 1
   return ncpus

<|endoftext|>