<commit_before># coding: utf-8
from __future__ import print_function
from __future__ import unicode_literals

import re
from collections import defaultdict, namedtuple
import operator
import sys
import urllib

from slacker import Slacker
from slack_token import slack_token
from aliases import DEBT_BOT_ALIASES

from transaction import Transaction, TransactionParseError
slack = Slacker(slack_token)

DEBT_CHANNEL_ID = 'C02CWS8H0'

def help_message():
    return """
    Welcome to #debt! I respond to the following commands:
    help: show this menu
    status: show your debtors and creditors
    transactions: show all of your transactions

    new transactions must be in the following format:
    (*user1*) (*operator*) *user2* (*value*) (*notes*), where:
    *user1 and *user2* are names, prefixed with an at-sign;
    *operator* is one of -> or <-;
    *value* is a numerical value;
    *notes* is a field for entering additional text.
    """

def list_channels():
    response = slack.channels.list()
    channels = [{'id': c['id'], 'name': c['name']}
                for c in response.body['channels']]
    print(channels)

def users():
    response = slack.users.list()
    return {m['id']: m['name'] for m in response.body['members']}

def get_channel_history(channel_id):
    response = slack.channels.history(channel=channel_id, count=1000)
    results = [m for m in response.body['messages'] if m.get('text')]

    while response.body['has_more']:
        oldest = min(m.get('ts') for m in response.body['messages'] if m.get('ts'))
        response = slack.channels.history(channel=channel_id, count=1000, latest=oldest)
        results.extend([m for m in response.body['messages'] if m.get('text')])
    return results

def transactions(user_id):
    messages = get_channel_history(DEBT_CHANNEL_ID)
    messages = [m for m in messages if re.search(user_id, m.get('text', ''))]
    return messages

def printable_transactions(user_id):
    def stringify_transaction(t):
        try:
            t = Transaction(m.get('text'))
            if t.parsed:
                return m.get('text')
            else:
                return 'SKIPPED ' + m.get('text')
        except TransactionParseError as err:
            return '❌FAILED! {}\n error ({})'.format(
                m.get('text'), err)

    messages = transactions(user_id)
    return [stringify_transaction(urllib.unquote(m.get('text'))) for m in messages]

def status_for_user(user_id, show_unparsed=False):
    user_list = users()
    balances = defaultdict(float)
    unparsed = list()
    errors = list()
    messages = transactions(user_id)
    for m in messages:
        try:
            transaction = Transaction(m.get('text'))
            if transaction.parsed:
                value, party = transaction.obligation_to_user(user_id)
                balances[party] += value
            else:
                unparsed.append(m.get('text'))
        except Exception as err:
            errors.append((err, m))
            # return "exception %s handling message %s" % (err, m)

    response = response_for_balances(balances, user_list)
    if show_unparsed and len(unparsed):
        response += "\n" + \
            "\n".join(["unabled to parse message: %s" % m for m in unparsed]) + "\n" + \
            "\n".join(["error %s in message %s" % (e, m) for e, m in errors])

    total_value = sum(b for b in balances.values())
    decorator = decoration_for_balance(total_value)
    preamble = "%s status for %s %s\n" % (
        decorator, user_name_for_id(user_id, user_list), decorator)
    return preamble + response

def response_for_balances(balances, user_list):
    responses = list()
    response_str = ""
    for other_user_id, value in reversed(sorted(
            balances.items(),
            key=operator.itemgetter(1))):
        if value > 0:
            responses.append(
                "*%s* owes you $%0.2f" % (user_name_for_id(other_user_id, user_list), abs(value)))
        elif value < 0:
            responses.append(
                "you owe *%s* $%0.2f" % (user_name_for_id(other_user_id, user_list), abs(value)))
    if len(responses):
        response_str = "\n".join(responses)
    else:
        response_str = " %s (%s), you are ominously debt free" % (
            user_list.get(user_id, "<$NAME NOT FOUND$>"), user_id)
    return response_str

def decoration_for_balance(balance):
    if balance > 0:
        return "📈"
    else:
        return "📉"

def user_name_for_id(user_id, user_list):
    user = user_list.get(user_id, user_id).lower()
    return DEBT_BOT_ALIASES.get(user, user)

# def _all_channel_posts():
#     all_messages = get_channel_history(DEBT_CHANNEL_ID)
#     for message in all_messages:
#         text = message.get('text')
#         parsed = re.search(r'<@(.*)>(.*)<@(.*?)>.?\$?([0-9\.]+)\$?(.*)', text)
#         if parsed:
#             transaction = Transaction(parsed.group(1), parsed.group(2), parsed.group(3), parsed.group(4), parsed.group(5))
#             print(transaction)
#         else:
#             parsed = re.search(r'<@(.*)>(.*)<@(.*?)>(.*?)([0-9\.]+)', text)
#             if parsed:
#                 transaction = Transaction(parsed.group(1), parsed.group(2), parsed.group(3), parsed.group(5), parsed.group(4))
#                 print(transaction)
#             else:
#                 print(text)

def all_transactions():
    all_messages = [m.get('text')
                    for m in get_channel_history(DEBT_CHANNEL_ID)]
    return [Transaction(p) for p in all_messages if p is not None]

def main():
    print(status_for_user("U024H5LFB", True))
    # print([u[1] for u in users().items()])
    # print(get_channel_history(DEBT_CHANNEL_ID))
    # print([h.get('text') for h in get_channel_history(DEBT_CHANNEL_ID)])
    # print(all_transactions(), sep='\n')
    # print(transactions("U02G8SVCB"))
    # _all_channel_posts()

    # transactions = all_transactions()
    # for t in transactions:
    #     if not t.parsed:
    #         print(t.raw_text, "|||", t)

if __name__ == "__main__":
    main()
<commit_msg>fixed ominous bug<commit_after># coding: utf-8
from __future__ import print_function
from __future__ import unicode_literals

import re
from collections import defaultdict, namedtuple
import operator
import sys
import urllib

from slacker import Slacker
from slack_token import slack_token
from aliases import DEBT_BOT_ALIASES

from transaction import Transaction, TransactionParseError
slack = Slacker(slack_token)

DEBT_CHANNEL_ID = 'C02CWS8H0'

def help_message():
    return """
    Welcome to #debt! I respond to the following commands:
    help: show this menu
    status: show your debtors and creditors
    transactions: show all of your transactions

    new transactions must be in the following format:
    (*user1*) (*operator*) *user2* (*value*) (*notes*), where:
    *user1 and *user2* are names, prefixed with an at-sign;
    *operator* is one of -> or <-;
    *value* is a numerical value;
    *notes* is a field for entering additional text.
    """

def list_channels():
    response = slack.channels.list()
    channels = [{'id': c['id'], 'name': c['name']}
                for c in response.body['channels']]
    print(channels)

def users():
    response = slack.users.list()
    return {m['id']: m['name'] for m in response.body['members']}

def get_channel_history(channel_id):
    response = slack.channels.history(channel=channel_id, count=1000)
    results = [m for m in response.body['messages'] if m.get('text')]

    while response.body['has_more']:
        oldest = min(m.get('ts') for m in response.body['messages'] if m.get('ts'))
        response = slack.channels.history(channel=channel_id, count=1000, latest=oldest)
        results.extend([m for m in response.body['messages'] if m.get('text')])
    return results

def transactions(user_id):
    messages = get_channel_history(DEBT_CHANNEL_ID)
    messages = [m for m in messages if re.search(user_id, m.get('text', ''))]
    return messages

def printable_transactions(user_id):
    def stringify_transaction(t):
        try:
            t = Transaction(m.get('text'))
            if t.parsed:
                return m.get('text')
            else:
                return 'SKIPPED ' + m.get('text')
        except TransactionParseError as err:
            return '❌FAILED! {}\n error ({})'.format(
                m.get('text'), err)

    messages = transactions(user_id)
    return [stringify_transaction(urllib.unquote(m.get('text'))) for m in messages]

def status_for_user(user_id, show_unparsed=False):
    user_list = users()
    balances = defaultdict(float)
    unparsed = list()
    errors = list()
    messages = transactions(user_id)
    for m in messages:
        try:
            transaction = Transaction(m.get('text'))
            if transaction.parsed:
                value, party = transaction.obligation_to_user(user_id)
                balances[party] += value
            else:
                unparsed.append(m.get('text'))
        except Exception as err:
            errors.append((err, m))
            # return "exception %s handling message %s" % (err, m)

    response = response_for_balances(balances, user_list, user_id)
    if show_unparsed and len(unparsed):
        response += "\n" + \
            "\n".join(["unabled to parse message: %s" % m for m in unparsed]) + "\n" + \
            "\n".join(["error %s in message %s" % (e, m) for e, m in errors])

    total_value = sum(b for b in balances.values())
    decorator = decoration_for_balance(total_value)
    preamble = "%s status for %s %s\n" % (
        decorator, user_name_for_id(user_id, user_list), decorator)
    return preamble + response

def response_for_balances(balances, user_list, user_id):
    responses = list()
    response_str = ""
    for other_user_id, value in reversed(sorted(
            balances.items(),
            key=operator.itemgetter(1))):
        if value > 0:
            responses.append(
                "*%s* owes you $%0.2f" % (user_name_for_id(other_user_id, user_list), abs(value)))
        elif value < 0:
            responses.append(
                "you owe *%s* $%0.2f" % (user_name_for_id(other_user_id, user_list), abs(value)))
    if len(responses):
        response_str = "\n".join(responses)
    else:
        response_str = " %s (%s), you are ominously debt free" % (
            user_list.get(user_id, "<$NAME NOT FOUND$>"), user_id)
    return response_str

def decoration_for_balance(balance):
    if balance > 0:
        return "📈"
    else:
        return "📉"

def user_name_for_id(user_id, user_list):
    user = user_list.get(user_id, user_id).lower()
    return DEBT_BOT_ALIASES.get(user, user)

# def _all_channel_posts():
#     all_messages = get_channel_history(DEBT_CHANNEL_ID)
#     for message in all_messages:
#         text = message.get('text')
#         parsed = re.search(r'<@(.*)>(.*)<@(.*?)>.?\$?([0-9\.]+)\$?(.*)', text)
#         if parsed:
#             transaction = Transaction(parsed.group(1), parsed.group(2), parsed.group(3), parsed.group(4), parsed.group(5))
#             print(transaction)
#         else:
#             parsed = re.search(r'<@(.*)>(.*)<@(.*?)>(.*?)([0-9\.]+)', text)
#             if parsed:
#                 transaction = Transaction(parsed.group(1), parsed.group(2), parsed.group(3), parsed.group(5), parsed.group(4))
#                 print(transaction)
#             else:
#                 print(text)

def all_transactions():
    all_messages = [m.get('text')
                    for m in get_channel_history(DEBT_CHANNEL_ID)]
    return [Transaction(p) for p in all_messages if p is not None]

def main():
    print(status_for_user("U024H5LFB", True))
    # print([u[1] for u in users().items()])
    # print(get_channel_history(DEBT_CHANNEL_ID))
    # print([h.get('text') for h in get_channel_history(DEBT_CHANNEL_ID)])
    # print(all_transactions(), sep='\n')
    # print(transactions("U02G8SVCB"))
    # _all_channel_posts()

    # transactions = all_transactions()
    # for t in transactions:
    #     if not t.parsed:
    #         print(t.raw_text, "|||", t)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of Eigen, a lightweight C++ template library
# for linear algebra.
#
# Copyright (C) 2009 Benjamin Schindler <bschindler@inf.ethz.ch>
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Pretty printers for Eigen::Matrix
# This is still pretty basic as the python extension to gdb is still pretty basic.
# It cannot handle complex eigen types and it doesn't support any of the other eigen types
# Such as quaternion or some other type.
# This code supports fixed size as well as dynamic size matrices

# To use it:
#
# * Create a directory and put the file as well as an empty __init__.py in
#   that directory.
# * Create a ~/.gdbinit file, that contains the following:
#      python
#      import sys
#      sys.path.insert(0, '/path/to/eigen/printer/directory')
#      from printers import register_eigen_printers
#      register_eigen_printers (None)
#      end

import gdb
import re
import itertools

class EigenMatrixPrinter:
	"Print Eigen Matrix or Array of some kind"

	def __init__(self, variety, val):
		"Extract all the necessary information"

		# Save the variety (presumably "Matrix" or "Array") for later usage
		self.variety = variety

		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		tag = self.type.tag
		regex = re.compile('\<.*\>')
		m = regex.findall(tag)[0][1:-1]
		template_params = m.split(',')
		template_params = map(lambda x:x.replace(" ", ""), template_params)

		if template_params[1] == '-0x00000000000000001' or template_params[1] == '-0x000000001' or template_params[1] == '-1':
			self.rows = val['m_storage']['m_rows']
		else:
			self.rows = int(template_params[1])

		if template_params[2] == '-0x00000000000000001' or template_params[2] == '-0x000000001' or template_params[2] == '-1':
			self.cols = val['m_storage']['m_cols']
		else:
			self.cols = int(template_params[2])

		self.options = 0 # default value
		if len(template_params) > 3:
			self.options = template_params[3];

		self.rowMajor = (int(self.options) & 0x1)

		self.innerType = self.type.template_argument(0)

		self.val = val

		# Fixed size matrices have a struct as their storage, so we need to walk through this
		self.data = self.val['m_storage']['m_data']
		if self.data.type.code == gdb.TYPE_CODE_STRUCT:
			self.data = self.data['array']
			self.data = self.data.cast(self.innerType.pointer())

	class _iterator:
		def __init__ (self, rows, cols, dataPtr, rowMajor):
			self.rows = rows
			self.cols = cols
			self.dataPtr = dataPtr
			self.currentRow = 0
			self.currentCol = 0
			self.rowMajor = rowMajor

		def __iter__ (self):
			return self

		def next(self):

			row = self.currentRow
			col = self.currentCol
			if self.rowMajor == 0:
				if self.currentCol >= self.cols:
					raise StopIteration

				self.currentRow = self.currentRow + 1
				if self.currentRow >= self.rows:
					self.currentRow = 0
					self.currentCol = self.currentCol + 1
			else:
				if self.currentRow >= self.rows:
					raise StopIteration

				self.currentCol = self.currentCol + 1
				if self.currentCol >= self.cols:
					self.currentCol = 0
					self.currentRow = self.currentRow + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1
			if (self.cols == 1): #if it's a column vector
				return ('[%d]' % (row,), item)
			elif (self.rows == 1): #if it's a row vector
				return ('[%d]' % (col,), item)
			return ('[%d,%d]' % (row, col), item)

	def children(self):

		return self._iterator(self.rows, self.cols, self.data, self.rowMajor)

	def to_string(self):
		return "Eigen::%s<%s,%d,%d,%s> (data ptr: %s)" % (self.variety, self.innerType, self.rows, self.cols, "RowMajor" if self.rowMajor else  "ColMajor", self.data)

class EigenQuaternionPrinter:
	"Print an Eigen Quaternion"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		self.innerType = self.type.template_argument(0)
		self.val = val

		# Quaternions have a struct as their storage, so we need to walk through this
		self.data = self.val['m_coeffs']['m_storage']['m_data']['array']
		self.data = self.data.cast(self.innerType.pointer())

	class _iterator:
		def __init__ (self, dataPtr):
			self.dataPtr = dataPtr
			self.currentElement = 0
			self.elementNames = ['x', 'y', 'z', 'w']

		def __iter__ (self):
			return self

		def next(self):
			element = self.currentElement

			if self.currentElement >= 4: #there are 4 elements in a quanternion
				raise StopIteration

			self.currentElement = self.currentElement + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1
			return ('[%s]' % (self.elementNames[element],), item)

	def children(self):

		return self._iterator(self.data)

	def to_string(self):
		return "Eigen::Quaternion<%s> (data ptr: %s)" % (self.innerType, self.data)

def build_eigen_dictionary ():
	pretty_printers_dict[re.compile('^Eigen::Quaternion<.*>$')] = lambda val: EigenQuaternionPrinter(val)
	pretty_printers_dict[re.compile('^Eigen::Matrix<.*>$')] = lambda val: EigenMatrixPrinter("Matrix", val)
	pretty_printers_dict[re.compile('^Eigen::Array<.*>$')]  = lambda val: EigenMatrixPrinter("Array",  val)

def register_eigen_printers(obj):
	"Register eigen pretty-printers with objfile Obj"

	if obj == None:
		obj = gdb
	obj.pretty_printers.append(lookup_function)

def lookup_function(val):
	"Look-up and return a pretty-printer that can print va."

	type = val.type

	if type.code == gdb.TYPE_CODE_REF:
		type = type.target()

	type = type.unqualified().strip_typedefs()

	typename = type.tag
	if typename == None:
		return None

	for function in pretty_printers_dict:
		if function.search(typename):
			return pretty_printers_dict[function](val)

	return None

pretty_printers_dict = {}

build_eigen_dictionary ()
<commit_msg>Make gdb pretty printer Python3-compatible (bug 800).<commit_after># -*- coding: utf-8 -*-
# This file is part of Eigen, a lightweight C++ template library
# for linear algebra.
#
# Copyright (C) 2009 Benjamin Schindler <bschindler@inf.ethz.ch>
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Pretty printers for Eigen::Matrix
# This is still pretty basic as the python extension to gdb is still pretty basic.
# It cannot handle complex eigen types and it doesn't support any of the other eigen types
# Such as quaternion or some other type.
# This code supports fixed size as well as dynamic size matrices

# To use it:
#
# * Create a directory and put the file as well as an empty __init__.py in
#   that directory.
# * Create a ~/.gdbinit file, that contains the following:
#      python
#      import sys
#      sys.path.insert(0, '/path/to/eigen/printer/directory')
#      from printers import register_eigen_printers
#      register_eigen_printers (None)
#      end

import gdb
import re
import itertools

class EigenMatrixPrinter:
	"Print Eigen Matrix or Array of some kind"

	def __init__(self, variety, val):
		"Extract all the necessary information"

		# Save the variety (presumably "Matrix" or "Array") for later usage
		self.variety = variety

		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		tag = self.type.tag
		regex = re.compile('\<.*\>')
		m = regex.findall(tag)[0][1:-1]
		template_params = m.split(',')
		template_params = [x.replace(" ", "") for x in template_params]

		if template_params[1] == '-0x00000000000000001' or template_params[1] == '-0x000000001' or template_params[1] == '-1':
			self.rows = val['m_storage']['m_rows']
		else:
			self.rows = int(template_params[1])

		if template_params[2] == '-0x00000000000000001' or template_params[2] == '-0x000000001' or template_params[2] == '-1':
			self.cols = val['m_storage']['m_cols']
		else:
			self.cols = int(template_params[2])

		self.options = 0 # default value
		if len(template_params) > 3:
			self.options = template_params[3];

		self.rowMajor = (int(self.options) & 0x1)

		self.innerType = self.type.template_argument(0)

		self.val = val

		# Fixed size matrices have a struct as their storage, so we need to walk through this
		self.data = self.val['m_storage']['m_data']
		if self.data.type.code == gdb.TYPE_CODE_STRUCT:
			self.data = self.data['array']
			self.data = self.data.cast(self.innerType.pointer())

	class _iterator:
		def __init__ (self, rows, cols, dataPtr, rowMajor):
			self.rows = rows
			self.cols = cols
			self.dataPtr = dataPtr
			self.currentRow = 0
			self.currentCol = 0
			self.rowMajor = rowMajor

		def __iter__ (self):
			return self

		def next(self):
                        return self.__next__()  # Python 2.x compatibility

		def __next__(self):

			row = self.currentRow
			col = self.currentCol
			if self.rowMajor == 0:
				if self.currentCol >= self.cols:
					raise StopIteration

				self.currentRow = self.currentRow + 1
				if self.currentRow >= self.rows:
					self.currentRow = 0
					self.currentCol = self.currentCol + 1
			else:
				if self.currentRow >= self.rows:
					raise StopIteration

				self.currentCol = self.currentCol + 1
				if self.currentCol >= self.cols:
					self.currentCol = 0
					self.currentRow = self.currentRow + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1
			if (self.cols == 1): #if it's a column vector
				return ('[%d]' % (row,), item)
			elif (self.rows == 1): #if it's a row vector
				return ('[%d]' % (col,), item)
			return ('[%d,%d]' % (row, col), item)

	def children(self):

		return self._iterator(self.rows, self.cols, self.data, self.rowMajor)

	def to_string(self):
		return "Eigen::%s<%s,%d,%d,%s> (data ptr: %s)" % (self.variety, self.innerType, self.rows, self.cols, "RowMajor" if self.rowMajor else  "ColMajor", self.data)

class EigenQuaternionPrinter:
	"Print an Eigen Quaternion"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		self.innerType = self.type.template_argument(0)
		self.val = val

		# Quaternions have a struct as their storage, so we need to walk through this
		self.data = self.val['m_coeffs']['m_storage']['m_data']['array']
		self.data = self.data.cast(self.innerType.pointer())

	class _iterator:
		def __init__ (self, dataPtr):
			self.dataPtr = dataPtr
			self.currentElement = 0
			self.elementNames = ['x', 'y', 'z', 'w']

		def __iter__ (self):
			return self

		def next(self):
                        return self.__next__()  # Python 2.x compatibility

		def __next__(self):
			element = self.currentElement

			if self.currentElement >= 4: #there are 4 elements in a quanternion
				raise StopIteration

			self.currentElement = self.currentElement + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1
			return ('[%s]' % (self.elementNames[element],), item)

	def children(self):

		return self._iterator(self.data)

	def to_string(self):
		return "Eigen::Quaternion<%s> (data ptr: %s)" % (self.innerType, self.data)

def build_eigen_dictionary ():
	pretty_printers_dict[re.compile('^Eigen::Quaternion<.*>$')] = lambda val: EigenQuaternionPrinter(val)
	pretty_printers_dict[re.compile('^Eigen::Matrix<.*>$')] = lambda val: EigenMatrixPrinter("Matrix", val)
	pretty_printers_dict[re.compile('^Eigen::Array<.*>$')]  = lambda val: EigenMatrixPrinter("Array",  val)

def register_eigen_printers(obj):
	"Register eigen pretty-printers with objfile Obj"

	if obj == None:
		obj = gdb
	obj.pretty_printers.append(lookup_function)

def lookup_function(val):
	"Look-up and return a pretty-printer that can print va."

	type = val.type

	if type.code == gdb.TYPE_CODE_REF:
		type = type.target()

	type = type.unqualified().strip_typedefs()

	typename = type.tag
	if typename == None:
		return None

	for function in pretty_printers_dict:
		if function.search(typename):
			return pretty_printers_dict[function](val)

	return None

pretty_printers_dict = {}

build_eigen_dictionary ()
<|endoftext|>


<commit_before># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
import collections
import hashlib
import logging
import os
import sys

import pkg_resources

import guild.plugin

from guild import config
from guild import entry_point_util
from guild import modelfile
from guild import namespace
from guild import resource

log = logging.getLogger("core")

_models = entry_point_util.EntryPointResources("guild.models", "model")

ModelRef = collections.namedtuple(
    "ModelRef", [
        "dist_type",
        "dist_name",
        "dist_version",
        "model_name"
    ])

class Model(object):

    def __init__(self, ep):
        self.name = ep.name
        self.dist = ep.dist
        self.modeldef = self._init_modeldef()
        self._fullname = None # lazy
        self._reference = None # lazy

    def __repr__(self):
        return "<%s.%s '%s'>" % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.name)

    @property
    def fullname(self):
        if self._fullname is None:
            pkg_name = namespace.apply_namespace(self.dist.project_name)
            self._fullname = "%s/%s" % (pkg_name, self.name)
        return self._fullname

    @property
    def reference(self):
        if self._reference is None:
            self._reference = self._init_reference()
        return self._reference

    def _init_modeldef(self):
        raise NotImplementedError()

    def _init_reference(self):
        raise NotImplementedError()

class ModelfileModel(Model):
    """A model associated with a modelfile.

    These are generated from ModelfileDistribution instances
    (i.e. distributions that are derrived from modefiles).
    """

    def _init_modeldef(self):
        assert isinstance(self.dist, ModelfileDistribution), self.dist
        return self.dist.get_modeldef(self.name)

    def _init_reference(self):
        if self.dist.modelfile.src:
            version = self._modelfile_hash(self.dist.modelfile.src)
        else:
            version = "unknown"
        path = os.path.abspath(self.dist.modelfile.dir)
        return ModelRef("modelfile", path, version, self.name)

    @staticmethod
    def _modelfile_hash(path):
        try:
            path_bytes = open(path, "rb").read()
        except IOError:
            log.warning(
                "unable to read %s to calculate modelfile hash", path)
            return "-"
        else:
            return hashlib.md5(path_bytes).hexdigest()

class PackageModel(Model):
    """A model associated with a package.

    These are generated by Guild packages.
    """

    def _init_modeldef(self):
        modeldef = _find_dist_modeldef(self.name, self.dist)
        if modeldef is None:
            raise ValueError(
                "undefined model '%s' in %s" % (self.name, self.dist))
        return modeldef

    def _init_reference(self):
        pkg_name = namespace.apply_namespace(self.dist.project_name)
        return ModelRef(
            "package",
            pkg_name,
            self.dist.version,
            self.name)

def _find_dist_modeldef(name, dist):
    for modeldef in _ensure_dist_modeldefs(dist):
        if modeldef.name == name:
            return modeldef
    return None

def _ensure_dist_modeldefs(dist):
    if not hasattr(dist, "_modelefs"):
        dist._modeldefs = _load_dist_modeldefs(dist)
    return dist._modeldefs

def _load_dist_modeldefs(dist):
    modeldefs = []
    try:
        record = dist.get_metadata_lines("RECORD")
    except IOError:
        log.warning(
            "distribution %s missing RECORD metadata - unable to find models",
            dist)
    else:
        for line in record:
            path = line.split(",", 1)[0]
            if os.path.basename(path) in modelfile.NAMES:
                fullpath = os.path.join(dist.location, path)
                _try_acc_modeldefs(fullpath, modeldefs)
    return modeldefs

def _try_acc_modeldefs(path, acc):
    try:
        models = modelfile.from_file(path)
    except Exception as e:
        log.warning("unable to load models from %s: %s", path, e)
    else:
        for modeldef in models:
            acc.append(modeldef)

class ModelfileDistribution(pkg_resources.Distribution):

    def __init__(self, modelfile):
        super(ModelfileDistribution, self).__init__(
            modelfile.dir,
            project_name=self._init_project_name(modelfile))
        self.modelfile = modelfile
        self._entry_map = self._init_entry_map()

    def __repr__(self):
        return "<guild.model.ModelfileDistribution '%s'>" % self.modelfile.dir

    def get_entry_map(self, group=None):
        if group is None:
            return self._entry_map
        else:
            return self._entry_map.get(group, {})

    def get_modeldef(self, name):
        for modeldef in self.modelfile:
            if modeldef.name == name:
                return modeldef
        raise ValueError(name)

    @staticmethod
    def _init_project_name(modelfile):
        """Returns a project name for a modelfile distribution.

        Modelfile distribution project names are of the format:

            '.modelfile.' + ESCAPED_MODELFILE_PATH

        ESCAPED_MODELFILE_PATH is a 'safe' project name (i.e. will not be
        modified in a call to `pkg_resources.safe_name`) that, when
        unescaped using `_unescape_project_name`, is the relative path of
        the directory containing the modelfile. The modefile name itself
        (e.g. 'MODEL' or 'MODELS') is not contained in the path.

        Modelfile paths are relative to the current working directory
        (i.e. the value of os.getcwd() at the time they are generated) and
        always start with '.'.
        """
        pkg_path = os.path.relpath(modelfile.dir)
        if pkg_path[0] != ".":
            pkg_path = os.path.join(".", pkg_path)
        safe_path = _escape_project_name(pkg_path)
        return ".modelfile.%s" % safe_path

    def _init_entry_map(self):
        return {
            "guild.models": {
                model.name: self._model_entry_point(model)
                for model in self.modelfile
            },
            "guild.resources": {
                res.fullname: self._resource_entry_point(res.fullname)
                for res in self._modelfile_resources()
            }
        }

    def _model_entry_point(self, model):
        return pkg_resources.EntryPoint(
            name=model.name,
            module_name="guild.model",
            attrs=("ModelfileModel",),
            dist=self)

    def _modelfile_resources(self):
        for modeldef in self.modelfile:
            for res in modeldef.resources:
                yield res

    def _resource_entry_point(self, name):
        return pkg_resources.EntryPoint(
            name=name,
            module_name="guild.model",
            attrs=("ModelfileResource",),
            dist=self)

def _escape_project_name(name):
    """Escapes name for use as a valie pkg_resources project name."""
    return str(base64.b16encode(name.encode("utf-8")).decode("utf-8"))

def _unescape_project_name(escaped_name):
    """Unescapes names escaped with `_escape_project_name`."""
    return str(base64.b16decode(escaped_name).decode("utf-8"))

class ModelfileResource(resource.Resource):

    def _init_resdef(self):
        assert isinstance(self.dist, ModelfileDistribution), self.dist
        model_name, res_name = _split_res_name(self.name)
        modeldef = self.dist.modelfile.get(model_name)
        assert modeldef, (self.name, self.dist)
        resdef = modeldef.get_resource(res_name)
        assert resdef, (self.name, self.dist)
        return resdef

def _split_res_name(name):
    parts = name.split(":", 1)
    if len(parts) != 2:
        raise ValueError("invalid resource name: %s" % name)
    return parts

class PackageModelResource(resource.Resource):

    def _init_resdef(self):
        model_name, res_name = _split_res_name(self.name)
        modeldef = _find_dist_modeldef(model_name, self.dist)
        if modeldef is None:
            raise ValueError(
                "undefined model '%s' in %s"
                % (model_name, self.dist))
        resdef = modeldef.get_resource(res_name)
        if resdef is None:
            raise ValueError(
                "undefined resource '%s%s' in %s"
                % (model_name, res_name, self.dist))
        return resdef

class ModelImportError(ImportError):
    pass

class ModelImporter(object):

    undef = object()

    def __init__(self, path):
        if not self._is_modelfile_dir(path):
            raise ModelImportError(path)
        self.path = path
        self._dist = self.undef # lazy

    @staticmethod
    def _is_modelfile_dir(path):
        return (modelfile.dir_has_modelfile(path) or
                os.path.abspath(path) == os.path.abspath(config.cwd()))

    @property
    def dist(self):
        if self._dist is self.undef:
            self._dist = self._init_dist()
        return self._dist

    def _init_dist(self):
        if not os.path.isdir(self.path):
            return None
        try:
            mf = modelfile.from_dir(self.path)
        except modelfile.NoModels:
            return self._plugin_model_dist()
        else:
            return ModelfileDistribution(mf)

    def _plugin_model_dist(self):
        models_data = []
        for _, plugin in guild.plugin.iter_plugins():
            for data in plugin.find_models(self.path):
                models_data.append(data)
        if models_data:
            mf = modelfile.Modelfile(models_data, dir=self.path)
            return ModelfileDistribution(mf)
        else:
            return None

    @staticmethod
    def find_module(_fullname, _path=None):
        return None

def _model_finder(importer, path, _only=False):
    assert isinstance(importer, ModelImporter)
    assert importer.path == path, (importer.path, path)
    if importer.dist:
        yield importer.dist

class ModelfileNamespace(namespace.PrefixNamespace):

    prefix = ".modelfile."

    @staticmethod
    def pip_info(_name):
        raise TypeError("modelfiles cannot be installed using pip")

    def package_name(self, project_name):
        pkg = super(ModelfileNamespace, self).package_name(project_name)
        parts = pkg.split("/", 1)
        decoded_project_name = _unescape_project_name(parts[0])
        rest = "/" + parts[1] if len(parts) == 2 else ""
        return decoded_project_name + rest

def get_path():
    return _models.path()

def set_path(path):
    _models.set_path(path)

def insert_path(item):
    path = _models.path()
    try:
        path.remove(item)
    except ValueError:
        pass
    path.insert(0, item)
    _models.set_path(path)

def iter_models():
    for _name, model in _models:
        if not model.modeldef.private:
            yield model

def for_name(name):
    return _models.for_name(name)

def iter_():
    for _name, model in _models:
        if not model.modeldef.private:
            yield model

def _register_model_finder():
    sys.path_hooks.insert(0, ModelImporter)
    pkg_resources.register_finder(ModelImporter, _model_finder)

_register_model_finder()
<commit_msg>Handle model discovery when there's a bad modelfile<commit_after># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
import collections
import hashlib
import logging
import os
import sys

import pkg_resources

import guild.plugin

from guild import config
from guild import entry_point_util
from guild import modelfile
from guild import namespace
from guild import resource

log = logging.getLogger("core")

_models = entry_point_util.EntryPointResources("guild.models", "model")

ModelRef = collections.namedtuple(
    "ModelRef", [
        "dist_type",
        "dist_name",
        "dist_version",
        "model_name"
    ])

class Model(object):

    def __init__(self, ep):
        self.name = ep.name
        self.dist = ep.dist
        self.modeldef = self._init_modeldef()
        self._fullname = None # lazy
        self._reference = None # lazy

    def __repr__(self):
        return "<%s.%s '%s'>" % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.name)

    @property
    def fullname(self):
        if self._fullname is None:
            pkg_name = namespace.apply_namespace(self.dist.project_name)
            self._fullname = "%s/%s" % (pkg_name, self.name)
        return self._fullname

    @property
    def reference(self):
        if self._reference is None:
            self._reference = self._init_reference()
        return self._reference

    def _init_modeldef(self):
        raise NotImplementedError()

    def _init_reference(self):
        raise NotImplementedError()

class ModelfileModel(Model):
    """A model associated with a modelfile.

    These are generated from ModelfileDistribution instances
    (i.e. distributions that are derrived from modefiles).
    """

    def _init_modeldef(self):
        assert isinstance(self.dist, ModelfileDistribution), self.dist
        return self.dist.get_modeldef(self.name)

    def _init_reference(self):
        if self.dist.modelfile.src:
            version = self._modelfile_hash(self.dist.modelfile.src)
        else:
            version = "unknown"
        path = os.path.abspath(self.dist.modelfile.dir)
        return ModelRef("modelfile", path, version, self.name)

    @staticmethod
    def _modelfile_hash(path):
        try:
            path_bytes = open(path, "rb").read()
        except IOError:
            log.warning(
                "unable to read %s to calculate modelfile hash", path)
            return "-"
        else:
            return hashlib.md5(path_bytes).hexdigest()

class PackageModel(Model):
    """A model associated with a package.

    These are generated by Guild packages.
    """

    def _init_modeldef(self):
        modeldef = _find_dist_modeldef(self.name, self.dist)
        if modeldef is None:
            raise ValueError(
                "undefined model '%s' in %s" % (self.name, self.dist))
        return modeldef

    def _init_reference(self):
        pkg_name = namespace.apply_namespace(self.dist.project_name)
        return ModelRef(
            "package",
            pkg_name,
            self.dist.version,
            self.name)

def _find_dist_modeldef(name, dist):
    for modeldef in _ensure_dist_modeldefs(dist):
        if modeldef.name == name:
            return modeldef
    return None

def _ensure_dist_modeldefs(dist):
    if not hasattr(dist, "_modelefs"):
        dist._modeldefs = _load_dist_modeldefs(dist)
    return dist._modeldefs

def _load_dist_modeldefs(dist):
    modeldefs = []
    try:
        record = dist.get_metadata_lines("RECORD")
    except IOError:
        log.warning(
            "distribution %s missing RECORD metadata - unable to find models",
            dist)
    else:
        for line in record:
            path = line.split(",", 1)[0]
            if os.path.basename(path) in modelfile.NAMES:
                fullpath = os.path.join(dist.location, path)
                _try_acc_modeldefs(fullpath, modeldefs)
    return modeldefs

def _try_acc_modeldefs(path, acc):
    try:
        models = modelfile.from_file(path)
    except Exception as e:
        log.warning("unable to load models from %s: %s", path, e)
    else:
        for modeldef in models:
            acc.append(modeldef)

class ModelfileDistribution(pkg_resources.Distribution):

    def __init__(self, modelfile):
        super(ModelfileDistribution, self).__init__(
            modelfile.dir,
            project_name=self._init_project_name(modelfile))
        self.modelfile = modelfile
        self._entry_map = self._init_entry_map()

    def __repr__(self):
        return "<guild.model.ModelfileDistribution '%s'>" % self.modelfile.dir

    def get_entry_map(self, group=None):
        if group is None:
            return self._entry_map
        else:
            return self._entry_map.get(group, {})

    def get_modeldef(self, name):
        for modeldef in self.modelfile:
            if modeldef.name == name:
                return modeldef
        raise ValueError(name)

    @staticmethod
    def _init_project_name(modelfile):
        """Returns a project name for a modelfile distribution.

        Modelfile distribution project names are of the format:

            '.modelfile.' + ESCAPED_MODELFILE_PATH

        ESCAPED_MODELFILE_PATH is a 'safe' project name (i.e. will not be
        modified in a call to `pkg_resources.safe_name`) that, when
        unescaped using `_unescape_project_name`, is the relative path of
        the directory containing the modelfile. The modefile name itself
        (e.g. 'MODEL' or 'MODELS') is not contained in the path.

        Modelfile paths are relative to the current working directory
        (i.e. the value of os.getcwd() at the time they are generated) and
        always start with '.'.
        """
        pkg_path = os.path.relpath(modelfile.dir)
        if pkg_path[0] != ".":
            pkg_path = os.path.join(".", pkg_path)
        safe_path = _escape_project_name(pkg_path)
        return ".modelfile.%s" % safe_path

    def _init_entry_map(self):
        return {
            "guild.models": {
                model.name: self._model_entry_point(model)
                for model in self.modelfile
            },
            "guild.resources": {
                res.fullname: self._resource_entry_point(res.fullname)
                for res in self._modelfile_resources()
            }
        }

    def _model_entry_point(self, model):
        return pkg_resources.EntryPoint(
            name=model.name,
            module_name="guild.model",
            attrs=("ModelfileModel",),
            dist=self)

    def _modelfile_resources(self):
        for modeldef in self.modelfile:
            for res in modeldef.resources:
                yield res

    def _resource_entry_point(self, name):
        return pkg_resources.EntryPoint(
            name=name,
            module_name="guild.model",
            attrs=("ModelfileResource",),
            dist=self)

def _escape_project_name(name):
    """Escapes name for use as a valie pkg_resources project name."""
    return str(base64.b16encode(name.encode("utf-8")).decode("utf-8"))

def _unescape_project_name(escaped_name):
    """Unescapes names escaped with `_escape_project_name`."""
    return str(base64.b16decode(escaped_name).decode("utf-8"))

class ModelfileResource(resource.Resource):

    def _init_resdef(self):
        assert isinstance(self.dist, ModelfileDistribution), self.dist
        model_name, res_name = _split_res_name(self.name)
        modeldef = self.dist.modelfile.get(model_name)
        assert modeldef, (self.name, self.dist)
        resdef = modeldef.get_resource(res_name)
        assert resdef, (self.name, self.dist)
        return resdef

def _split_res_name(name):
    parts = name.split(":", 1)
    if len(parts) != 2:
        raise ValueError("invalid resource name: %s" % name)
    return parts

class PackageModelResource(resource.Resource):

    def _init_resdef(self):
        model_name, res_name = _split_res_name(self.name)
        modeldef = _find_dist_modeldef(model_name, self.dist)
        if modeldef is None:
            raise ValueError(
                "undefined model '%s' in %s"
                % (model_name, self.dist))
        resdef = modeldef.get_resource(res_name)
        if resdef is None:
            raise ValueError(
                "undefined resource '%s%s' in %s"
                % (model_name, res_name, self.dist))
        return resdef

class ModelImportError(ImportError):
    pass

class BadModelfileDistribution(pkg_resources.Distribution):
    """Distribution for a modelfile that can't be read."""

    def __init__(self, path):
        super(BadModelfileDistribution, self).__init__(path)

    def __repr__(self):
        return "<guild.model.BadModelfileDistribution '%s'>" % self.location

    def get_entry_map(self, group=None):
        return {}

class ModelImporter(object):

    undef = object()

    def __init__(self, path):
        if not self._is_modelfile_dir(path):
            raise ModelImportError(path)
        self.path = path
        self._dist = self.undef # lazy

    @staticmethod
    def _is_modelfile_dir(path):
        return (modelfile.dir_has_modelfile(path) or
                os.path.abspath(path) == os.path.abspath(config.cwd()))

    @property
    def dist(self):
        if self._dist is self.undef:
            self._dist = self._init_dist()
        return self._dist

    def _init_dist(self):
        if not os.path.isdir(self.path):
            return None
        try:
            mf = modelfile.from_dir(self.path)
        except modelfile.NoModels:
            return self._plugin_model_dist()
        except Exception as e:
            log.warning(
                "error loading modelfile %s: %s", self.path, e)
            return BadModelfileDistribution(self.path)
        else:
            return ModelfileDistribution(mf)

    def _plugin_model_dist(self):
        models_data = []
        for _, plugin in guild.plugin.iter_plugins():
            for data in plugin.find_models(self.path):
                models_data.append(data)
        if models_data:
            mf = modelfile.Modelfile(models_data, dir=self.path)
            return ModelfileDistribution(mf)
        else:
            return None

    @staticmethod
    def find_module(_fullname, _path=None):
        return None

def _model_finder(importer, path, _only=False):
    assert isinstance(importer, ModelImporter)
    assert importer.path == path, (importer.path, path)
    if importer.dist:
        yield importer.dist

class ModelfileNamespace(namespace.PrefixNamespace):

    prefix = ".modelfile."

    @staticmethod
    def pip_info(_name):
        raise TypeError("modelfiles cannot be installed using pip")

    def package_name(self, project_name):
        pkg = super(ModelfileNamespace, self).package_name(project_name)
        parts = pkg.split("/", 1)
        decoded_project_name = _unescape_project_name(parts[0])
        rest = "/" + parts[1] if len(parts) == 2 else ""
        return decoded_project_name + rest

def get_path():
    return _models.path()

def set_path(path):
    _models.set_path(path)

def insert_path(item):
    path = _models.path()
    try:
        path.remove(item)
    except ValueError:
        pass
    path.insert(0, item)
    _models.set_path(path)

def iter_models():
    for _name, model in _models:
        if not model.modeldef.private:
            yield model

def for_name(name):
    return _models.for_name(name)

def iter_():
    for _name, model in _models:
        if not model.modeldef.private:
            yield model

def _register_model_finder():
    sys.path_hooks.insert(0, ModelImporter)
    pkg_resources.register_finder(ModelImporter, _model_finder)

_register_model_finder()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of Eigen, a lightweight C++ template library
# for linear algebra.
#
# Copyright (C) 2009 Benjamin Schindler <bschindler@inf.ethz.ch>
#
# Eigen is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3 of the License, or (at your option) any later version.
#
# Alternatively, you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# Eigen is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License and a copy of the GNU General Public License along with
# Eigen. If not, see <http://www.gnu.org/licenses/>.

# Pretty printers for Eigen::Matrix
# This is still pretty basic as the python extension to gdb is still pretty basic.
# It cannot handle complex eigen types and it doesn't support any of the other eigen types
# Such as quaternion or some other type.
# This code supports fixed size as well as dynamic size matrices

# To use it:
#
# * create a directory and put the file as well as an empty __init__.py in that directory
# * Create a ~/.gdbinit file, that contains the following:

import gdb
import re
import itertools

class EigenMatrixPrinter:
	"Print Eigen Matrix of some kind"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		tag = self.type.tag
		regex = re.compile('\<.*\>')
		m = regex.findall(tag)[0][1:-1]
		template_params = m.split(',')
		template_params = map(lambda x:x.replace(" ", ""), template_params)

		self.rows = int(template_params[1])
		self.cols = int(template_params[2])

		if self.rows == 10000:
			self.rows = val['m_storage']['m_rows']

		if self.cols == 10000:
			self.cols = val['m_storage']['m_cols']

		self.innerType = self.type.template_argument(0)

		self.val = val

	class _iterator:
		def __init__ (self, rows, cols, dataPtr):
			self.rows = rows
			self.cols = cols
			self.dataPtr = dataPtr
			self.currentRow = 0
			self.currentCol = 0

		def __iter__ (self):
			return self

		def next(self):
			if self.currentCol >= self.cols:
				raise StopIteration

			row = self.currentRow
			col = self.currentCol
			self.currentRow = self.currentRow + 1
			if self.currentRow >= self.rows:
				self.currentRow = 0
				self.currentCol = self.currentCol + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1

			return ('[%d, %d]' % (row, col), item)

	def children(self):
		data = self.val['m_storage']['m_data']
		# Fixed size matrices have a struct as their storage, so we need to walk through this
		if data.type.code == gdb.TYPE_CODE_STRUCT:
			data = data['array']
			data = data.cast(self.innerType.pointer())
		return self._iterator(self.rows, self.cols, data)

	def to_string(self):
		return "Eigen::Matrix<%s,%d,%d>" % (self.innerType, self.rows, self.cols)

def build_eigen_dictionary ():
	pretty_printers_dict[re.compile('^Eigen::Matrix<.*>$')] = lambda val: EigenMatrixPrinter(val)

def register_eigen_printers(obj):
	"Register eigen pretty-printers with objfile Obj"

	if obj == None:
		obj = gdb
	obj.pretty_printers.append(lookup_function)

def lookup_function(val):
	"Look-up and return a pretty-printer that can print va."

	type = val.type

	if type.code == gdb.TYPE_CODE_REF:
		type = type.target()

	type = type.unqualified().strip_typedefs()

	typename = type.tag
	if typename == None:
		return None

	for function in pretty_printers_dict:
		if function.search(typename):
			return pretty_printers_dict[function](val)

	return None

pretty_printers_dict = {}

build_eigen_dictionary ()
<commit_msg>Display the data ptr as part of the header<commit_after># -*- coding: utf-8 -*-
# This file is part of Eigen, a lightweight C++ template library
# for linear algebra.
#
# Copyright (C) 2009 Benjamin Schindler <bschindler@inf.ethz.ch>
#
# Eigen is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3 of the License, or (at your option) any later version.
#
# Alternatively, you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# Eigen is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License and a copy of the GNU General Public License along with
# Eigen. If not, see <http://www.gnu.org/licenses/>.

# Pretty printers for Eigen::Matrix
# This is still pretty basic as the python extension to gdb is still pretty basic.
# It cannot handle complex eigen types and it doesn't support any of the other eigen types
# Such as quaternion or some other type.
# This code supports fixed size as well as dynamic size matrices

# To use it:
#
# * create a directory and put the file as well as an empty __init__.py in that directory
# * Create a ~/.gdbinit file, that contains the following:

import gdb
import re
import itertools

class EigenMatrixPrinter:
	"Print Eigen Matrix of some kind"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		tag = self.type.tag
		regex = re.compile('\<.*\>')
		m = regex.findall(tag)[0][1:-1]
		template_params = m.split(',')
		template_params = map(lambda x:x.replace(" ", ""), template_params)

		self.rows = int(template_params[1])
		self.cols = int(template_params[2])

		if self.rows == 10000:
			self.rows = val['m_storage']['m_rows']

		if self.cols == 10000:
			self.cols = val['m_storage']['m_cols']

		self.innerType = self.type.template_argument(0)

		self.val = val

		# Fixed size matrices have a struct as their storage, so we need to walk through this
		self.data = self.val['m_storage']['m_data']
		if self.data.type.code == gdb.TYPE_CODE_STRUCT:
			self.data = self.data['array']
			self.data = self.data.cast(self.innerType.pointer())

	class _iterator:
		def __init__ (self, rows, cols, dataPtr):
			self.rows = rows
			self.cols = cols
			self.dataPtr = dataPtr
			self.currentRow = 0
			self.currentCol = 0

		def __iter__ (self):
			return self

		def next(self):
			if self.currentCol >= self.cols:
				raise StopIteration

			row = self.currentRow
			col = self.currentCol
			self.currentRow = self.currentRow + 1
			if self.currentRow >= self.rows:
				self.currentRow = 0
				self.currentCol = self.currentCol + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1

			return ('[%d, %d]' % (row, col), item)

	def children(self):

		return self._iterator(self.rows, self.cols, self.data)

	def to_string(self):
		return "Eigen::Matrix<%s,%d,%d> (data ptr: %s)" % (self.innerType, self.rows, self.cols, self.data)

def build_eigen_dictionary ():
	pretty_printers_dict[re.compile('^Eigen::Matrix<.*>$')] = lambda val: EigenMatrixPrinter(val)

def register_eigen_printers(obj):
	"Register eigen pretty-printers with objfile Obj"

	if obj == None:
		obj = gdb
	obj.pretty_printers.append(lookup_function)

def lookup_function(val):
	"Look-up and return a pretty-printer that can print va."

	type = val.type

	if type.code == gdb.TYPE_CODE_REF:
		type = type.target()

	type = type.unqualified().strip_typedefs()

	typename = type.tag
	if typename == None:
		return None

	for function in pretty_printers_dict:
		if function.search(typename):
			return pretty_printers_dict[function](val)

	return None

pretty_printers_dict = {}

build_eigen_dictionary ()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of Eigen, a lightweight C++ template library
# for linear algebra.
#
# Copyright (C) 2009 Benjamin Schindler <bschindler@inf.ethz.ch>
#
# Eigen is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3 of the License, or (at your option) any later version.
#
# Alternatively, you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# Eigen is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License and a copy of the GNU General Public License along with
# Eigen. If not, see <http://www.gnu.org/licenses/>.

# Pretty printers for Eigen::Matrix
# This is still pretty basic as the python extension to gdb is still pretty basic.
# It cannot handle complex eigen types and it doesn't support any of the other eigen types
# Such as quaternion or some other type.
# This code supports fixed size as well as dynamic size matrices

# To use it:
#
# * create a directory and put the file as well as an empty __init__.py in that directory
# * Create a ~/.gdbinit file, that contains the following:

import gdb
import re
import itertools

class EigenMatrixPrinter:
	"Print Eigen Matrix of some kind"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		tag = self.type.tag
		regex = re.compile('\<.*\>')
		m = regex.findall(tag)[0][1:-1]
		template_params = m.split(',')
		template_params = map(lambda x:x.replace(" ", ""), template_params)

		if template_params[1] == '-0x00000000000000001':
			self.rows = val['m_storage']['m_rows']
		else:
			self.rows = int(template_params[1])

		if template_params[2] == '-0x00000000000000001':
			self.cols = val['m_storage']['m_cols']
		else:
			self.cols = int(template_params[2])

		self.options = 0 # default value
		if len(template_params) > 3:
			self.options = template_params[3];

		self.rowMajor = (int(self.options) & 0x1)

		self.innerType = self.type.template_argument(0)

		self.val = val

		# Fixed size matrices have a struct as their storage, so we need to walk through this
		self.data = self.val['m_storage']['m_data']
		if self.data.type.code == gdb.TYPE_CODE_STRUCT:
			self.data = self.data['array']
			self.data = self.data.cast(self.innerType.pointer())

	class _iterator:
		def __init__ (self, rows, cols, dataPtr, rowMajor):
			self.rows = rows
			self.cols = cols
			self.dataPtr = dataPtr
			self.currentRow = 0
			self.currentCol = 0
			self.rowMajor = rowMajor

		def __iter__ (self):
			return self

		def next(self):

			row = self.currentRow
			col = self.currentCol
			if self.rowMajor == 0:
				if self.currentCol >= self.cols:
					raise StopIteration

				self.currentRow = self.currentRow + 1
				if self.currentRow >= self.rows:
					self.currentRow = 0
					self.currentCol = self.currentCol + 1
			else:
				if self.currentRow >= self.rows:
					raise StopIteration

				self.currentCol = self.currentCol + 1
				if self.currentCol >= self.cols:
					self.currentCol = 0
					self.currentRow = self.currentRow + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1
			if (self.cols == 1): #if it's a column vector
				return ('[%d]' % (row,), item)
			elif (self.rows == 1): #if it's a row vector
				return ('[%d]' % (col,), item)
			return ('[%d,%d]' % (row, col), item)

	def children(self):

		return self._iterator(self.rows, self.cols, self.data, self.rowMajor)

	def to_string(self):
		return "Eigen::Matrix<%s,%d,%d,%s> (data ptr: %s)" % (self.innerType, self.rows, self.cols, "RowMajor" if self.rowMajor else  "ColMajor", self.data)

class EigenQuaternionPrinter:
	"Print an Eigen Quaternion"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		self.innerType = self.type.template_argument(0)
		self.val = val

		# Quaternions have a struct as their storage, so we need to walk through this
		self.data = self.val['m_coeffs']['m_storage']['m_data']['array']
		self.data = self.data.cast(self.innerType.pointer())

	class _iterator:
		def __init__ (self, dataPtr):
			self.dataPtr = dataPtr
			self.currentElement = 0
			self.elementNames = ['x', 'y', 'z', 'w']

		def __iter__ (self):
			return self

		def next(self):
			element = self.currentElement

			if self.currentElement >= 4: #there are 4 elements in a quanternion
				raise StopIteration

			self.currentElement = self.currentElement + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1
			return ('[%s]' % (self.elementNames[element],), item)

	def children(self):

		return self._iterator(self.data)

	def to_string(self):
		return "Eigen::Quaternion<%s> (data ptr: %s)" % (self.innerType, self.data)

def build_eigen_dictionary ():
	pretty_printers_dict[re.compile('^Eigen::Quaternion<.*>$')] = lambda val: EigenQuaternionPrinter(val)
	pretty_printers_dict[re.compile('^Eigen::Matrix<.*>$')] = lambda val: EigenMatrixPrinter(val)

def register_eigen_printers(obj):
	"Register eigen pretty-printers with objfile Obj"

	if obj == None:
		obj = gdb
	obj.pretty_printers.append(lookup_function)

def lookup_function(val):
	"Look-up and return a pretty-printer that can print va."

	type = val.type

	if type.code == gdb.TYPE_CODE_REF:
		type = type.target()

	type = type.unqualified().strip_typedefs()

	typename = type.tag
	if typename == None:
		return None

	for function in pretty_printers_dict:
		if function.search(typename):
			return pretty_printers_dict[function](val)

	return None

pretty_printers_dict = {}

build_eigen_dictionary ()
<commit_msg>Fix truncated instructions for printers.py ... as noted by kp0987 on forum<commit_after># -*- coding: utf-8 -*-
# This file is part of Eigen, a lightweight C++ template library
# for linear algebra.
#
# Copyright (C) 2009 Benjamin Schindler <bschindler@inf.ethz.ch>
#
# Eigen is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3 of the License, or (at your option) any later version.
#
# Alternatively, you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# Eigen is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License and a copy of the GNU General Public License along with
# Eigen. If not, see <http://www.gnu.org/licenses/>.

# Pretty printers for Eigen::Matrix
# This is still pretty basic as the python extension to gdb is still pretty basic.
# It cannot handle complex eigen types and it doesn't support any of the other eigen types
# Such as quaternion or some other type.
# This code supports fixed size as well as dynamic size matrices

# To use it:
#
# * Create a directory and put the file as well as an empty __init__.py in
#   that directory.
# * Create a ~/.gdbinit file, that contains the following:
#      python
#      import sys
#      sys.path.insert(0, '/path/to/eigen/printer/directory')
#      from printers import register_eigen_printers
#      register_eigen_printers (None)
#      end

import gdb
import re
import itertools

class EigenMatrixPrinter:
	"Print Eigen Matrix of some kind"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		tag = self.type.tag
		regex = re.compile('\<.*\>')
		m = regex.findall(tag)[0][1:-1]
		template_params = m.split(',')
		template_params = map(lambda x:x.replace(" ", ""), template_params)

		if template_params[1] == '-0x00000000000000001':
			self.rows = val['m_storage']['m_rows']
		else:
			self.rows = int(template_params[1])

		if template_params[2] == '-0x00000000000000001':
			self.cols = val['m_storage']['m_cols']
		else:
			self.cols = int(template_params[2])

		self.options = 0 # default value
		if len(template_params) > 3:
			self.options = template_params[3];

		self.rowMajor = (int(self.options) & 0x1)

		self.innerType = self.type.template_argument(0)

		self.val = val

		# Fixed size matrices have a struct as their storage, so we need to walk through this
		self.data = self.val['m_storage']['m_data']
		if self.data.type.code == gdb.TYPE_CODE_STRUCT:
			self.data = self.data['array']
			self.data = self.data.cast(self.innerType.pointer())

	class _iterator:
		def __init__ (self, rows, cols, dataPtr, rowMajor):
			self.rows = rows
			self.cols = cols
			self.dataPtr = dataPtr
			self.currentRow = 0
			self.currentCol = 0
			self.rowMajor = rowMajor

		def __iter__ (self):
			return self

		def next(self):

			row = self.currentRow
			col = self.currentCol
			if self.rowMajor == 0:
				if self.currentCol >= self.cols:
					raise StopIteration

				self.currentRow = self.currentRow + 1
				if self.currentRow >= self.rows:
					self.currentRow = 0
					self.currentCol = self.currentCol + 1
			else:
				if self.currentRow >= self.rows:
					raise StopIteration

				self.currentCol = self.currentCol + 1
				if self.currentCol >= self.cols:
					self.currentCol = 0
					self.currentRow = self.currentRow + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1
			if (self.cols == 1): #if it's a column vector
				return ('[%d]' % (row,), item)
			elif (self.rows == 1): #if it's a row vector
				return ('[%d]' % (col,), item)
			return ('[%d,%d]' % (row, col), item)

	def children(self):

		return self._iterator(self.rows, self.cols, self.data, self.rowMajor)

	def to_string(self):
		return "Eigen::Matrix<%s,%d,%d,%s> (data ptr: %s)" % (self.innerType, self.rows, self.cols, "RowMajor" if self.rowMajor else  "ColMajor", self.data)

class EigenQuaternionPrinter:
	"Print an Eigen Quaternion"

	def __init__(self, val):
		"Extract all the necessary information"
		# The gdb extension does not support value template arguments - need to extract them by hand
		type = val.type
		if type.code == gdb.TYPE_CODE_REF:
			type = type.target()
		self.type = type.unqualified().strip_typedefs()
		self.innerType = self.type.template_argument(0)
		self.val = val

		# Quaternions have a struct as their storage, so we need to walk through this
		self.data = self.val['m_coeffs']['m_storage']['m_data']['array']
		self.data = self.data.cast(self.innerType.pointer())

	class _iterator:
		def __init__ (self, dataPtr):
			self.dataPtr = dataPtr
			self.currentElement = 0
			self.elementNames = ['x', 'y', 'z', 'w']

		def __iter__ (self):
			return self

		def next(self):
			element = self.currentElement

			if self.currentElement >= 4: #there are 4 elements in a quanternion
				raise StopIteration

			self.currentElement = self.currentElement + 1

			item = self.dataPtr.dereference()
			self.dataPtr = self.dataPtr + 1
			return ('[%s]' % (self.elementNames[element],), item)

	def children(self):

		return self._iterator(self.data)

	def to_string(self):
		return "Eigen::Quaternion<%s> (data ptr: %s)" % (self.innerType, self.data)

def build_eigen_dictionary ():
	pretty_printers_dict[re.compile('^Eigen::Quaternion<.*>$')] = lambda val: EigenQuaternionPrinter(val)
	pretty_printers_dict[re.compile('^Eigen::Matrix<.*>$')] = lambda val: EigenMatrixPrinter(val)

def register_eigen_printers(obj):
	"Register eigen pretty-printers with objfile Obj"

	if obj == None:
		obj = gdb
	obj.pretty_printers.append(lookup_function)

def lookup_function(val):
	"Look-up and return a pretty-printer that can print va."

	type = val.type

	if type.code == gdb.TYPE_CODE_REF:
		type = type.target()

	type = type.unqualified().strip_typedefs()

	typename = type.tag
	if typename == None:
		return None

	for function in pretty_printers_dict:
		if function.search(typename):
			return pretty_printers_dict[function](val)

	return None

pretty_printers_dict = {}

build_eigen_dictionary ()
<|endoftext|>


<commit_before>#######################################################################
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
# Permission given to modify the code as long as you keep this        #
# declaration at the top                                              #
#######################################################################

import numpy as np
import os
import re

class Plotter:
    COLORS = ['blue', 'green', 'red', 'black', 'cyan', 'magenta', 'yellow', 'brown', 'purple', 'pink',
              'orange', 'teal', 'coral', 'lightblue', 'lime', 'lavender', 'turquoise',
              'darkgreen', 'tan', 'salmon', 'gold', 'lightpurple', 'darkred', 'darkblue']

    RETURN_TRAIN = 'episodic_return_train'
    RETURN_TEST = 'episodic_return_test'

    def __init__(self):
        pass

    def _rolling_window(self, a, window):
        shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
        strides = a.strides + (a.strides[-1],)
        return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)

    def _window_func(self, x, y, window, func):
        yw = self._rolling_window(y, window)
        yw_func = func(yw, axis=-1)
        return x[window - 1:], yw_func

    def load_results(self, dirs, **kwargs):
        kwargs.setdefault('tag', self.RETURN_TRAIN)
        kwargs.setdefault('right_align', False)
        kwargs.setdefault('window', 0)
        kwargs.setdefault('top_k', 0)
        kwargs.setdefault('top_k_measure', None)
        kwargs.setdefault('interpolation', 100)
        xy_list = self.load_log_dirs(dirs, **kwargs)

        if kwargs['top_k']:
            perf = [kwargs['top_k_measure'](y) for _, y in xy_list]
            top_k_runs = np.argsort(perf)[-kwargs['top_k']:]
            new_xy_list = []
            for r, (x, y) in enumerate(xy_list):
                if r in top_k_runs:
                    new_xy_list.append((x, y))
            xy_list = new_xy_list

        if kwargs['interpolation']:
            x_right = float('inf')
            for x, y in xy_list:
                x_right = min(x_right, x[-1])
            x = np.arange(0, x_right, kwargs['interpolation'])
            y = []
            for x_, y_ in xy_list:
                y.append(np.interp(x, x_, y_))
            y = np.asarray(y)
        else:
            x = xy_list[0][0]
            y = [y for _, y in xy_list]
            x = np.asarray(x)
            y = np.asarray(y)

        return x, y

    def filter_log_dirs(self, pattern, negative_pattern=' ', root='./log', **kwargs):
        dirs = [item[0] for item in os.walk(root)]
        leaf_dirs = []
        for i in range(len(dirs)):
            if i + 1 < len(dirs) and dirs[i + 1].startswith(dirs[i]):
                continue
            leaf_dirs.append(dirs[i])
        names = []
        p = re.compile(pattern)
        np = re.compile(negative_pattern)
        for dir in leaf_dirs:
            if p.match(dir) and not np.match(dir):
                names.append(dir)
                print(dir)
        print('')
        return sorted(names)

    def load_log_dirs(self, dirs, **kwargs):
        kwargs.setdefault('right_align', False)
        kwargs.setdefault('window', 0)
        kwargs.setdefault('right_most', 0)
        xy_list = []
        from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
        for dir in dirs:
            event_acc = EventAccumulator(dir)
            event_acc.Reload()
            _, x, y = zip(*event_acc.Scalars(kwargs['tag']))
            xy_list.append([x, y])
        if kwargs['right_align']:
            x_max = float('inf')
            for x, y in xy_list:
                x_max = min(x_max, len(y))
            xy_list = [[x[:x_max], y[:x_max]] for x, y in xy_list]
        x_max = kwargs['right_most']
        if x_max:
            xy_list = [[x[:x_max], y[:x_max]] for x, y in xy_list]
        if kwargs['window']:
            xy_list = [self._window_func(np.asarray(x), np.asarray(y), kwargs['window'], np.mean) for x, y in xy_list]
        return xy_list

    def plot_mean(self, data, x=None, **kwargs):
        import matplotlib.pyplot as plt
        if x is None:
            x = np.arange(data.shape[1])
        if kwargs['error'] == 'se':
            e_x = np.std(data, axis=0) / np.sqrt(data.shape[0])
        elif kwargs['error'] == 'std':
            e_x = np.std(data, axis=0)
        else:
            raise NotImplementedError
        m_x = np.mean(data, axis=0)
        del kwargs['error']
        plt.plot(x, m_x, **kwargs)
        del kwargs['label']
        plt.fill_between(x, m_x + e_x, m_x - e_x, alpha=0.3, **kwargs)

    def plot_median_std(self, data, x=None, **kwargs):
        import matplotlib.pyplot as plt
        if x is None:
            x = np.arange(data.shape[1])
        e_x = np.std(data, axis=0)
        m_x = np.median(data, axis=0)
        plt.plot(x, m_x, **kwargs)
        del kwargs['label']
        plt.fill_between(x, m_x + e_x, m_x - e_x, alpha=0.3, **kwargs)

    def plot_games(self, games, **kwargs):
        kwargs.setdefault('agg', 'mean')
        import matplotlib.pyplot as plt
        l = len(games)
        plt.figure(figsize=(l * 5, 5))
        for i, game in enumerate(games):
            plt.subplot(1, l, i + 1)
            for j, p in enumerate(kwargs['patterns']):
                label = kwargs['labels'][j]
                color = self.COLORS[j]
                log_dirs = self.filter_log_dirs(pattern='.*%s.*%s' % (game, p), **kwargs)
                x, y = self.load_results(log_dirs, **kwargs)
                if kwargs['downsample']:
                    indices = np.linspace(0, len(x) - 1, kwargs['downsample']).astype(np.int)
                    x = x[indices]
                    y = y[:, indices]
                if kwargs['agg'] == 'mean':
                    self.plot_mean(y, x, label=label, color=color, error='se')
                elif kwargs['agg'] == 'mean_std':
                    self.plot_mean(y, x, label=label, color=color, error='std')
                elif kwargs['agg'] == 'median':
                    self.plot_median_std(y, x, label=label, color=color)
                else:
                    for k in range(y.shape[0]):
                        plt.plot(x, y[i], label=label, color=color)
                        label = None
            plt.xlabel('steps')
            if not i:
                plt.ylabel(kwargs['tag'])
            plt.title(game)
            plt.legend()

    def select_best_parameters(self, patterns, **kwargs):
        scores = []
        for pattern in patterns:
            log_dirs = self.filter_log_dirs(pattern, **kwargs)
            xy_list = self.load_log_dirs(log_dirs, **kwargs)
            y = np.asarray([xy[1] for xy in xy_list])
            scores.append(kwargs['score'](y))
        indices = np.argsort(-np.asarray(scores))
        return indices

    def reduce_dir(self, root, tag, ids, score_fn):
        tf_log_info = {}
        for dir, _, files in os.walk(root):
            for file in files:
                if 'tfevents' in file:
                    dir = os.path.basename(dir)
                    dir = re.sub(r'hp_\d+', 'placeholder', dir)
                    dir = re.sub(r'run.*', 'run', dir)
                    tf_log_info[dir] = {}
        for key in tf_log_info.keys():
            scores = []
            for id in ids:
                dir = key.replace('placeholder', 'hp_%s' % (id))
                names = self.filter_log_dirs('.*%s.*' % (dir), root=root)
                xy_list = self.load_log_dirs(names, tag=tag, right_align=True)
                scores.append(score_fn(np.asarray([y for x, y in xy_list])))
            best = np.nanargmax(scores)
            tf_log_info[key]['hp'] = ids[best]
            tf_log_info[key]['score'] = scores[best]
        return tf_log_info

    def reduce_patterns(self, patterns, root, tag, ids, score_fn):
        new_patterns = []
        best_ids = []
        for pattern in patterns:
            scores = []
            pattern = re.sub(r'hp_\d+', 'placeholder', pattern)
            ps = []
            for id in ids:
                p = pattern.replace('placeholder', 'hp_%s' % (id))
                ps.append(p)
                names = self.filter_log_dirs('.*%s.*' % (p), root=root)
                xy_list = self.load_log_dirs(names, tag=tag, right_align=True)
                scores.append(score_fn(np.asarray([y for x, y in xy_list])))
            try:
                best = np.nanargmax(scores)
            except ValueError as e:
                print(e)
                best = 0
            best_ids.append(best)
            new_patterns.append(ps[best])
        return dict(patterns=patterns, ids=best_ids)

<commit_msg>Improve hyperparameter search<commit_after>#######################################################################
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
# Permission given to modify the code as long as you keep this        #
# declaration at the top                                              #
#######################################################################

import numpy as np
import os
import re

class Plotter:
    COLORS = ['blue', 'green', 'red', 'black', 'cyan', 'magenta', 'yellow', 'brown', 'purple', 'pink',
              'orange', 'teal', 'coral', 'lightblue', 'lime', 'lavender', 'turquoise',
              'darkgreen', 'tan', 'salmon', 'gold', 'lightpurple', 'darkred', 'darkblue']

    RETURN_TRAIN = 'episodic_return_train'
    RETURN_TEST = 'episodic_return_test'

    def __init__(self):
        pass

    def _rolling_window(self, a, window):
        shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
        strides = a.strides + (a.strides[-1],)
        return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)

    def _window_func(self, x, y, window, func):
        yw = self._rolling_window(y, window)
        yw_func = func(yw, axis=-1)
        return x[window - 1:], yw_func

    def load_results(self, dirs, **kwargs):
        kwargs.setdefault('tag', self.RETURN_TRAIN)
        kwargs.setdefault('right_align', False)
        kwargs.setdefault('window', 0)
        kwargs.setdefault('top_k', 0)
        kwargs.setdefault('top_k_measure', None)
        kwargs.setdefault('interpolation', 100)
        xy_list = self.load_log_dirs(dirs, **kwargs)

        if kwargs['top_k']:
            perf = [kwargs['top_k_measure'](y) for _, y in xy_list]
            top_k_runs = np.argsort(perf)[-kwargs['top_k']:]
            new_xy_list = []
            for r, (x, y) in enumerate(xy_list):
                if r in top_k_runs:
                    new_xy_list.append((x, y))
            xy_list = new_xy_list

        if kwargs['interpolation']:
            x_right = float('inf')
            for x, y in xy_list:
                x_right = min(x_right, x[-1])
            x = np.arange(0, x_right, kwargs['interpolation'])
            y = []
            for x_, y_ in xy_list:
                y.append(np.interp(x, x_, y_))
            y = np.asarray(y)
        else:
            x = xy_list[0][0]
            y = [y for _, y in xy_list]
            x = np.asarray(x)
            y = np.asarray(y)

        return x, y

    def filter_log_dirs(self, pattern, negative_pattern=' ', root='./log', **kwargs):
        dirs = [item[0] for item in os.walk(root)]
        leaf_dirs = []
        for i in range(len(dirs)):
            if i + 1 < len(dirs) and dirs[i + 1].startswith(dirs[i]):
                continue
            leaf_dirs.append(dirs[i])
        names = []
        p = re.compile(pattern)
        np = re.compile(negative_pattern)
        for dir in leaf_dirs:
            if p.match(dir) and not np.match(dir):
                names.append(dir)
                print(dir)
        print('')
        return sorted(names)

    def load_log_dirs(self, dirs, **kwargs):
        kwargs.setdefault('right_align', False)
        kwargs.setdefault('window', 0)
        kwargs.setdefault('right_most', 0)
        xy_list = []
        from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
        for dir in dirs:
            event_acc = EventAccumulator(dir)
            event_acc.Reload()
            _, x, y = zip(*event_acc.Scalars(kwargs['tag']))
            xy_list.append([x, y])
        if kwargs['right_align']:
            x_max = float('inf')
            for x, y in xy_list:
                x_max = min(x_max, len(y))
            xy_list = [[x[:x_max], y[:x_max]] for x, y in xy_list]
        x_max = kwargs['right_most']
        if x_max:
            xy_list = [[x[:x_max], y[:x_max]] for x, y in xy_list]
        if kwargs['window']:
            xy_list = [self._window_func(np.asarray(x), np.asarray(y), kwargs['window'], np.mean) for x, y in xy_list]
        return xy_list

    def plot_mean(self, data, x=None, **kwargs):
        import matplotlib.pyplot as plt
        if x is None:
            x = np.arange(data.shape[1])
        if kwargs['error'] == 'se':
            e_x = np.std(data, axis=0) / np.sqrt(data.shape[0])
        elif kwargs['error'] == 'std':
            e_x = np.std(data, axis=0)
        else:
            raise NotImplementedError
        m_x = np.mean(data, axis=0)
        del kwargs['error']
        plt.plot(x, m_x, **kwargs)
        del kwargs['label']
        plt.fill_between(x, m_x + e_x, m_x - e_x, alpha=0.3, **kwargs)

    def plot_median_std(self, data, x=None, **kwargs):
        import matplotlib.pyplot as plt
        if x is None:
            x = np.arange(data.shape[1])
        e_x = np.std(data, axis=0)
        m_x = np.median(data, axis=0)
        plt.plot(x, m_x, **kwargs)
        del kwargs['label']
        plt.fill_between(x, m_x + e_x, m_x - e_x, alpha=0.3, **kwargs)

    def plot_games(self, games, **kwargs):
        kwargs.setdefault('agg', 'mean')
        import matplotlib.pyplot as plt
        l = len(games)
        plt.figure(figsize=(l * 5, 5))
        for i, game in enumerate(games):
            plt.subplot(1, l, i + 1)
            for j, p in enumerate(kwargs['patterns']):
                label = kwargs['labels'][j]
                color = self.COLORS[j]
                log_dirs = self.filter_log_dirs(pattern='.*%s.*%s' % (game, p), **kwargs)
                x, y = self.load_results(log_dirs, **kwargs)
                if kwargs['downsample']:
                    indices = np.linspace(0, len(x) - 1, kwargs['downsample']).astype(np.int)
                    x = x[indices]
                    y = y[:, indices]
                if kwargs['agg'] == 'mean':
                    self.plot_mean(y, x, label=label, color=color, error='se')
                elif kwargs['agg'] == 'mean_std':
                    self.plot_mean(y, x, label=label, color=color, error='std')
                elif kwargs['agg'] == 'median':
                    self.plot_median_std(y, x, label=label, color=color)
                else:
                    for k in range(y.shape[0]):
                        plt.plot(x, y[i], label=label, color=color)
                        label = None
            plt.xlabel('steps')
            if not i:
                plt.ylabel(kwargs['tag'])
            plt.title(game)
            plt.legend()

    def select_best_parameters(self, patterns, **kwargs):
        scores = []
        for pattern in patterns:
            log_dirs = self.filter_log_dirs(pattern, **kwargs)
            xy_list = self.load_log_dirs(log_dirs, **kwargs)
            y = np.asarray([xy[1] for xy in xy_list])
            scores.append(kwargs['score'](y))
        indices = np.argsort(-np.asarray(scores))
        return indices

    def reduce_dir(self, root, tag, ids, score_fn):
        tf_log_info = {}
        for dir, _, files in os.walk(root):
            for file in files:
                if 'tfevents' in file:
                    dir = os.path.basename(dir)
                    dir = re.sub(r'hp_\d+', 'placeholder', dir)
                    dir = re.sub(r'run.*', 'run', dir)
                    tf_log_info[dir] = {}
        for key in tf_log_info.keys():
            scores = []
            for id in ids:
                dir = key.replace('placeholder', 'hp_%s' % (id))
                names = self.filter_log_dirs('.*%s.*' % (dir), root=root)
                xy_list = self.load_log_dirs(names, tag=tag, right_align=True)
                scores.append(score_fn(np.asarray([y for x, y in xy_list])))
            best = np.nanargmax(scores)
            tf_log_info[key]['hp'] = ids[best]
            tf_log_info[key]['score'] = scores[best]
        return tf_log_info

    def reduce_patterns(self, patterns, root, tag, ids, score_fn):
        new_patterns = []
        best_ids = []
        for pattern in patterns:
            scores = []
            pattern = re.sub(r'hp_\d+', 'placeholder', pattern)
            ps = []
            for id in ids:
                p = pattern.replace('placeholder', 'hp_%s' % (id))
                ps.append(p)
                names = self.filter_log_dirs('.*%s.*' % (p), root=root)
                xy_list = self.load_log_dirs(names, tag=tag, right_align=True)
                scores.append(score_fn(np.asarray([y for x, y in xy_list])))
            try:
                best = np.nanargmax(scores)
            except ValueError as e:
                print(e)
                best = 0
            best_ids.append(best)
            new_patterns.append(ps[best])
        return dict(patterns=new_patterns, ids=best_ids)

<|endoftext|>


<commit_before>import numpy as np
import cudarray as ca
from .base import Unary

class Reduce(Unary):
    def __init__(self, axis=None, keepdims=False):
        self.axis = axis
        self.keepdims = keepdims

    def setup(self):
        self.shape = ca.sum(self.x.array, axis=self.axis,
                            keepdims=self.keepdims).shape
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

class Mean(Reduce):
    def setup(self):
        super(Mean, self).setup()
        self.n = np.prod(self.x.shape)

    def fprop(self):
        ca.mean(self.x.array, axis=self.axis, out=self.array,
                keepdims=self.keepdims)

    def bprop(self):
        self.x.grad_array.fill(1.0/self.n)
        self.x.grad_array *= self.grad_array

class Sum(Reduce):
    def fprop(self):
        ca.sum(self.x.array, axis=self.axis, out=self.array,
               keepdims=self.keepdims)

    def bprop(self):
        self.x.grad_array.fill(1.0)
        self.x.grad_array *= self.grad_array

def mean(x, axis=None, keepdims=False):
    return Mean(axis, keepdims)(x)

def sum(x, axis=None, keepdims=False):
    return Sum(axis, keepdims)(x)
<commit_msg>fix Mean<commit_after>import numpy as np
import cudarray as ca
from .base import Unary

class Reduce(Unary):
    def __init__(self, axis=None, keepdims=False):
        self.axis = axis
        self.keepdims = keepdims

    def setup(self):
        self.shape = ca.sum(self.x.array, axis=self.axis,
                            keepdims=self.keepdims).shape
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

class Mean(Reduce):
    def setup(self):
        super(Mean, self).setup()
        self.n = np.prod(self.x.shape)
        self.scale = 1.0/np.prod(self.x.shape[self.axis])

    def fprop(self):
        ca.mean(self.x.array, axis=self.axis, out=self.array,
                keepdims=self.keepdims)

    def bprop(self):
        self.x.grad_array.fill(self.scale)
        self.x.grad_array *= self.grad_array

class Sum(Reduce):
    def fprop(self):
        ca.sum(self.x.array, axis=self.axis, out=self.array,
               keepdims=self.keepdims)

    def bprop(self):
        self.x.grad_array.fill(1.0)
        self.x.grad_array *= self.grad_array

def mean(x, axis=None, keepdims=False):
    return Mean(axis, keepdims)(x)

def sum(x, axis=None, keepdims=False):
    return Sum(axis, keepdims)(x)
<|endoftext|>