<commit_before>from globals import *

import profiles
import items

import logging
import json

def _capsule():
	return {'uid': None,
	        'date': WORLD_INFO['ticks'],
	        '_on_disk': False,
	        '_allow_dump': False}

def _write_to_cache(cache_name, data):
	with open(os.path.join(profiles.get_world_directory(WORLD_INFO['id']), '%s_history.dat' % cache_name), 'a') as f:
		f.write(json.dumps(data)+'\n')

def _read_from_cache(cache_name, uid):
	with open(os.path.join(profiles.get_world_directory(WORLD_INFO['id']), '%s_history.dat' % cache_name)) as f:
		for item in [json.loads(s) for s in f]:
			if item['uid'] == uid:
				item['_on_disk'] = False
				item['date'] = WORLD_INFO['ticks']
				ITEMS_HISTORY[item['uid']].update(item)

				logging.debug('Cache: Loaded item %s from cache.' % uid)
				return item

def save_cache(cache_name):
	_path = os.path.join(profiles.get_world_directory(WORLD_INFO['id']), '%s_history.dat' % cache_name)
	_write_cache = []

	if not os.path.exists(_path):
		return False

	with open(_path, 'r') as f:
		_cache = f.readlines()

		for line in _cache:
			_historic_item = json.loads(line.strip())

			if not _historic_item['_allow_dump']:
				continue

			_write_cache.append(json.dumps(_historic_item))

	with open(_path, 'w') as f:
		f.write('\n'.join(_write_cache))

	logging.debug('Cache: Saved to disk.')

def commit_cache(cache_name):
	_path = os.path.join(profiles.get_world_directory(WORLD_INFO['id']), '%s_history.dat' % cache_name)
	_write_cache = []

	if not os.path.exists(_path):
		return False

	with open(_path, 'r') as f:
		_cache = f.readlines()

		for line in _cache:
			_historic_item = json.loads(line.strip())
			_historic_item['_allow_dump'] = True
			_write_cache.append(json.dumps(_historic_item))

	if cache_name == 'items':
		for item_uid in ITEMS_HISTORY:
			_historic_item = ITEMS_HISTORY[item_uid]
			if not _historic_item['_on_disk']:
				continue

			_historic_item['_allow_dump'] = True

	with open(_path, 'w') as f:
		f.write('\n'.join(_write_cache))

	logging.debug('Cache: Committed.')

def scan_cache():
	for item_uid in ITEMS_HISTORY:
		_historic_item = ITEMS_HISTORY[item_uid]

		if _historic_item['_on_disk']:
			continue

		if WORLD_INFO['ticks']-_historic_item['date']>=100:
			logging.debug('Cache: Moved item %s to disk' % item_uid)
			_historic_item['_on_disk'] = True
			_write_to_cache('items', _historic_item)

			if 'item' in _historic_item:
				del _historic_item['item']

def offload_item(raw_item):
	_item = _capsule()
	items.clean_item_for_save(raw_item)
	_item['uid'] = raw_item['uid']
	_item['item'] = raw_item

	ITEMS_HISTORY[_item['uid']] = _item

	logging.debug('Cache: Offloaded item (in memory)')

def retrieve_item(item_uid):
	_historic_item = ITEMS_HISTORY[item_uid]

	if _historic_item['_on_disk']:
		raise Exception('Ahhhhhhhhhhhh')
		#TODO: Grab from disk
		pass

	return _historic_item['item']<commit_msg>Cache work<commit_after>from globals import *

import profiles
import items

import logging
import json

def _capsule():
	return {'uid': None,
	        'date': WORLD_INFO['ticks'],
	        '_on_disk': False,
	        '_allow_dump': False}

def _write_to_cache(cache_name, data):
	with open(os.path.join(profiles.get_world_directory(WORLD_INFO['id']), '%s_history.dat' % cache_name), 'a') as f:
		f.write(json.dumps(data)+'\n')

def _read_from_cache(cache_name, uid):
	with open(os.path.join(profiles.get_world_directory(WORLD_INFO['id']), '%s_history.dat' % cache_name)) as f:
		for item in [json.loads(s) for s in f]:
			if item['uid'] == uid:
				item['_on_disk'] = False
				item['date'] = WORLD_INFO['ticks']
				ITEMS_HISTORY[item['uid']].update(item)

				logging.debug('Cache: Loaded item %s from cache.' % uid)
				return item

def save_cache(cache_name):
	_path = os.path.join(profiles.get_world_directory(WORLD_INFO['id']), '%s_history.dat' % cache_name)
	_write_cache = []

	if not os.path.exists(_path):
		return False

	with open(_path, 'r') as f:
		_cache = f.readlines()

		for line in _cache:
			if line.endswith('\n'):
				line.rstrip()

			print repr(line)
			_historic_item = json.loads(line)

			if not _historic_item['_allow_dump']:
				continue

			_write_cache.append(json.dumps(_historic_item))

	with open(_path, 'w') as f:
		f.write('\n'.join(_write_cache))

	logging.debug('Cache: Saved to disk.')

def commit_cache(cache_name):
	_path = os.path.join(profiles.get_world_directory(WORLD_INFO['id']), '%s_history.dat' % cache_name)
	_write_cache = []

	if not os.path.exists(_path):
		return False

	with open(_path, 'r') as f:
		_cache = f.readlines()

		for line in _cache:
			print repr(line)
			_historic_item = json.loads(line.strip())
			_historic_item['_allow_dump'] = True
			_write_cache.append(json.dumps(_historic_item))

	if cache_name == 'items':
		for item_uid in ITEMS_HISTORY:
			_historic_item = ITEMS_HISTORY[item_uid]
			if not _historic_item['_on_disk']:
				continue

			_historic_item['_allow_dump'] = True

	with open(_path, 'w') as f:
		f.write('\n'.join(_write_cache))

	logging.debug('Cache: Committed.')

def scan_cache():
	for item_uid in ITEMS_HISTORY:
		_historic_item = ITEMS_HISTORY[item_uid]

		if _historic_item['_on_disk']:
			continue

		if WORLD_INFO['ticks']-_historic_item['date']>=100:
			logging.debug('Cache: Moved item %s to disk' % item_uid)
			_historic_item['_on_disk'] = True
			_write_to_cache('items', _historic_item)

			if 'item' in _historic_item:
				del _historic_item['item']

def offload_item(raw_item):
	_item = _capsule()
	items.clean_item_for_save(raw_item)
	_item['uid'] = raw_item['uid']
	_item['item'] = raw_item

	ITEMS_HISTORY[_item['uid']] = _item

	logging.debug('Cache: Offloaded item (in memory)')

def retrieve_item(item_uid):
	_historic_item = ITEMS_HISTORY[item_uid]

	if _historic_item['_on_disk']:
		raise Exception('Ahhhhhhhhhhhh')
		#TODO: Grab from disk
		pass

	return _historic_item['item']<|endoftext|>


<commit_before>import logging

from angr.procedures.stubs.format_parser import FormatParser

l = logging.getLogger("angr.procedures.libc.printf")

######################################
# _printf
######################################

class printf(FormatParser):

    def run(self):
        # The format str is at index 0
        fmt_str = self._parse(0)
        out_str = fmt_str.replace(1, self.arg)

        self.state.posix.write(1, out_str, out_str.size() / 8)

        # This function returns
        # Add another exit to the retn_addr that is at the top of the stack now
        return out_str.size() / 8
        # l.debug("Got return address for %s: 0x%08x.", __file__, self._exits[0].concretize())
<commit_msg>Add __printf_chk simprocedure<commit_after>import logging

from angr.procedures.stubs.format_parser import FormatParser

l = logging.getLogger("angr.procedures.libc.printf")

class printf(FormatParser):

    def run(self):
        # The format str is at index 0
        fmt_str = self._parse(0)
        out_str = fmt_str.replace(1, self.arg)

        self.state.posix.write(1, out_str, out_str.size() / 8)

        # This function returns
        # Add another exit to the retn_addr that is at the top of the stack now
        return out_str.size() / 8
        # l.debug("Got return address for %s: 0x%08x.", __file__, self._exits[0].concretize())

class __printf_chk(FormatParser):

    def run(self):
        # The format str is at index 1
        fmt_str = self._parse(1)
        out_str = fmt_str.replace(2, self.arg)

        self.state.posix.write(1, out_str, out_str.size() / 8)

        return out_str.size() / 8
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright 2016 Red Hat, Inc.
#
# Authors:
#     Fam Zheng <famz@redhat.com>
#
# This work is licensed under the MIT License.  Please see the LICENSE file or
# http://opensource.org/licenses/MIT.

from mod import PatchewModule
from mbox import parse_address
from event import register_handler, emit_event, declare_event
from api.models import Message

REV_BY_PREFIX = "Reviewed-by:"

_default_config = """
[default]
tags = Reviewed-by, Tested-by, Reported-by, Acked-by, Suggested-by

"""

class SeriesTagsModule(PatchewModule):
    """

Documentation
-------------

This module is configured in "INI" style.

It has only one section named `[default]`. The only supported option is tags:

    [default]
    tags = Reviewed-by, Tested-by, Reported-by, Acked-by, Suggested-by

The `tags` option contains the tag line prefixes (must be followed by colon)
that should be treated as meaningful patch status tags, and picked up from
series cover letter, patch mail body and their replies.

"""
    name = "tags"
    default_config = _default_config

    def __init__(self):
        register_handler("MessageAdded", self.on_message_added)
        declare_event("TagsUpdate", series="message object that is updated")

        # XXX: get this list through module config?
    def get_tag_prefixes(self):
        return [x.strip() for x in self.get_config("default", "tags").split(",")]

    def update_tags(self, s):
        old = s.get_property("tags", [])
        new = self.get_tags(s)
        if set(old) != set(new):
            s.set_property("tags", list(set(new)))
            return True

    def on_message_added(self, event, message):
        series = message.get_series_head()
        if not series:
            return

        for m in series.get_alternative_revisions():
            if m.version > series.version:
                series.set_property("obsoleted-by", m.message_id)
            elif m.version < series.version:
                m.set_property("obsoleted-by", series.message_id)

        updated = self.update_tags(series)

        for p in series.get_patches():
            updated = updated or self.update_tags(p)

        reviewers = set()
        num_reviewed = 0
        def _find_reviewers(what):
            ret = set()
            for rev_tag in [x for x in what.get_property("tags", []) if x.lower().startswith(REV_BY_PREFIX.lower())]:
                ret.add(parse_address(rev_tag[len(REV_BY_PREFIX):]))
            return ret
        for p in series.get_patches():
            first = True
            this_reviewers = _find_reviewers(p)
            if this_reviewers:
                if first:
                    num_reviewed += 1
                    first = False
                reviewers = reviewers.union(this_reviewers)
        series_reviewers = _find_reviewers(series)
        reviewers = reviewers.union(series_reviewers)
        if (num_reviewed, num_reviewed) == series.get_num() or series_reviewers:
            series.set_property("reviewed", True)
            series.set_property("reviewers", list(reviewers))
        if updated:
            emit_event("TagsUpdate", series=series)

    def parse_message_tags(self, m):
        r = []
        for l in m.get_body().splitlines():
            for p in self.get_tag_prefixes():
                if l.lower().startswith(p.lower()):
                    r.append(l)
        return r

    def get_tags(self, m):
        # Incorporate tags from non-patch replies
        r = self.parse_message_tags(m)
        for x in m.get_replies():
            if x.is_patch:
                continue
            r += self.get_tags(x)
        return r

    def prepare_message_hook(self, request, message):
        if not message.is_series_head:
            return
        if message.get_property("reviewed"):
            reviewers = message.get_property("reviewers")
            message.status_tags.append({
                "title": "Reviewed by " + ", ".join([x for x, y in reviewers]),
                "type": "success",
                "char": "R",
                })
        ob = message.get_property("obsoleted-by")
        if ob:
            new = Message.objects.find_series(ob, message.project.name)
            message.status_tags.append({
                "title": "Has a newer version: " + new.subject,
                "type": "default",
                "char": "O",
                })

<commit_msg>tags: Set default in getting config<commit_after>#!/usr/bin/env python3
#
# Copyright 2016 Red Hat, Inc.
#
# Authors:
#     Fam Zheng <famz@redhat.com>
#
# This work is licensed under the MIT License.  Please see the LICENSE file or
# http://opensource.org/licenses/MIT.

from mod import PatchewModule
from mbox import parse_address
from event import register_handler, emit_event, declare_event
from api.models import Message

REV_BY_PREFIX = "Reviewed-by:"

_default_config = """
[default]
tags = Reviewed-by, Tested-by, Reported-by, Acked-by, Suggested-by

"""

class SeriesTagsModule(PatchewModule):
    """

Documentation
-------------

This module is configured in "INI" style.

It has only one section named `[default]`. The only supported option is tags:

    [default]
    tags = Reviewed-by, Tested-by, Reported-by, Acked-by, Suggested-by

The `tags` option contains the tag line prefixes (must be followed by colon)
that should be treated as meaningful patch status tags, and picked up from
series cover letter, patch mail body and their replies.

"""
    name = "tags"
    default_config = _default_config

    def __init__(self):
        register_handler("MessageAdded", self.on_message_added)
        declare_event("TagsUpdate", series="message object that is updated")

        # XXX: get this list through module config?
    def get_tag_prefixes(self):
        tagsconfig = self.get_config("default", "tags", default="")
        return [x.strip() for x in tagsconfig.split(",")]

    def update_tags(self, s):
        old = s.get_property("tags", [])
        new = self.get_tags(s)
        if set(old) != set(new):
            s.set_property("tags", list(set(new)))
            return True

    def on_message_added(self, event, message):
        series = message.get_series_head()
        if not series:
            return

        for m in series.get_alternative_revisions():
            if m.version > series.version:
                series.set_property("obsoleted-by", m.message_id)
            elif m.version < series.version:
                m.set_property("obsoleted-by", series.message_id)

        updated = self.update_tags(series)

        for p in series.get_patches():
            updated = updated or self.update_tags(p)

        reviewers = set()
        num_reviewed = 0
        def _find_reviewers(what):
            ret = set()
            for rev_tag in [x for x in what.get_property("tags", []) if x.lower().startswith(REV_BY_PREFIX.lower())]:
                ret.add(parse_address(rev_tag[len(REV_BY_PREFIX):]))
            return ret
        for p in series.get_patches():
            first = True
            this_reviewers = _find_reviewers(p)
            if this_reviewers:
                if first:
                    num_reviewed += 1
                    first = False
                reviewers = reviewers.union(this_reviewers)
        series_reviewers = _find_reviewers(series)
        reviewers = reviewers.union(series_reviewers)
        if (num_reviewed, num_reviewed) == series.get_num() or series_reviewers:
            series.set_property("reviewed", True)
            series.set_property("reviewers", list(reviewers))
        if updated:
            emit_event("TagsUpdate", series=series)

    def parse_message_tags(self, m):
        r = []
        for l in m.get_body().splitlines():
            for p in self.get_tag_prefixes():
                if l.lower().startswith(p.lower()):
                    r.append(l)
        return r

    def get_tags(self, m):
        # Incorporate tags from non-patch replies
        r = self.parse_message_tags(m)
        for x in m.get_replies():
            if x.is_patch:
                continue
            r += self.get_tags(x)
        return r

    def prepare_message_hook(self, request, message):
        if not message.is_series_head:
            return
        if message.get_property("reviewed"):
            reviewers = message.get_property("reviewers")
            message.status_tags.append({
                "title": "Reviewed by " + ", ".join([x for x, y in reviewers]),
                "type": "success",
                "char": "R",
                })
        ob = message.get_property("obsoleted-by")
        if ob:
            new = Message.objects.find_series(ob, message.project.name)
            message.status_tags.append({
                "title": "Has a newer version: " + new.subject,
                "type": "default",
                "char": "O",
                })

<|endoftext|>


<commit_before>"""Parser.py: a main for invoking code in coverage/parser.py"""

from __future__ import division

import glob, os, sys
import collections
from optparse import OptionParser

import disgen

from coverage.misc import CoverageException
from coverage.parser import ByteParser, PythonParser
from coverage.python import get_python_source

opcode_counts = collections.Counter()

class ParserMain(object):
    """A main for code parsing experiments."""

    def main(self, args):
        """A main function for trying the code from the command line."""

        parser = OptionParser()
        parser.add_option(
            "-c", action="store_true", dest="chunks",
            help="Show basic block chunks"
            )
        parser.add_option(
            "-d", action="store_true", dest="dis",
            help="Disassemble"
            )
        parser.add_option(
            "-H", action="store_true", dest="histogram",
            help="Count occurrences of opcodes"
            )
        parser.add_option(
            "-R", action="store_true", dest="recursive",
            help="Recurse to find source files"
            )
        parser.add_option(
            "-s", action="store_true", dest="source",
            help="Show analyzed source"
            )
        parser.add_option(
            "-t", action="store_true", dest="tokens",
            help="Show tokens"
            )

        options, args = parser.parse_args()
        if options.recursive:
            if args:
                root = args[0]
            else:
                root = "."
            for root, _, _ in os.walk(root):
                for f in glob.glob(root + "/*.py"):
                    self.one_file(options, f)
        elif not args:
            parser.print_help()
        else:
            self.one_file(options, args[0])

        if options.histogram:
            total = sum(opcode_counts.values())
            print("{} total opcodes".format(total))
            for opcode, number in opcode_counts.most_common():
                print("{:20s} {:6d}  {:.1%}".format(opcode, number, number/total))

    def one_file(self, options, filename):
        """Process just one file."""

        try:
            text = get_python_source(filename)
            bp = ByteParser(text, filename=filename)
        except Exception as err:
            print("%s" % (err,))
            return

        if options.dis:
            print("Main code:")
            self.disassemble(bp, histogram=options.histogram)

        arcs = bp._all_arcs()
        if options.chunks:# and not options.dis:
            chunks = bp._all_chunks()
            if options.recursive:
                print("%6d: %s" % (len(chunks), filename))
            else:
                print("Chunks: %r" % chunks)
                print("Arcs: %r" % sorted(arcs))

        if options.source or options.tokens:
            cp = PythonParser(filename=filename, exclude=r"no\s*cover")
            cp.show_tokens = options.tokens
            cp._raw_parse()

            if options.source:
                if options.chunks:
                    arc_width, arc_chars = self.arc_ascii_art(arcs)
                else:
                    arc_width, arc_chars = 0, {}

                exit_counts = cp.exit_counts()

                for lineno, ltext in enumerate(cp.lines, start=1):
                    m0 = m1 = m2 = m3 = a = ' '
                    if lineno in cp.statement_starts:
                        m0 = '-'
                    exits = exit_counts.get(lineno, 0)
                    if exits > 1:
                        m1 = str(exits)
                    if lineno in cp.docstrings:
                        m2 = '"'
                    if lineno in cp.classdefs:
                        m2 = 'C'
                    if lineno in cp.excluded:
                        m3 = 'x'
                    a = arc_chars.get(lineno, '').ljust(arc_width)
                    print("%4d %s%s%s%s%s %s" %
                                (lineno, m0, m1, m2, m3, a, ltext)
                        )

    def disassemble(self, byte_parser, histogram=False):
        """Disassemble code, for ad-hoc experimenting."""

        for bp in byte_parser.child_parsers():
            chunks = bp._split_into_chunks()
            chunkd = dict((chunk.byte, chunk) for chunk in chunks)
            if bp.text:
                srclines = bp.text.splitlines()
            else:
                srclines = None
            print("\n%s: " % bp.code)
            upto = None
            for disline in disgen.disgen(bp.code):
                if histogram:
                    opcode_counts[disline.opcode] += 1
                    continue
                if disline.first:
                    if srclines:
                        upto = upto or disline.lineno-1
                        while upto <= disline.lineno-1:
                            print("%100s%s" % ("", srclines[upto]))
                            upto += 1
                    elif disline.offset > 0:
                        print("")
                line = disgen.format_dis_line(disline)
                chunk = chunkd.get(disline.offset)
                if chunk:
                    chunkstr = ":: %r" % chunk
                else:
                    chunkstr = ""
                print("%-70s%s" % (line, chunkstr))

        print("")

    def arc_ascii_art(self, arcs):
        """Draw arcs as ascii art.

        Returns a width of characters needed to draw all the arcs, and a
        dictionary mapping line numbers to ascii strings to draw for that line.

        """
        arc_chars = {}
        for lfrom, lto in sorted(arcs):
            if lfrom < 0:
                arc_chars[lto] = arc_chars.get(lto, '') + 'v'
            elif lto < 0:
                arc_chars[lfrom] = arc_chars.get(lfrom, '') + '^'
            else:
                if lfrom == lto - 1:
                    # Don't show obvious arcs.
                    continue
                if lfrom < lto:
                    l1, l2 = lfrom, lto
                else:
                    l1, l2 = lto, lfrom
                w = max([len(arc_chars.get(l, '')) for l in range(l1, l2+1)])
                for l in range(l1, l2+1):
                    if l == lfrom:
                        ch = '<'
                    elif l == lto:
                        ch = '>'
                    else:
                        ch = '|'
                    arc_chars[l] = arc_chars.get(l, '').ljust(w) + ch
                arc_width = 0

        if arc_chars:
            arc_width = max([len(a) for a in arc_chars.values()])
        else:
            arc_width = 0

        return arc_width, arc_chars

if __name__ == '__main__':
    ParserMain().main(sys.argv[1:])

<commit_msg>Modernize the implementation of ascii-art arcs in lab/parser.py<commit_after>"""Parser.py: a main for invoking code in coverage/parser.py"""

from __future__ import division

import glob, os, sys
import collections
from optparse import OptionParser

import disgen

from coverage.misc import CoverageException
from coverage.parser import ByteParser, PythonParser
from coverage.python import get_python_source

opcode_counts = collections.Counter()

class ParserMain(object):
    """A main for code parsing experiments."""

    def main(self, args):
        """A main function for trying the code from the command line."""

        parser = OptionParser()
        parser.add_option(
            "-c", action="store_true", dest="chunks",
            help="Show basic block chunks"
            )
        parser.add_option(
            "-d", action="store_true", dest="dis",
            help="Disassemble"
            )
        parser.add_option(
            "-H", action="store_true", dest="histogram",
            help="Count occurrences of opcodes"
            )
        parser.add_option(
            "-R", action="store_true", dest="recursive",
            help="Recurse to find source files"
            )
        parser.add_option(
            "-s", action="store_true", dest="source",
            help="Show analyzed source"
            )
        parser.add_option(
            "-t", action="store_true", dest="tokens",
            help="Show tokens"
            )

        options, args = parser.parse_args()
        if options.recursive:
            if args:
                root = args[0]
            else:
                root = "."
            for root, _, _ in os.walk(root):
                for f in glob.glob(root + "/*.py"):
                    self.one_file(options, f)
        elif not args:
            parser.print_help()
        else:
            self.one_file(options, args[0])

        if options.histogram:
            total = sum(opcode_counts.values())
            print("{} total opcodes".format(total))
            for opcode, number in opcode_counts.most_common():
                print("{:20s} {:6d}  {:.1%}".format(opcode, number, number/total))

    def one_file(self, options, filename):
        """Process just one file."""

        try:
            text = get_python_source(filename)
            bp = ByteParser(text, filename=filename)
        except Exception as err:
            print("%s" % (err,))
            return

        if options.dis:
            print("Main code:")
            self.disassemble(bp, histogram=options.histogram)

        arcs = bp._all_arcs()
        if options.chunks:# and not options.dis:
            chunks = bp._all_chunks()
            if options.recursive:
                print("%6d: %s" % (len(chunks), filename))
            else:
                print("Chunks: %r" % chunks)
                print("Arcs: %r" % sorted(arcs))

        if options.source or options.tokens:
            cp = PythonParser(filename=filename, exclude=r"no\s*cover")
            cp.show_tokens = options.tokens
            cp._raw_parse()

            if options.source:
                if options.chunks:
                    arc_width, arc_chars = self.arc_ascii_art(arcs)
                else:
                    arc_width, arc_chars = 0, {}

                exit_counts = cp.exit_counts()

                for lineno, ltext in enumerate(cp.lines, start=1):
                    m0 = m1 = m2 = m3 = a = ' '
                    if lineno in cp.statement_starts:
                        m0 = '-'
                    exits = exit_counts.get(lineno, 0)
                    if exits > 1:
                        m1 = str(exits)
                    if lineno in cp.docstrings:
                        m2 = '"'
                    if lineno in cp.classdefs:
                        m2 = 'C'
                    if lineno in cp.excluded:
                        m3 = 'x'
                    a = arc_chars[lineno].ljust(arc_width)
                    print("%4d %s%s%s%s%s %s" %
                                (lineno, m0, m1, m2, m3, a, ltext)
                        )

    def disassemble(self, byte_parser, histogram=False):
        """Disassemble code, for ad-hoc experimenting."""

        for bp in byte_parser.child_parsers():
            chunks = bp._split_into_chunks()
            chunkd = dict((chunk.byte, chunk) for chunk in chunks)
            if bp.text:
                srclines = bp.text.splitlines()
            else:
                srclines = None
            print("\n%s: " % bp.code)
            upto = None
            for disline in disgen.disgen(bp.code):
                if histogram:
                    opcode_counts[disline.opcode] += 1
                    continue
                if disline.first:
                    if srclines:
                        upto = upto or disline.lineno-1
                        while upto <= disline.lineno-1:
                            print("%100s%s" % ("", srclines[upto]))
                            upto += 1
                    elif disline.offset > 0:
                        print("")
                line = disgen.format_dis_line(disline)
                chunk = chunkd.get(disline.offset)
                if chunk:
                    chunkstr = ":: %r" % chunk
                else:
                    chunkstr = ""
                print("%-70s%s" % (line, chunkstr))

        print("")

    def arc_ascii_art(self, arcs):
        """Draw arcs as ascii art.

        Returns a width of characters needed to draw all the arcs, and a
        dictionary mapping line numbers to ascii strings to draw for that line.

        """
        arc_chars = collections.defaultdict(str)
        for lfrom, lto in sorted(arcs):
            if lfrom < 0:
                arc_chars[lto] += 'v'
            elif lto < 0:
                arc_chars[lfrom] += '^'
            else:
                if lfrom == lto - 1:
                    # Don't show obvious arcs.
                    continue
                if lfrom < lto:
                    l1, l2 = lfrom, lto
                else:
                    l1, l2 = lto, lfrom
                w = max(len(arc_chars[l]) for l in range(l1, l2+1))
                for l in range(l1, l2+1):
                    if l == lfrom:
                        ch = '<'
                    elif l == lto:
                        ch = '>'
                    else:
                        ch = '|'
                    arc_chars[l] = arc_chars[l].ljust(w) + ch
                arc_width = 0

        if arc_chars:
            arc_width = max(len(a) for a in arc_chars.values())
        else:
            arc_width = 0

        return arc_width, arc_chars

if __name__ == '__main__':
    ParserMain().main(sys.argv[1:])

<|endoftext|>


<commit_before>import os, eHive
from igf_data.utils.dbutils import read_dbconf_json
from igf_data.task_tracking.igf_slack import IGF_slack
from igf_data.task_tracking.igf_asana import IGF_asana
from igf_data.igfdb.baseadaptor import BaseAdaptor

class IGFBaseRunnable(eHive.BaseRunnable):
  '''
  Base runnable class for IGF pipelines
  '''
  def param_defaults(self):
    return { 'log_slack':True,
             'log_asana':True
           }

  def fetch_input(self):
    dbconfig = self.param_required('dbconfig')
    dbparams = read_dbconf_json(dbconfig)
    base = BaseAdaptor(**dbparams)
    session_class = base.get_session_class()
    self.param('igf_session_class', session_class)      # set session class for pipeline

    if self.param('log_slack'):
      slack_config = self.param_required('slack_config')
      igf_slack = IGF_slack(slack_config=slack_config)
      self.param('igf_slack', igf_slack)

    if self.param('log_asana'):
      asana_config = self.param_required('asana_config')
      asana_project_id = self.param_required('asana_project_id')
      igf_asana = IGF_asana(asana_config=asana_config, asana_project_id=asana_project_id)
      self.param('igf_asana', igf_asana)

  def run(self):
    pass

  def write_output(self):
    pass

  def post_message_to_slack(self,message,reaction=''):
    '''
    A method for posing message to slack channel
    required params:
    message: A text message
    reaction: Optional parameter for slack emoji
    '''
    if self.param('log_slack'):
      igf_slack = self.param_required('igf_slack')
      igf_slack.post_message_to_channel(message,reaction)

  def get_job_id(self):
    '''
    A method for fetching job process id
    '''
    job_pid=os.getpid()
    return job_pid

  def job_name(selfself):
    '''
    A method for getting a job name
    '''
    class_name=self.__class__.__name__
    job_id=self.get_job_id()
    job_name='{0}_{1}'.format(class_name,job_id)
    return job_name

class IGFBaseJobFactory(IGFBaseRunnable):
  '''
  Base jobfactory class for igf pipelines
  '''
  def write_output(self):
    sub_tasks = self.param_required('sub_tasks')
    self.dataflow(sub_tasks, 2)

class IGFBaseProcess(IGFBaseRunnable):
  '''
  Base process class for igf pipelines
  '''
  def write_output(self):
    sub_tasks = self.param_required('sub_tasks')
    self.dataflow(sub_tasks, 1)

<commit_msg>minor bugfix to the base runnable<commit_after>import os, eHive
from igf_data.utils.dbutils import read_dbconf_json
from igf_data.task_tracking.igf_slack import IGF_slack
from igf_data.task_tracking.igf_asana import IGF_asana
from igf_data.igfdb.baseadaptor import BaseAdaptor

class IGFBaseRunnable(eHive.BaseRunnable):
  '''
  Base runnable class for IGF pipelines
  '''
  def param_defaults(self):
    return { 'log_slack':True,
             'log_asana':True
           }

  def fetch_input(self):
    dbconfig = self.param_required('dbconfig')
    dbparams = read_dbconf_json(dbconfig)
    base = BaseAdaptor(**dbparams)
    session_class = base.get_session_class()
    self.param('igf_session_class', session_class)      # set session class for pipeline

    if self.param('log_slack'):
      slack_config = self.param_required('slack_config')
      igf_slack = IGF_slack(slack_config=slack_config)
      self.param('igf_slack', igf_slack)

    if self.param('log_asana'):
      asana_config = self.param_required('asana_config')
      asana_project_id = self.param_required('asana_project_id')
      igf_asana = IGF_asana(asana_config=asana_config, asana_project_id=asana_project_id)
      self.param('igf_asana', igf_asana)

  def run(self):
    pass

  def write_output(self):
    pass

  def post_message_to_slack(self,message,reaction=''):
    '''
    A method for posing message to slack channel
    required params:
    message: A text message
    reaction: Optional parameter for slack emoji
    '''
    if self.param('log_slack'):
      igf_slack = self.param_required('igf_slack')
      igf_slack.post_message_to_channel(message,reaction)

  def get_job_id(self):
    '''
    A method for fetching job process id
    '''
    job_pid=os.getpid()
    return job_pid

  def job_name(self):
    '''
    A method for getting a job name
    '''
    class_name=self.__class__.__name__
    job_id=self.get_job_id()
    job_name='{0}_{1}'.format(class_name,job_id)
    return job_name

class IGFBaseJobFactory(IGFBaseRunnable):
  '''
  Base jobfactory class for igf pipelines
  '''
  def write_output(self):
    sub_tasks = self.param_required('sub_tasks')
    self.dataflow(sub_tasks, 2)

class IGFBaseProcess(IGFBaseRunnable):
  '''
  Base process class for igf pipelines
  '''
  def write_output(self):
    sub_tasks = self.param_required('sub_tasks')
    self.dataflow(sub_tasks, 1)

<|endoftext|>


<commit_before>import datetime
import json
import logging
import socket
from threading import Thread

import zmq
from flask import Flask, render_template, Response
from isodate import parse_duration

from motorway.intersection import Intersection
from motorway.utils import DateTimeAwareJsonEncoder

logger = logging.getLogger(__name__)

class WebserverIntersection(Intersection):
    """
    Simple Flask webserver that exposes information sent from the controller(s).

    It groups processes by name and furhter more sets a "status" for the different processes, based on
    how busy they are.
    """

    send_control_messages = False

    def get_last_minutes(self, minute_count=10):
        now = datetime.datetime.now()
        return [(now - datetime.timedelta(minutes=i)).minute for i in range(0, minute_count)]

    def __init__(self, *args, **kwargs):
        super(WebserverIntersection, self).__init__(*args, **kwargs)

        self.process_statistics = {}
        self.stream_consumers = {}
        self.failed_messages = {}
        self.groups = {}

        app = Flask(__name__)

        @app.route("/")
        def index():
            return render_template("index.html")

        @app.route("/app.js")
        def js():
            return Response(render_template("app.js"), mimetype='application/javascript')

        @app.route("/style.css")
        def css():
            return Response(render_template("style.css"), mimetype='text/css')

        @app.route("/detail/<process>/")
        def detail(process):
            return render_template(
                "detail.html",
                process=process,
                hostname=socket.gethostname(),
                messages_being_processed=self._get_messages_being_processed_for_process(process),
                process_stats=self.process_statistics.get(process),
                failed_messages=reversed(sorted([msg for msg in self.failed_messages.values() if
                                 msg[1] == process], key=lambda itm: itm[0])[-20:]),

            )

        @app.route("/api/status/")
        def api_status():
            return Response(json.dumps(dict(
                groups=self.groups,
                last_minutes=self.get_last_minutes()
            ), cls=DateTimeAwareJsonEncoder), mimetype='application/json', headers={
                'Access-Control-Allow-Origin': '*'
            })

        # @app.route("/api/detail/<process>/")
        # def api_detail(process):
        #     return Response(json.dumps(dict(
        #         failed_messages=[msg[1] for msg in self.failed_messages.values() if self.process_id_to_name[msg[0]] == process],
        #     ), cls=DateTimeAwareJsonEncoder), mimetype='application/json')

        p = Thread(target=app.run, name="motorway-webserver", kwargs=dict(
            port=5000,
            host="0.0.0.0",
        ))
        p.start()

    def _get_messages_being_processed_for_process(self, process_uuid):
        report_socket_address = self.process_id_to_report_address.get(process_uuid)
        messages_being_processed = []
        if report_socket_address:
            messages_being_processed = self._get_messages_being_processed_from_socket(report_socket_address)

        return messages_being_processed

    def _get_messages_being_processed_from_socket(self, socket_address):
        context = zmq.Context()
        report_socket = context.socket(zmq.REQ)

        with report_socket.connect(socket_address) as conn:  # closes connection upon exit
            conn.send_string('')
            messages_being_processed_json = conn.recv_json()

        return [msg for msg in json.loads(messages_being_processed_json)]

    def process(self, message):
        self.process_statistics = message.content['process_statistics']
        self.stream_consumers = message.content['stream_consumers']
        self.failed_messages = message.content['failed_messages']

        for process_id, stats in self.process_statistics.items():
            if process_id in self.process_id_to_name:

                stats['state'] = 'available'
                if stats['waiting'] > 0:
                    stats['state'] = 'busy'
                    processed_last_few_minutes = 0
                    for minute in self.get_last_minutes(3):
                        processed_last_few_minutes += stats['histogram'][str(minute)]['processed_count']
                    if stats['waiting'] > processed_last_few_minutes:
                        stats['state'] = 'overloaded'

                group_name = self.process_id_to_name[process_id].split('-')[0]
                new_stats = self.groups.get(group_name, {'processes': {}.copy()}.copy())
                new_stats['processes'][process_id] = stats
                self.groups[group_name] = new_stats

        for group in self.groups.values():
            group['waiting'] = sum([process['waiting'] for process in group['processes'].values()])
            group['time_taken'] = datetime.timedelta()
            group['histogram'] = {str(minute): {'error_count': 0, 'success_count': 0, 'timeout_count': 0, 'processed_count': 0}.copy() for minute in range(0, 60)}.copy()
            for process_id, process in group['processes'].items():

                # Remove stale processes (those no longer in the connection thread)
                if process_id not in self.process_id_to_name:
                    del group['processes'][process_id]
                    continue

                # Calculate statistics on the active processes
                group['time_taken'] += parse_duration(process['time_taken']) or datetime.timedelta(seconds=0)
                for minute, histogram_dict in process.get('histogram').items():
                    group['histogram'][minute]['error_count'] += histogram_dict['error_count']
                    group['histogram'][minute]['success_count'] += histogram_dict['success_count']
                    group['histogram'][minute]['timeout_count'] += histogram_dict['timeout_count']
                    group['histogram'][minute]['processed_count'] += histogram_dict['processed_count']
            group['frequency'] = sum([sum(process['frequency'].values()) for process in group['processes'].values()]) or 1  # Fallback to at least one, otherwise division fails below

            group['avg_time_taken'] = group['time_taken'] / group['frequency'] / len(group['processes']) if len(group['processes']) else 0

        yield<commit_msg>Remove unsupported context manager due to the version of pyzmq used<commit_after>import datetime
import json
import logging
import socket
from threading import Thread

import zmq
from flask import Flask, render_template, Response
from isodate import parse_duration

from motorway.intersection import Intersection
from motorway.utils import DateTimeAwareJsonEncoder

logger = logging.getLogger(__name__)

class WebserverIntersection(Intersection):
    """
    Simple Flask webserver that exposes information sent from the controller(s).

    It groups processes by name and furhter more sets a "status" for the different processes, based on
    how busy they are.
    """

    send_control_messages = False

    def get_last_minutes(self, minute_count=10):
        now = datetime.datetime.now()
        return [(now - datetime.timedelta(minutes=i)).minute for i in range(0, minute_count)]

    def __init__(self, *args, **kwargs):
        super(WebserverIntersection, self).__init__(*args, **kwargs)

        self.process_statistics = {}
        self.stream_consumers = {}
        self.failed_messages = {}
        self.groups = {}

        app = Flask(__name__)

        @app.route("/")
        def index():
            return render_template("index.html")

        @app.route("/app.js")
        def js():
            return Response(render_template("app.js"), mimetype='application/javascript')

        @app.route("/style.css")
        def css():
            return Response(render_template("style.css"), mimetype='text/css')

        @app.route("/detail/<process>/")
        def detail(process):
            return render_template(
                "detail.html",
                process=process,
                hostname=socket.gethostname(),
                messages_being_processed=self._get_messages_being_processed_for_process(process),
                process_stats=self.process_statistics.get(process),
                failed_messages=reversed(sorted([msg for msg in self.failed_messages.values() if
                                 msg[1] == process], key=lambda itm: itm[0])[-20:]),

            )

        @app.route("/api/status/")
        def api_status():
            return Response(json.dumps(dict(
                groups=self.groups,
                last_minutes=self.get_last_minutes()
            ), cls=DateTimeAwareJsonEncoder), mimetype='application/json', headers={
                'Access-Control-Allow-Origin': '*'
            })

        # @app.route("/api/detail/<process>/")
        # def api_detail(process):
        #     return Response(json.dumps(dict(
        #         failed_messages=[msg[1] for msg in self.failed_messages.values() if self.process_id_to_name[msg[0]] == process],
        #     ), cls=DateTimeAwareJsonEncoder), mimetype='application/json')

        p = Thread(target=app.run, name="motorway-webserver", kwargs=dict(
            port=5000,
            host="0.0.0.0",
        ))
        p.start()

    def _get_messages_being_processed_for_process(self, process_uuid):
        report_socket_address = self.process_id_to_report_address.get(process_uuid)
        messages_being_processed = []
        if report_socket_address:
            messages_being_processed = self._get_messages_being_processed_from_socket(report_socket_address)

        return messages_being_processed

    def _get_messages_being_processed_from_socket(self, socket_address):
        context = zmq.Context()
        report_socket = context.socket(zmq.REQ)

        report_socket.connect(socket_address)
        report_socket.send_string('')
        messages_being_processed_json = report_socket.recv_json()

        return [msg for msg in json.loads(messages_being_processed_json)]

    def process(self, message):
        self.process_statistics = message.content['process_statistics']
        self.stream_consumers = message.content['stream_consumers']
        self.failed_messages = message.content['failed_messages']

        for process_id, stats in self.process_statistics.items():
            if process_id in self.process_id_to_name:

                stats['state'] = 'available'
                if stats['waiting'] > 0:
                    stats['state'] = 'busy'
                    processed_last_few_minutes = 0
                    for minute in self.get_last_minutes(3):
                        processed_last_few_minutes += stats['histogram'][str(minute)]['processed_count']
                    if stats['waiting'] > processed_last_few_minutes:
                        stats['state'] = 'overloaded'

                group_name = self.process_id_to_name[process_id].split('-')[0]
                new_stats = self.groups.get(group_name, {'processes': {}.copy()}.copy())
                new_stats['processes'][process_id] = stats
                self.groups[group_name] = new_stats

        for group in self.groups.values():
            group['waiting'] = sum([process['waiting'] for process in group['processes'].values()])
            group['time_taken'] = datetime.timedelta()
            group['histogram'] = {str(minute): {'error_count': 0, 'success_count': 0, 'timeout_count': 0, 'processed_count': 0}.copy() for minute in range(0, 60)}.copy()
            for process_id, process in group['processes'].items():

                # Remove stale processes (those no longer in the connection thread)
                if process_id not in self.process_id_to_name:
                    del group['processes'][process_id]
                    continue

                # Calculate statistics on the active processes
                group['time_taken'] += parse_duration(process['time_taken']) or datetime.timedelta(seconds=0)
                for minute, histogram_dict in process.get('histogram').items():
                    group['histogram'][minute]['error_count'] += histogram_dict['error_count']
                    group['histogram'][minute]['success_count'] += histogram_dict['success_count']
                    group['histogram'][minute]['timeout_count'] += histogram_dict['timeout_count']
                    group['histogram'][minute]['processed_count'] += histogram_dict['processed_count']
            group['frequency'] = sum([sum(process['frequency'].values()) for process in group['processes'].values()]) or 1  # Fallback to at least one, otherwise division fails below

            group['avg_time_taken'] = group['time_taken'] / group['frequency'] / len(group['processes']) if len(group['processes']) else 0

        yield<|endoftext|>


<commit_before>"""A generic state machine."""
from mpf.core.device_monitor import DeviceMonitor

from mpf.core.mode import Mode
from mpf.core.player import Player
from mpf.core.mode_device import ModeDevice
from mpf.core.system_wide_device import SystemWideDevice

@DeviceMonitor("state")
class StateMachine(SystemWideDevice, ModeDevice):

    """A generic state machine."""

    config_section = 'state_machines'
    collection = 'state_machines'
    class_label = 'state_machine'

    __slots__ = ["player", "_state", "_handlers", "_show"]

    def __init__(self, machine, name):
        """Initialise state machine."""
        super().__init__(machine, name)
        self.player = None
        self._state = None
        self._handlers = []
        self._show = None

    async def device_added_system_wide(self):
        """Initialise internal state."""
        await super().device_added_system_wide()

        if self.config['persist_state']:
            self.raise_config_error("Cannot set persist_state for system-wide state_machine", 1)

        self._start_state(self.config['starting_state'])

    def validate_and_parse_config(self, config: dict, is_mode_config: bool, debug_prefix: str = None):
        """Validate transitions."""
        result = super().validate_and_parse_config(config, is_mode_config, debug_prefix)
        states = config.get("states", {})
        for transition in config.get("transitions", []):
            for state in transition["source"]:
                if state not in states:
                    self.raise_config_error("Source {} of transition {} not found in states.".format(
                        transition["source"], transition), 2)
            if transition["target"] not in states:
                self.raise_config_error("Target {} of transition {} not found in states.".format(
                    transition["target"], transition), 3)
        return result

    @property
    def state(self):
        """Return the current state."""
        if self.config['persist_state']:
            return self.player["state_machine_{}".format(self.name)]

        return self._state

    @state.setter
    def state(self, value):
        """Set the current state."""
        old = self.state
        if self.config['persist_state']:
            old = self.player["state_machine_{}".format(self.name)]
            self.player["state_machine_{}".format(self.name)] = value
            self.notify_virtual_change(self, old, value)
        else:
            self._state = value

        # notify monitors
        self.notify_virtual_change("state", old, self.state)

    def device_loaded_in_mode(self, mode: Mode, player: Player):
        """Restore internal state from player if persist_state is set or create new state."""
        super().device_loaded_in_mode(mode, player)
        self.player = player
        if not self.state:
            self._start_state(self.config['starting_state'])
        else:
            self._add_handlers_for_current_state()
            self._run_show_for_current_state()

    def device_removed_from_mode(self, mode: Mode):
        """Unset internal state to prevent leakage."""
        super().device_removed_from_mode(mode)
        self._remove_handlers()
        self._state = None
        self.player = None

        if self._show:
            self._show.stop()
            self._show = None

    def _stop_current_state(self):
        self._remove_handlers()
        state_config = self.config['states'][self.state]
        if state_config['events_when_stopped']:
            for event_name in state_config['events_when_stopped']:
                self.machine.events.post(event_name)

        if self._show:
            self._show.stop()
            self._show = None

        self.state = None

    def _start_state(self, state):
        if state not in self.config['states']:
            raise AssertionError("Invalid state {}".format(state))

        state_config = self.config['states'][state]
        self.state = state
        if state_config['events_when_started']:
            for event_name in state_config['events_when_started']:
                self.machine.events.post(event_name)

        self._add_handlers_for_current_state()
        self._run_show_for_current_state()

    def _run_show_for_current_state(self):
        assert not self._show
        state_config = self.config['states'][self.state]
        if state_config['show_when_active']:
            self._show = self.machine.show_controller.play_show_with_config(state_config['show_when_active'],
                                                                            self.mode)

    def _add_handlers_for_current_state(self):
        for transition in self.config['transitions']:
            if self.state in transition['source']:
                for event in transition['events']:
                    self._handlers.append(self.machine.events.add_handler(event, self._transition,
                                                                          transition_config=transition))

    def _transition(self, transition_config, **kwargs):
        del kwargs
        self._stop_current_state()
        if transition_config['events_when_transitioning']:
            for event_name in transition_config['events_when_transitioning']:
                self.machine.events.post(event_name)
        self._start_state(transition_config['target'])

    def _remove_handlers(self):
        self.machine.events.remove_handlers_by_keys(self._handlers)
<commit_msg>add some info and debug logging to state machines<commit_after>"""A generic state machine."""
from mpf.core.device_monitor import DeviceMonitor

from mpf.core.mode import Mode
from mpf.core.player import Player
from mpf.core.mode_device import ModeDevice
from mpf.core.system_wide_device import SystemWideDevice

@DeviceMonitor("state")
class StateMachine(SystemWideDevice, ModeDevice):

    """A generic state machine."""

    config_section = 'state_machines'
    collection = 'state_machines'
    class_label = 'state_machine'

    __slots__ = ["player", "_state", "_handlers", "_show"]

    def __init__(self, machine, name):
        """Initialise state machine."""
        super().__init__(machine, name)
        self.player = None
        self._state = None
        self._handlers = []
        self._show = None

    async def device_added_system_wide(self):
        """Initialise internal state."""
        await super().device_added_system_wide()

        if self.config['persist_state']:
            self.raise_config_error("Cannot set persist_state for system-wide state_machine", 1)

        self._start_state(self.config['starting_state'])

    def validate_and_parse_config(self, config: dict, is_mode_config: bool, debug_prefix: str = None):
        """Validate transitions."""
        result = super().validate_and_parse_config(config, is_mode_config, debug_prefix)
        states = config.get("states", {})
        for transition in config.get("transitions", []):
            for state in transition["source"]:
                if state not in states:
                    self.raise_config_error("Source {} of transition {} not found in states.".format(
                        transition["source"], transition), 2)
            if transition["target"] not in states:
                self.raise_config_error("Target {} of transition {} not found in states.".format(
                    transition["target"], transition), 3)
        return result

    @property
    def state(self):
        """Return the current state."""
        if self.config['persist_state']:
            return self.player["state_machine_{}".format(self.name)]

        return self._state

    @state.setter
    def state(self, value):
        """Set the current state."""
        old = self.state
        if self.config['persist_state']:
            old = self.player["state_machine_{}".format(self.name)]
            self.player["state_machine_{}".format(self.name)] = value
            self.notify_virtual_change(self, old, value)
        else:
            self._state = value

        # notify monitors
        self.notify_virtual_change("state", old, self.state)

    def device_loaded_in_mode(self, mode: Mode, player: Player):
        """Restore internal state from player if persist_state is set or create new state."""
        super().device_loaded_in_mode(mode, player)
        self.player = player
        if not self.state:
            self._start_state(self.config['starting_state'])
        else:
            self._add_handlers_for_current_state()
            self._run_show_for_current_state()

    def device_removed_from_mode(self, mode: Mode):
        """Unset internal state to prevent leakage."""
        super().device_removed_from_mode(mode)
        self._remove_handlers()
        self._state = None
        self.player = None

        if self._show:
            self._show.stop()
            self._show = None

    def _stop_current_state(self):
        self.log.debug("Stopping state %s", self.state)
        self._remove_handlers()
        state_config = self.config['states'][self.state]
        if state_config['events_when_stopped']:
            for event_name in state_config['events_when_stopped']:
                self.machine.events.post(event_name)

        if self._show:
            self.log.debug("Stopping show %s", self._show)
            self._show.stop()
            self._show = None

        self.state = None

    def _start_state(self, state):
        self.log.debug("Starting state %s", state)
        if state not in self.config['states']:
            raise AssertionError("Invalid state {}".format(state))

        state_config = self.config['states'][state]
        self.state = state
        if state_config['events_when_started']:
            for event_name in state_config['events_when_started']:
                self.machine.events.post(event_name)

        self._add_handlers_for_current_state()
        self._run_show_for_current_state()

    def _run_show_for_current_state(self):
        assert not self._show
        state_config = self.config['states'][self.state]
        if state_config['show_when_active']:
            self.log.debug("Starting show %s", state_config['show_when_active'])
            self._show = self.machine.show_controller.play_show_with_config(state_config['show_when_active'],
                                                                            self.mode)

    def _add_handlers_for_current_state(self):
        for transition in self.config['transitions']:
            if self.state in transition['source']:
                for event in transition['events']:
                    self._handlers.append(self.machine.events.add_handler(event, self._transition,
                                                                          transition_config=transition))

    def _transition(self, transition_config, **kwargs):
        del kwargs
        self.log.info("Transitioning from %s to %s", self.state, transition_config["target"])
        self._stop_current_state()
        if transition_config['events_when_transitioning']:
            for event_name in transition_config['events_when_transitioning']:
                self.machine.events.post(event_name)
        self._start_state(transition_config['target'])

    def _remove_handlers(self):
        self.machine.events.remove_handlers_by_keys(self._handlers)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging

from django.conf import settings
from django.db.models import ForeignKey, SET_NULL
from django.utils.translation import ugettext_lazy as _

from django_userforeignkey.request import get_current_user

logger = logging.getLogger(__name__)

class UserForeignKey(ForeignKey):
    """Model field UserForeignKey

    Saves a reference to a user. By using the `django_userforeignkey.middleware.UserForeignKeyMiddleware`
    middleware, the field can save the reference automatically.
    """
    description = _(u"User")

    def __init__(self, auto_user=False, auto_user_add=False, **kwargs):
        self.auto_user, self.auto_user_add = auto_user, auto_user_add

        kwargs['to'] = settings.AUTH_USER_MODEL

        if auto_user or auto_user_add:
            kwargs['on_delete'] = SET_NULL
            kwargs['null'] = True
            kwargs['blank'] = True
            kwargs['editable'] = kwargs.get('editable', False)

        super(UserForeignKey, self).__init__(**kwargs)

    def pre_save(self, model_instance, add):
        object_user = self.value_from_object(model_instance)
        current_user = get_current_user()

        if current_user.pk and (self.auto_user or (self.auto_user_add and add and not object_user)):
            setattr(model_instance, self.name, current_user)
        elif not current_user.pk:
            # log if the current user cannot be referenced in DB (e.g. AnonymousUser)
            logger.info(u"Cannot save reference for non-persistent user")

        return super(UserForeignKey, self).pre_save(model_instance, add)
<commit_msg>Fixed gettext imports that caused RemovedInDjango40Warning warnings<commit_after># -*- coding: utf-8 -*-
import logging

from django.conf import settings
from django.db.models import ForeignKey, SET_NULL
from django.utils.translation import gettext_lazy as _

from django_userforeignkey.request import get_current_user

logger = logging.getLogger(__name__)

class UserForeignKey(ForeignKey):
    """Model field UserForeignKey

    Saves a reference to a user. By using the `django_userforeignkey.middleware.UserForeignKeyMiddleware`
    middleware, the field can save the reference automatically.
    """
    description = _(u"User")

    def __init__(self, auto_user=False, auto_user_add=False, **kwargs):
        self.auto_user, self.auto_user_add = auto_user, auto_user_add

        kwargs['to'] = settings.AUTH_USER_MODEL

        if auto_user or auto_user_add:
            kwargs['on_delete'] = SET_NULL
            kwargs['null'] = True
            kwargs['blank'] = True
            kwargs['editable'] = kwargs.get('editable', False)

        super(UserForeignKey, self).__init__(**kwargs)

    def pre_save(self, model_instance, add):
        object_user = self.value_from_object(model_instance)
        current_user = get_current_user()

        if current_user.pk and (self.auto_user or (self.auto_user_add and add and not object_user)):
            setattr(model_instance, self.name, current_user)
        elif not current_user.pk:
            # log if the current user cannot be referenced in DB (e.g. AnonymousUser)
            logger.info(u"Cannot save reference for non-persistent user")

        return super(UserForeignKey, self).pre_save(model_instance, add)
<|endoftext|>


<commit_before>"""Tests for window_utils"""

import sys
import test.support
import unittest
import unittest.mock

if sys.platform != 'win32':
    raise unittest.SkipTest('Windows only')

import _winapi

from asyncio import windows_utils
from asyncio import _overlapped

class WinsocketpairTests(unittest.TestCase):

    def test_winsocketpair(self):
        ssock, csock = windows_utils.socketpair()

        csock.send(b'xxx')
        self.assertEqual(b'xxx', ssock.recv(1024))

        csock.close()
        ssock.close()

    @unittest.mock.patch('asyncio.windows_utils.socket')
    def test_winsocketpair_exc(self, m_socket):
        m_socket.socket.return_value.getsockname.return_value = ('', 12345)
        m_socket.socket.return_value.accept.return_value = object(), object()
        m_socket.socket.return_value.connect.side_effect = OSError()

        self.assertRaises(OSError, windows_utils.socketpair)

class PipeTests(unittest.TestCase):

    def test_pipe_overlapped(self):
        h1, h2 = windows_utils.pipe(overlapped=(True, True))
        try:
            ov1 = _overlapped.Overlapped()
            self.assertFalse(ov1.pending)
            self.assertEqual(ov1.error, 0)

            ov1.ReadFile(h1, 100)
            self.assertTrue(ov1.pending)
            self.assertEqual(ov1.error, _winapi.ERROR_IO_PENDING)
            ERROR_IO_INCOMPLETE = 996
            try:
                ov1.getresult()
            except OSError as e:
                self.assertEqual(e.winerror, ERROR_IO_INCOMPLETE)
            else:
                raise RuntimeError('expected ERROR_IO_INCOMPLETE')

            ov2 = _overlapped.Overlapped()
            self.assertFalse(ov2.pending)
            self.assertEqual(ov2.error, 0)

            ov2.WriteFile(h2, b"hello")
            self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING})

            res = _winapi.WaitForMultipleObjects([ov2.event], False, 100)
            self.assertEqual(res, _winapi.WAIT_OBJECT_0)

            self.assertFalse(ov1.pending)
            self.assertEqual(ov1.error, ERROR_IO_INCOMPLETE)
            self.assertFalse(ov2.pending)
            self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING})
            self.assertEqual(ov1.getresult(), b"hello")
        finally:
            _winapi.CloseHandle(h1)
            _winapi.CloseHandle(h2)

    def test_pipe_handle(self):
        h, _ = windows_utils.pipe(overlapped=(True, True))
        _winapi.CloseHandle(_)
        p = windows_utils.PipeHandle(h)
        self.assertEqual(p.fileno(), h)
        self.assertEqual(p.handle, h)

        # check garbage collection of p closes handle
        del p
        test.support.gc_collect()
        try:
            _winapi.CloseHandle(h)
        except OSError as e:
            self.assertEqual(e.winerror, 6)     # ERROR_INVALID_HANDLE
        else:
            raise RuntimeError('expected ERROR_INVALID_HANDLE')

class PopenTests(unittest.TestCase):

    def test_popen(self):
        command = r"""if 1:
            import sys
            s = sys.stdin.readline()
            sys.stdout.write(s.upper())
            sys.stderr.write('stderr')
            """
        msg = b"blah\n"

        p = windows_utils.Popen([sys.executable, '-c', command],
                                stdin=windows_utils.PIPE,
                                stdout=windows_utils.PIPE,
                                stderr=windows_utils.PIPE)

        for f in [p.stdin, p.stdout, p.stderr]:
            self.assertIsInstance(f, windows_utils.PipeHandle)

        ovin = _overlapped.Overlapped()
        ovout = _overlapped.Overlapped()
        overr = _overlapped.Overlapped()

        ovin.WriteFile(p.stdin.handle, msg)
        ovout.ReadFile(p.stdout.handle, 100)
        overr.ReadFile(p.stderr.handle, 100)

        events = [ovin.event, ovout.event, overr.event]
        res = _winapi.WaitForMultipleObjects(events, True, 2000)
        self.assertEqual(res, _winapi.WAIT_OBJECT_0)
        self.assertFalse(ovout.pending)
        self.assertFalse(overr.pending)
        self.assertFalse(ovin.pending)

        self.assertEqual(ovin.getresult(), len(msg))
        out = ovout.getresult().rstrip()
        err = overr.getresult().rstrip()

        self.assertGreater(len(out), 0)
        self.assertGreater(len(err), 0)
        # allow for partial reads...
        self.assertTrue(msg.upper().rstrip().startswith(out))
        self.assertTrue(b"stderr".startswith(err))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Increase timeout in test_popen() for buildbots.<commit_after>"""Tests for window_utils"""

import sys
import test.support
import unittest
import unittest.mock

if sys.platform != 'win32':
    raise unittest.SkipTest('Windows only')

import _winapi

from asyncio import windows_utils
from asyncio import _overlapped

class WinsocketpairTests(unittest.TestCase):

    def test_winsocketpair(self):
        ssock, csock = windows_utils.socketpair()

        csock.send(b'xxx')
        self.assertEqual(b'xxx', ssock.recv(1024))

        csock.close()
        ssock.close()

    @unittest.mock.patch('asyncio.windows_utils.socket')
    def test_winsocketpair_exc(self, m_socket):
        m_socket.socket.return_value.getsockname.return_value = ('', 12345)
        m_socket.socket.return_value.accept.return_value = object(), object()
        m_socket.socket.return_value.connect.side_effect = OSError()

        self.assertRaises(OSError, windows_utils.socketpair)

class PipeTests(unittest.TestCase):

    def test_pipe_overlapped(self):
        h1, h2 = windows_utils.pipe(overlapped=(True, True))
        try:
            ov1 = _overlapped.Overlapped()
            self.assertFalse(ov1.pending)
            self.assertEqual(ov1.error, 0)

            ov1.ReadFile(h1, 100)
            self.assertTrue(ov1.pending)
            self.assertEqual(ov1.error, _winapi.ERROR_IO_PENDING)
            ERROR_IO_INCOMPLETE = 996
            try:
                ov1.getresult()
            except OSError as e:
                self.assertEqual(e.winerror, ERROR_IO_INCOMPLETE)
            else:
                raise RuntimeError('expected ERROR_IO_INCOMPLETE')

            ov2 = _overlapped.Overlapped()
            self.assertFalse(ov2.pending)
            self.assertEqual(ov2.error, 0)

            ov2.WriteFile(h2, b"hello")
            self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING})

            res = _winapi.WaitForMultipleObjects([ov2.event], False, 100)
            self.assertEqual(res, _winapi.WAIT_OBJECT_0)

            self.assertFalse(ov1.pending)
            self.assertEqual(ov1.error, ERROR_IO_INCOMPLETE)
            self.assertFalse(ov2.pending)
            self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING})
            self.assertEqual(ov1.getresult(), b"hello")
        finally:
            _winapi.CloseHandle(h1)
            _winapi.CloseHandle(h2)

    def test_pipe_handle(self):
        h, _ = windows_utils.pipe(overlapped=(True, True))
        _winapi.CloseHandle(_)
        p = windows_utils.PipeHandle(h)
        self.assertEqual(p.fileno(), h)
        self.assertEqual(p.handle, h)

        # check garbage collection of p closes handle
        del p
        test.support.gc_collect()
        try:
            _winapi.CloseHandle(h)
        except OSError as e:
            self.assertEqual(e.winerror, 6)     # ERROR_INVALID_HANDLE
        else:
            raise RuntimeError('expected ERROR_INVALID_HANDLE')

class PopenTests(unittest.TestCase):

    def test_popen(self):
        command = r"""if 1:
            import sys
            s = sys.stdin.readline()
            sys.stdout.write(s.upper())
            sys.stderr.write('stderr')
            """
        msg = b"blah\n"

        p = windows_utils.Popen([sys.executable, '-c', command],
                                stdin=windows_utils.PIPE,
                                stdout=windows_utils.PIPE,
                                stderr=windows_utils.PIPE)

        for f in [p.stdin, p.stdout, p.stderr]:
            self.assertIsInstance(f, windows_utils.PipeHandle)

        ovin = _overlapped.Overlapped()
        ovout = _overlapped.Overlapped()
        overr = _overlapped.Overlapped()

        ovin.WriteFile(p.stdin.handle, msg)
        ovout.ReadFile(p.stdout.handle, 100)
        overr.ReadFile(p.stderr.handle, 100)

        events = [ovin.event, ovout.event, overr.event]
        # Super-long timeout for slow buildbots.
        res = _winapi.WaitForMultipleObjects(events, True, 10000)
        self.assertEqual(res, _winapi.WAIT_OBJECT_0)
        self.assertFalse(ovout.pending)
        self.assertFalse(overr.pending)
        self.assertFalse(ovin.pending)

        self.assertEqual(ovin.getresult(), len(msg))
        out = ovout.getresult().rstrip()
        err = overr.getresult().rstrip()

        self.assertGreater(len(out), 0)
        self.assertGreater(len(err), 0)
        # allow for partial reads...
        self.assertTrue(msg.upper().rstrip().startswith(out))
        self.assertTrue(b"stderr".startswith(err))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from mod import PatchewModule
from mbox import parse_address
from event import register_handler, emit_event, declare_event
from api.models import Message

REV_BY_PREFIX = "Reviewed-by:"

_default_config = """
[default]
tags = Reviewed-by, Tested-by, Reported-by, Acked-by, Suggested-by

"""

class SeriesTagsModule(PatchewModule):
    """

Documentation
-------------

This module is configured in "INI" style.

It has only one section named `[default]`. The only supported option is tags:

    [default]
    tags = Reviewed-by, Tested-by, Reported-by, Acked-by, Suggested-by

The `tags` option contains the tag line prefixes (must be followed by colon)
that should be treated as meaningful patch status tags, and picked up from
series cover letter, patch mail body and their replies.

"""
    name = "tags"
    default_config = _default_config

    def __init__(self):
        register_handler("MessageAdded", self.on_message_added)
        declare_event("TagsUpdate", series="message object that is updated")

        # XXX: get this list through module config?
    def get_tag_prefixes(self):
        return [x.strip() for x in self.get_config("default", "tags").split(",")]

    def update_tags(self, s):
        old = s.get_property("tags", [])
        new = self.get_tags(s)
        if set(old) != set(new):
            s.set_property("tags", list(set(new)))
            return True

    def on_message_added(self, event, message):
        series = message.get_series_head()
        if not series:
            return

        for m in series.get_alternative_revisions():
            if m.version > series.version:
                series.set_property("obsoleted-by", m.message_id)
            elif m.version < series.version:
                m.set_property("obsoleted-by", series.message_id)

        updated = self.update_tags(series)

        for p in series.get_patches():
            updated = updated or self.update_tags(p)

        reviewers = set()
        num_reviewed = 0
        for p in series.get_patches():
            first = True
            for rev_tag in filter(lambda x: \
                                  x.lower().startswith(REV_BY_PREFIX.lower()),
                                  p.get_property("tags", [])):
                if first:
                    num_reviewed += 1
                    first = False
                reviewers.add(parse_address(rev_tag[len(REV_BY_PREFIX):]))
        if (num_reviewed, num_reviewed) == series.get_num():
            series.set_property("reviewed", True)
            series.set_property("reviewers", list(reviewers))
        if updated:
            emit_event("TagsUpdate", series=series)

    def parse_message_tags(self, m):
        r = []
        for l in m.get_body().splitlines():
            for p in self.get_tag_prefixes():
                if l.lower().startswith(p.lower()):
                    r.append(l)
        return r

    def get_tags(self, m):
        # Incorporate tags from non-patch replies
        r = self.parse_message_tags(m)
        for x in m.get_replies():
            if x.is_patch:
                continue
            r += self.get_tags(x)
        return r

    def prepare_message_hook(self, message):
        if not message.is_series_head:
            return
        if message.get_property("reviewed"):
            reviewers = message.get_property("reviewers")
            message.status_tags.append({
                "title": "Reviewed by " + ",".join([x for x, y in reviewers]),
                "type": "success",
                "char": "R",
                })
        ob = message.get_property("obsoleted-by")
        if ob:
            new = Message.objects.find_series(ob, message.project.name)
            message.status_tags.append({
                "title": "Has a newer version: " + new.subject,
                "type": "default",
                "char": "O",
                })

<commit_msg>tags: Add whitespace between reviewer names in web<commit_after>from mod import PatchewModule
from mbox import parse_address
from event import register_handler, emit_event, declare_event
from api.models import Message

REV_BY_PREFIX = "Reviewed-by:"

_default_config = """
[default]
tags = Reviewed-by, Tested-by, Reported-by, Acked-by, Suggested-by

"""

class SeriesTagsModule(PatchewModule):
    """

Documentation
-------------

This module is configured in "INI" style.

It has only one section named `[default]`. The only supported option is tags:

    [default]
    tags = Reviewed-by, Tested-by, Reported-by, Acked-by, Suggested-by

The `tags` option contains the tag line prefixes (must be followed by colon)
that should be treated as meaningful patch status tags, and picked up from
series cover letter, patch mail body and their replies.

"""
    name = "tags"
    default_config = _default_config

    def __init__(self):
        register_handler("MessageAdded", self.on_message_added)
        declare_event("TagsUpdate", series="message object that is updated")

        # XXX: get this list through module config?
    def get_tag_prefixes(self):
        return [x.strip() for x in self.get_config("default", "tags").split(",")]

    def update_tags(self, s):
        old = s.get_property("tags", [])
        new = self.get_tags(s)
        if set(old) != set(new):
            s.set_property("tags", list(set(new)))
            return True

    def on_message_added(self, event, message):
        series = message.get_series_head()
        if not series:
            return

        for m in series.get_alternative_revisions():
            if m.version > series.version:
                series.set_property("obsoleted-by", m.message_id)
            elif m.version < series.version:
                m.set_property("obsoleted-by", series.message_id)

        updated = self.update_tags(series)

        for p in series.get_patches():
            updated = updated or self.update_tags(p)

        reviewers = set()
        num_reviewed = 0
        for p in series.get_patches():
            first = True
            for rev_tag in filter(lambda x: \
                                  x.lower().startswith(REV_BY_PREFIX.lower()),
                                  p.get_property("tags", [])):
                if first:
                    num_reviewed += 1
                    first = False
                reviewers.add(parse_address(rev_tag[len(REV_BY_PREFIX):]))
        if (num_reviewed, num_reviewed) == series.get_num():
            series.set_property("reviewed", True)
            series.set_property("reviewers", list(reviewers))
        if updated:
            emit_event("TagsUpdate", series=series)

    def parse_message_tags(self, m):
        r = []
        for l in m.get_body().splitlines():
            for p in self.get_tag_prefixes():
                if l.lower().startswith(p.lower()):
                    r.append(l)
        return r

    def get_tags(self, m):
        # Incorporate tags from non-patch replies
        r = self.parse_message_tags(m)
        for x in m.get_replies():
            if x.is_patch:
                continue
            r += self.get_tags(x)
        return r

    def prepare_message_hook(self, message):
        if not message.is_series_head:
            return
        if message.get_property("reviewed"):
            reviewers = message.get_property("reviewers")
            message.status_tags.append({
                "title": "Reviewed by " + ", ".join([x for x, y in reviewers]),
                "type": "success",
                "char": "R",
                })
        ob = message.get_property("obsoleted-by")
        if ob:
            new = Message.objects.find_series(ob, message.project.name)
            message.status_tags.append({
                "title": "Has a newer version: " + new.subject,
                "type": "default",
                "char": "O",
                })

<|endoftext|>


<commit_before>import numpy as np
import math
import random
import copy
from sklearn.tree import DecisionTreeRegressor
from multiprocessing import Pool
from RegressionTree import RegressionTree
from essemble import essemble_trees
import pandas as pd

def dcg(scores):
	return np.sum([
						(np.power(2, scores[i]) - 1) / np.log2(i + 2)
						for i in xrange(len(scores))
					])
	# total = 0
	# for i in xrange(len(scores)):
	# 	total += (np.power(2.0, scores[i - 1]) - 1.0) / np.log2(i + 1)
	# return total

def ideal_dcg(scores):
	scores = [score for score in sorted(scores) if score > 0]
	return dcg(scores)

def compare_arr(a, b):
	if len(a) != len(b):
		return False
	for i in xrange(len(a)):
		if a[i] != b[i]:
			return False
	return True

def single_dcg(scores, i, j):
	return (np.power(2, scores[i]) - 1) / np.log2(j + 2)

def delta_ndcg(scores, i, j, idcg):

	dcg_val = dcg(scores)
	temp_scores = copy.deepcopy(scores)
	temp_scores[i], temp_scores[j] = temp_scores[j], temp_scores[i]
	new_dcg_val = dcg(temp_scores)

	# ideal_scores = [score for score in sorted(temp_scores) if score > 0]

	# ideal_dcg = dcg(ideal_scores)
	return abs(dcg_val - new_dcg_val)/idcg

# useless parallel code
def lambda_parallel(args):
	true_scores, good_ij_pairs, predicted_scores = args
	# print scores
	# exit()

	# true_scores, predicted_scores = scores
	i, j = good_ij_pairs
	z_ndcg = delta_ndcg(true_scores, i, j)
	rho = 1 / (1 + np.exp(predicted_scores[i] - predicted_scores[j]))
	rho_complement = 1.0 - rho
	lambda_val = z_ndcg * rho
	# lambdas = []
	# lambdas.append(lambda_val)
	# lambdas.append(-lambda_val)
	# w = []
	w_val = rho * rho_complement * z_ndcg
	# w.append(w_val)
	# w.append(w_val)
	return i, j, lambda_val, w_val

#true_scores, predicted_scores
def compute_lambda(args):
	true_scores, predicted_scores, good_ij_pairs, idcg, query_key = args
	# true_scores, predicted_scores, good_ij_pairs = scores
	num_docs = len(true_scores)
	sorted_indexes = np.argsort(predicted_scores)[::-1]
	rev_indexes = np.argsort(sorted_indexes)
	true_scores = true_scores[sorted_indexes]
	predicted_scores = predicted_scores[sorted_indexes]

	lambdas = np.zeros(num_docs)
	w = np.zeros(num_docs)

	single_dcgs = {}
	for i,j in good_ij_pairs:
		if (i,i) not in single_dcgs:
			single_dcgs[(i,i)] = single_dcg(true_scores, i, i)
		single_dcgs[(i,j)] = single_dcg(true_scores, i, j)
		if (j,j) not in single_dcgs:
			single_dcgs[(j,j)] = single_dcg(true_scores, j, j)
		single_dcgs[(j,i)] = single_dcg(true_scores, j, i)
	### parallel portion
	# pool = Pool(processes=8)
	# for i, j, lambda_val, w_val in pool.map(lambda_parallel, zip([true_scores for i in xrange(len(good_ij_pairs))], good_ij_pairs, [predicted_scores for i in xrange(len(good_ij_pairs))])):
	# 	lambdas[i] += lambda_val
	# 	lambdas[j] -= lambda_val
	# 	w[i] += w_val
	# 	w[j] += w_val
	# pool.close()

	for i,j in good_ij_pairs:
		# z_ndcg = delta_ndcg(true_scores, i, j, idcg)
		z_ndcg = abs(single_dcgs[(i,j)] - single_dcgs[(i,i)] + single_dcgs[(j,i)] - single_dcgs[(j,j)]) / idcg
		rho = 1 / (1 + np.exp(predicted_scores[i] - predicted_scores[j]))
		rho_complement = 1.0 - rho
		lambda_val = z_ndcg * rho
		lambdas[i] += lambda_val
		lambdas[j] -= lambda_val

		w_val = rho * rho_complement * z_ndcg
		w[i] += w_val
		w[j] += w_val

	return lambdas[rev_indexes], w[rev_indexes], query_key

def group_queries(training_data):
	query_indexes = {}
	index = 0
	for record in training_data:
		query_indexes.setdefault(record[1], [])
		query_indexes[record[1]].append(index)
		index += 1
	return query_indexes

def get_pairs(scores):
	query_pair = []
	for query_scores in scores:
		temp = sorted(query_scores, reverse=True)
		pairs = []
		for i in xrange(len(temp)):
			for j in xrange(len(temp)):
				if temp[i] > temp[j]:
					pairs.append((i,j))
		query_pair.append(pairs)
	return query_pair

class LambdaMART:

	def __init__(self, training_data, number_of_trees, leaves_per_tree, learning_rate):
		'''
		The format for training data is as follows:
			[relevance, q_id, [feature vector]]
		'''
		self.training_data = np.array(training_data)
		self.number_of_trees = number_of_trees
		self.leaves_per_tree = leaves_per_tree
		self.learning_rate = learning_rate

	def fit(self):
		predicted_scores = np.zeros(len(self.training_data))
		query_indexes = group_queries(self.training_data)
		query_keys = query_indexes.keys()
		true_scores = [self.training_data[query_indexes[query], 0] for query in query_keys]
		good_ij_pairs = get_pairs(true_scores)
		tree_data = pd.DataFrame(self.training_data[:, 2:7])
		labels = self.training_data[:, 0]

		# ideal dcg calculation
		idcg = [ideal_dcg(scores) for scores in true_scores]

		for k in xrange(self.number_of_trees):
			print 'Tree %d' % (k)
			lambdas = np.zeros(len(predicted_scores))
			w = np.zeros(len(predicted_scores))
			pred_scores = [predicted_scores[query_indexes[query]] for query in query_keys]

			pool = Pool()
			for lambda_val, w_val, query_key in pool.map(compute_lambda, zip(true_scores, pred_scores, good_ij_pairs, idcg, query_keys), chunksize=1):
				indexes = query_indexes[query_key]
				lambdas[indexes] = lambda_val
				w[indexes] = w_val
			pool.close()

			## non parallel
			# for i in xrange(len(true_scores)):
			# 	query = query_keys[i]
			# 	lambdas[query_indexes[query]], w[query_indexes[query]] = compute_lambda(true_scores[i], pred_scores[i], good_ij_pairs[i])

			# sklearn tree
			# tree = DecisionTreeRegressor(max_depth=50)
			# tree.fit(self.training_data[:,2:], lambdas)
			# prediction = tree.predict(self.training_data[:,2:])
			# predicted_scores += prediction * self.learning_rate
			tree = RegressionTree(tree_data, predicted_scores, max_depth=10, ideal_ls= 0.001)
			print 'created tree'
			tree.fit()
			print 'fitted tree'
			prediction = tree.predict(self.training_data[:,2:])
			print 'predicted tree'
			print prediction
			# exit()
			predicted_scores = essemble_trees(lambdas, w, prediction, predicted_scores, self.learning_rate)
			print 'updates scores'

			###
			# Tree code here, already calculated lambdas and ws
			###

		print predicted_scores
	# def predict(self, data):

def main():
	f = open('vali.txt', 'r')
	count = 0
	training_data = []
	for line in f:
		if count > 200:
			break
		new_arr = []
		arr = line.split(' #')[0].split()
		score = arr[0]
		q_id = arr[1].split(':')[1]
		new_arr.append(int(score))
		new_arr.append(int(q_id))
		arr = arr[2:]
		for el in arr:
			new_arr.append(float(el.split(':')[1]))
		training_data.append(new_arr)
		count += 1
	f.close()
	model = LambdaMART(training_data, 200, 10, 0.1)
	model.fit()

if __name__ == '__main__':
	main()<commit_msg>added prediction and fixed bugs<commit_after>import numpy as np
import math
import random
import copy
from sklearn.tree import DecisionTreeRegressor
from multiprocessing import Pool
from RegressionTree import RegressionTree
from essemble import essemble_trees
import pandas as pd

def dcg(scores):
	return np.sum([
						(np.power(2, scores[i]) - 1) / np.log2(i + 2)
						for i in xrange(len(scores))
					])
	# total = 0
	# for i in xrange(len(scores)):
	# 	total += (np.power(2.0, scores[i - 1]) - 1.0) / np.log2(i + 1)
	# return total

def ideal_dcg(scores):
	scores = [score for score in sorted(scores)[::-1]]
	return dcg(scores)

def compare_arr(a, b):
	if len(a) != len(b):
		return False
	for i in xrange(len(a)):
		if a[i] != b[i]:
			return False
	return True

def single_dcg(scores, i, j):
	return (np.power(2, scores[i]) - 1) / np.log2(j + 2)

def delta_ndcg(scores, i, j, idcg):

	dcg_val = dcg(scores)
	temp_scores = copy.deepcopy(scores)
	temp_scores[i], temp_scores[j] = temp_scores[j], temp_scores[i]
	new_dcg_val = dcg(temp_scores)

	# ideal_scores = [score for score in sorted(temp_scores) if score > 0]

	# ideal_dcg = dcg(ideal_scores)
	return abs(dcg_val - new_dcg_val)/idcg

# useless parallel code
def lambda_parallel(args):
	true_scores, good_ij_pairs, predicted_scores = args
	# print scores
	# exit()

	# true_scores, predicted_scores = scores
	i, j = good_ij_pairs
	z_ndcg = delta_ndcg(true_scores, i, j)
	rho = 1 / (1 + np.exp(predicted_scores[i] - predicted_scores[j]))
	rho_complement = 1.0 - rho
	lambda_val = z_ndcg * rho
	# lambdas = []
	# lambdas.append(lambda_val)
	# lambdas.append(-lambda_val)
	# w = []
	w_val = rho * rho_complement * z_ndcg
	# w.append(w_val)
	# w.append(w_val)
	return i, j, lambda_val, w_val

#true_scores, predicted_scores
def compute_lambda(args):
	true_scores, predicted_scores, good_ij_pairs, idcg, query_key = args
	# true_scores, predicted_scores, good_ij_pairs = scores
	num_docs = len(true_scores)
	sorted_indexes = np.argsort(predicted_scores)[::-1]
	rev_indexes = np.argsort(sorted_indexes)
	true_scores = true_scores[sorted_indexes]
	predicted_scores = predicted_scores[sorted_indexes]

	lambdas = np.zeros(num_docs)
	w = np.zeros(num_docs)

	single_dcgs = {}
	for i,j in good_ij_pairs:
		if (i,i) not in single_dcgs:
			single_dcgs[(i,i)] = single_dcg(true_scores, i, i)
		single_dcgs[(i,j)] = single_dcg(true_scores, i, j)
		if (j,j) not in single_dcgs:
			single_dcgs[(j,j)] = single_dcg(true_scores, j, j)
		single_dcgs[(j,i)] = single_dcg(true_scores, j, i)
	### parallel portion
	# pool = Pool(processes=8)
	# for i, j, lambda_val, w_val in pool.map(lambda_parallel, zip([true_scores for i in xrange(len(good_ij_pairs))], good_ij_pairs, [predicted_scores for i in xrange(len(good_ij_pairs))])):
	# 	lambdas[i] += lambda_val
	# 	lambdas[j] -= lambda_val
	# 	w[i] += w_val
	# 	w[j] += w_val
	# pool.close()

	for i,j in good_ij_pairs:
		# z_ndcg = delta_ndcg(true_scores, i, j, idcg)
		z_ndcg = abs(single_dcgs[(i,j)] - single_dcgs[(i,i)] + single_dcgs[(j,i)] - single_dcgs[(j,j)]) / idcg
		rho = 1 / (1 + np.exp(predicted_scores[i] - predicted_scores[j]))
		rho_complement = 1.0 - rho
		lambda_val = z_ndcg * rho
		lambdas[i] += lambda_val
		lambdas[j] -= lambda_val

		w_val = rho * rho_complement * z_ndcg
		w[i] += w_val
		w[j] += w_val

	return lambdas[rev_indexes], w[rev_indexes], query_key

def group_queries(training_data):
	query_indexes = {}
	index = 0
	for record in training_data:
		query_indexes.setdefault(record[1], [])
		query_indexes[record[1]].append(index)
		index += 1
	return query_indexes

def get_pairs(scores):
	query_pair = []
	for query_scores in scores:
		temp = sorted(query_scores, reverse=True)
		pairs = []
		for i in xrange(len(temp)):
			for j in xrange(len(temp)):
				if temp[i] > temp[j]:
					pairs.append((i,j))
		query_pair.append(pairs)
	return query_pair

class LambdaMART:

	def __init__(self, training_data, number_of_trees, leaves_per_tree, learning_rate):
		'''
		The format for training data is as follows:
			[relevance, q_id, [feature vector]]
		'''
		self.training_data = np.array(training_data)
		self.number_of_trees = number_of_trees
		self.leaves_per_tree = leaves_per_tree
		self.learning_rate = learning_rate
		self.trees = []

	def fit(self):
		predicted_scores = np.zeros(len(self.training_data))
		query_indexes = group_queries(self.training_data)
		query_keys = query_indexes.keys()
		true_scores = [self.training_data[query_indexes[query], 0] for query in query_keys]
		good_ij_pairs = get_pairs(true_scores)
		tree_data = pd.DataFrame(self.training_data[:, 2:7])
		labels = self.training_data[:, 0]

		# ideal dcg calculation
		idcg = [ideal_dcg(scores) for scores in true_scores]

		for k in xrange(self.number_of_trees):
			print 'Tree %d' % (k)
			lambdas = np.zeros(len(predicted_scores))
			w = np.zeros(len(predicted_scores))
			pred_scores = [predicted_scores[query_indexes[query]] for query in query_keys]

			pool = Pool()
			for lambda_val, w_val, query_key in pool.map(compute_lambda, zip(true_scores, pred_scores, good_ij_pairs, idcg, query_keys), chunksize=1):
				indexes = query_indexes[query_key]
				lambdas[indexes] = lambda_val
				w[indexes] = w_val
			pool.close()

			## non parallel
			# for i in xrange(len(true_scores)):
			# 	query = query_keys[i]
			# 	lambdas[query_indexes[query]], w[query_indexes[query]] = compute_lambda(true_scores[i], pred_scores[i], good_ij_pairs[i])

			# sklearn tree
			tree = DecisionTreeRegressor(max_depth=50)
			tree.fit(self.training_data[:,2:], lambdas)
			self.trees.append(tree)
			prediction = tree.predict(self.training_data[:,2:])
			predicted_scores += prediction * self.learning_rate
			# tree = RegressionTree(tree_data, lambdas, max_depth=10, ideal_ls= 0.001)
			# print 'created tree'
			# tree.fit()
			# print 'fitted tree'
			# prediction = tree.predict(self.training_data[:,2:])
			# print 'predicted tree'
			# print prediction
			# # exit()
			# predicted_scores = essemble_trees(lambdas, w, prediction, predicted_scores, self.learning_rate)
			# print 'updates scores'

			###
			# Tree code here, already calculated lambdas and ws
			###

		# print predicted_scores

	def predict(self, data):
		data = np.array(data)
		query_indexes = group_queries(data)
		average_ndcg = 0
		predicted_scores = np.zeros(len(data))
		for query in query_indexes:
			results = np.zeros(len(query_indexes[query]))
			for tree in self.trees:
				results += self.learning_rate * tree.predict(data[query_indexes[query], 2:])
			predicted_scores[query_indexes[query]] = results
			ndcg_val = (dcg(results) / ideal_dcg(results))
			average_ndcg += ndcg_val
		average_ndcg /= len(query_indexes)
		return average_ndcg, predicted_scores

def main():
	f = open('vali.txt', 'r')
	count = 0
	training_data = []
	test_data = []
	for line in f:
		if count >= 240:
			break
		new_arr = []
		arr = line.split(' #')[0].split()
		score = arr[0]
		q_id = arr[1].split(':')[1]
		new_arr.append(int(score))
		new_arr.append(int(q_id))
		arr = arr[2:]
		for el in arr:
			new_arr.append(float(el.split(':')[1]))
		if count < 200:
			training_data.append(new_arr)
		else:
			test_data.append(new_arr)
		count += 1
	f.close()
	model = LambdaMART(training_data, 500, 10, 0.001)
	model.fit()
	average_ndcg, predicted_scores = model.predict(test_data)

if __name__ == '__main__':
	main()<|endoftext|>