<commit_before>import tornado.ioloop
import tornado.web
import tornado.websocket

from syncrae.events.event import Event
from syncrae.session import Session

from django.conf import settings

from events.queue import CampaignQueue
import logging
import simplejson
import tornado.ioloop
import tornado.web
import tornado.websocket
from django.contrib.auth import get_user
from django.utils.importlib import import_module
from time import time
from functools import wraps

logging = logging.getLogger('')

class Dummy(object):
    def __init__(self, session):
        self.session = session

class EventWebsocket(tornado.websocket.WebSocketHandler):

    def async(self, func, args=None, kwargs=None):
        if args is None:
            args = []
        if kwargs is None:
            kwargs = {}

        # wraps the function to pass the given args
        @wraps(func)
        def wrapper():
            return func(*args, **kwargs)

        # Use tornado's async timer, set to now
        tornado.ioloop.IOLoop.instance().add_callback(wrapper)

    def get_current_user(self):
        engine = import_module(settings.SESSION_ENGINE)
        sessionid = self.get_cookie(settings.SESSION_COOKIE_NAME)

        self.webdnd_session = engine.SessionStore(sessionid)
        request = Dummy(self.webdnd_session)

        return get_user(request)

    def open(self):
        user = self.get_current_user()
        if not user or not user.is_authenticated():
            return self.reject()

        self.user = user
        self.session = Session.get(self.user.id)
        if self.session is None:
            self.session = Session(self)
        else:
            self.session.listen(self)

        self.queue = CampaignQueue.get(self.webdnd_session['cid'])
        self.queue.listen(self)

        # Send the 'New user' event
        self.queue.write_message('/sessions/new', {
            'name':  self.user.name,
        })

    def on_message(self, raw_message):
        try:
            message = simplejson.loads(raw_message)
            topic = message['topic']
            data = message['data']
        except BaseException as err:
            logging.exception('Cannot parse message: %s' % raw_message, err)

        if topic in EventWebsocket.TOPICS.keys():
            if hasattr(self, EventWebsocket.TOPICS[topic]):
                return getattr(self, EventWebsocket.TOPICS[topic])(data)
            else:
                logging.error('Invalid Handler for topic: %s' % topic)
                return
        else:
            # emit event to all listeners
            self.queue.write_message(topic, data)

    def on_close(self):
        self.queue.drop(self)
        self.session.drop(self)

    ##############################################
    # Topic Handlers
    ##############################################

    TOPICS = {
    }

    def reject(self):
        '''
        Occurs when the session is rejected
        '''
        Event('/sessions/error', {
            'error': 'Your gameplay key was wrong, go back to the campaign and try again.',
        }).write_message(self)
        self.close()

application = None

def get_app(settings):
    global application
    if application is None:
        application = tornado.web.Application([
            # Application URI's
            (r'/event', EventWebsocket),
        ], **settings)

    return application
<commit_msg>Formatting, sorted imports.<commit_after>from django.conf import settings
from django.contrib.auth import get_user
from django.utils.importlib import import_module

from events.queue import CampaignQueue
from functools import wraps
from syncrae.events.event import Event
from syncrae.session import Session
from time import sleep
import logging
import simplejson
import tornado.ioloop
import tornado.web
import tornado.websocket

logging = logging.getLogger('')

class RequestDummy(object):
    def __init__(self, session):
        self.session = session

class EventWebsocket(tornado.websocket.WebSocketHandler):

    def async(self, func, args=None, kwargs=None):
        if args is None:
            args = []
        if kwargs is None:
            kwargs = {}

        # wraps the function to pass the given args
        @wraps(func)
        def wrapper():
            return func(*args, **kwargs)

        # Use tornado's async timer, set to now
        tornado.ioloop.IOLoop.instance().add_callback(wrapper)

    def get_current_user(self):
        engine = import_module(settings.SESSION_ENGINE)
        sessionid = self.get_cookie(settings.SESSION_COOKIE_NAME)

        self.webdnd_session = engine.SessionStore(sessionid)
        request = RequestDummy(self.webdnd_session)

        return get_user(request)

    def open(self):
        user = self.get_current_user()
        if not user or not user.is_authenticated():
            return self.reject()

        self.user = user
        self.session = Session.get(self.user.id)
        if self.session is None:
            self.session = Session(self)
        else:
            self.session.listen(self)

        self.queue = CampaignQueue.get(self.webdnd_session['cid'])
        self.queue.listen(self)

        # Send the 'New user' event
        self.queue.write_message('/sessions/new', {
            'name':  self.user.name,
        })

    def on_message(self, raw_message):
        try:
            message = simplejson.loads(raw_message)
            topic = message['topic']
            data = message['data']
        except BaseException as err:
            logging.exception('Cannot parse message: %s' % raw_message, err)

        if topic in EventWebsocket.TOPICS.keys():
            if hasattr(self, EventWebsocket.TOPICS[topic]):
                return getattr(self, EventWebsocket.TOPICS[topic])(data)
            else:
                logging.error('Invalid Handler for topic: %s' % topic)
                return
        else:
            # emit event to all listeners
            self.queue.write_message(topic, data)

    def on_close(self):
        self.queue.drop(self)
        self.session.drop(self)

    ##############################################
    # Topic Handlers
    ##############################################

    TOPICS = {
    }

    def reject(self):
        '''
        Occurs when the session is rejected
        '''
        Event('/sessions/error', {
            'error': 'Your gameplay key was wrong, go back to the campaign and try again.',
        }).write_message(self)
        self.close()

application = None

def get_app(settings):
    global application
    if application is None:
        application = tornado.web.Application([
            # Application URI's
            (r'/event', EventWebsocket),
        ], **settings)

    return application
<|endoftext|>


<commit_before>import collections

OUTPUT=0
INPUT=1
LOOP=2
ENDLOOP=3
MOVE=4
ADD=5
SET=6
MULCOPY=7
SCAN=8
LOADOUT=9
LOADOUTSET=10
IF=11
ENDIF=12

def parse(code):
    code = ''.join(i for i in code if i in '.,[]+-<>')
    tokens = []
    for i in code:
        if i == '+':
            tokens.append((ADD, (0, 1)))
        elif i == '-':
            tokens.append((ADD, (0, -1)))
        elif i == '>':
            tokens.append((MOVE, 1))
        elif i == '<':
            tokens.append((MOVE, -1))
        elif i == '[':
            tokens.append((LOOP, None))
        elif i == ']':
            tokens.append((ENDLOOP, None))
        elif i == ',':
            tokens.append((INPUT, 1))
        if i == '.':
            tokens.append((LOADOUT, (0, 0)))
            tokens.append((OUTPUT, None))
    return tokens

def optimize(tokens):
    # Optimize various things
    newtokens = []
    shift = 0
    # With normal dict, the order sometimes switches
    # in recursion, and the optimized never exits.
    vals = collections.OrderedDict()
    ops = {}
    do_output = False
    for token, value in tokens:
        if token not in (SET, ADD):
            for k, v in vals.items():
                newtokens.append((ops[k], (k, v)))
            vals.clear()
            ops.clear()
        elif do_output and token not in (LOADOUT, LOADOUTSET):
            newtokens.append((OUTPUT, None))
            do_output = False

        if token in (SET, ADD):
            offset, val = value
            offset += shift
            # ADD/SET then SET does nothing; remove it
            if offset in ops and token == SET:
                ops.pop(offset)
                vals.pop(offset)
            if offset not in ops:
                ops[offset] = token # SET or ADD
            vals[offset] = vals.get(offset, 0) + val
        elif token == MULCOPY:
            src, dest, mul = value
            newtokens.append((MULCOPY, (src+shift, dest+shift, mul)))
        # XXX Deal with shift in if
        elif token == IF:
            offset = shift + value
            newtokens.append((IF, offset))
        elif token == ENDIF:
            newtokens.append((ENDIF, None))
        elif token == MOVE:
            shift += value
        elif token == OUTPUT:
            do_output = True
        elif token in (LOOP, ENDLOOP, INPUT, SCAN, LOADOUT, LOADOUTSET):
            if shift:
                newtokens.append((MOVE, shift))
                shift = 0
            newtokens.append((token, value))
        else:
            raise ValueError('What is this ' + str(token) + ' doing here?')

    # Any remaining add/set/shift is ignored, as it would have no effect
    if do_output:
        newtokens.append((OUTPUT, None))

    newtokens2 = []

    i = 0
    while i < len(newtokens):
        optimized = False

        # Optimize scan loop
        if (i < len(newtokens)-2 and
             newtokens[i][0] == LOOP and
             newtokens[i+1][0] == MOVE and
             newtokens[i+2][0] == ENDLOOP):

            offset = newtokens[i+1][1]
            newtokens2.append((SCAN, offset))
            optimized = True
            i += 2

        # Optimize out clear loop / multiply move loop
        if not optimized and newtokens[i][0] == LOOP:
            j = i + 1
            adds = {}
            sets = {}
            while j < len(newtokens) and newtokens[j][0] != ENDLOOP:
                if newtokens[j][0] == ADD:
                    offset, add = newtokens[j][1]
                    if offset in sets:
                        sets[offset] += add
                    else:
                        adds[offset] = adds.get(offset, 0) + add
                elif newtokens[j][0] == SET and newtokens[j][1][0] != 0:
                    offset, val = newtokens[j][1]
                    if offset in adds:
                        del adds[offset]
                    sets[offset] = val
                else:
                    break
                j += 1
            else:
                if 0 not in adds:
                    pass
                    # print("Warning: Infinite loop detected.")
                elif len(adds) == 1:
                    if sets:
                        newtokens2.append((IF, 0))
                        for offset, val in sets.items():
                            newtokens2.append((SET, (offset, val)))
                    newtokens2.append((SET, (0, 0)))
                    if sets:
                        newtokens2.append((ENDIF, None))
                    i = j
                    optimized = True
                elif adds[0] == -1:
                    if sets:
                        newtokens2.append((IF, 0))
                        for offset, val in sets.items():
                            newtokens2.append((SET, (offset, val)))
                    for k, v in adds.items():
                        if k != 0:
                            newtokens2.append((MULCOPY, (0, k, v)))
                    if sets:
                        newtokens2.append((ENDIF, None))
                    newtokens2.append((SET, (0, 0)))
                    i = j
                    optimized = True

        # SET + MULCOPY = SET + ADD
        if (not optimized and
             i < len(newtokens)-1 and
             newtokens[i][0] == SET and
             newtokens[i+1][0] == MULCOPY and
             newtokens[i][1][0] == newtokens[i+1][1][0]):

            offset, value = newtokens[i][1]
            j = i+1
            while (j < len(newtokens)-1 and
                   newtokens[j][0] == MULCOPY and
                   newtokens[j][1][0] == offset):
                src, dest, mul = newtokens[j][1]
                newtokens2.append((ADD, (dest, value*mul)))
                j += 1

            newtokens2.append((SET, (offset, value)))
            i = j - 1
            optimized = True

        # Optimize ADD/SET/MOVE + OUTPUT + ADD/SET/MOVE
        if (not optimized and
             i < len(newtokens)-2 and
             newtokens[i][0] in (ADD, MOVE, SET)):
            j = i
            outputs = []
            adds = {}
            sets = {}
            shift = 0
            shifted = False
            while j < len(newtokens):
                if newtokens[j][0] == ADD:
                    offset, val = newtokens[j][1]
                    offset += shift
                    adds[offset] = adds.get(offset, 0) + val
                elif newtokens[j][0] == SET:
                    offset, val = newtokens[j][1]
                    offset += shift
                    adds[offset] = 0
                    sets[offset] = val
                elif newtokens[j][0] == LOADOUT:
                    offset, add = newtokens[j][1]
                    offset += shift
                    if offset in sets:
                        val = sets[offset] + adds.get(offset, 0) + add
                        outputs.append((None, None, val))
                    else:
                        outputs.append((offset, adds.get(offset, 0) + add, None))
                elif newtokens[j][0] == LOADOUTSET:
                    value = newtokens[j][1]
                    outputs.append((None, None, value))
                elif newtokens[j][0] == MOVE:
                    shift += newtokens[j][1]
                    shifted = True
                elif newtokens[j][0] == OUTPUT:
                    pass
                else:
                    j -= 1
                    break
                j += 1

            if (adds or shifted or sets) and outputs:
                for offset, add, _set in outputs:
                    if _set is not None:
                        newtokens2.append((LOADOUTSET, _set))
                    else:
                        newtokens2.append((LOADOUT, (offset, add)))
                newtokens2.append((OUTPUT, None))
                for offset, val in sets.items():
                    val = val + adds.get(offset, 0)
                    newtokens2.append((SET, (offset, val)))
                for offset, add in adds.items():
                    if add and (offset not in sets):
                        newtokens2.append((ADD, (offset, add)))
                if shift:
                    newtokens2.append((MOVE, shift))
                i = j
                optimized = True

        if not optimized:
            newtokens2.append(newtokens[i])

        i += 1

    # Optimize recursively
    if newtokens2 != tokens:
        return optimize(newtokens2)

    return newtokens2
<commit_msg>Move output optimization to new loop<commit_after>import collections

OUTPUT=0
INPUT=1
LOOP=2
ENDLOOP=3
MOVE=4
ADD=5
SET=6
MULCOPY=7
SCAN=8
LOADOUT=9
LOADOUTSET=10
IF=11
ENDIF=12

def parse(code):
    code = ''.join(i for i in code if i in '.,[]+-<>')
    tokens = []
    for i in code:
        if i == '+':
            tokens.append((ADD, (0, 1)))
        elif i == '-':
            tokens.append((ADD, (0, -1)))
        elif i == '>':
            tokens.append((MOVE, 1))
        elif i == '<':
            tokens.append((MOVE, -1))
        elif i == '[':
            tokens.append((LOOP, None))
        elif i == ']':
            tokens.append((ENDLOOP, None))
        elif i == ',':
            tokens.append((INPUT, 1))
        if i == '.':
            tokens.append((LOADOUT, (0, 0)))
            tokens.append((OUTPUT, None))
    return tokens

def optimize(tokens):
    # Optimize various things
    newtokens = []
    shift = 0
    # With normal dict, the order sometimes switches
    # in recursion, and the optimized never exits.
    vals = collections.OrderedDict()
    ops = {}
    do_output = False
    for token, value in tokens:
        if token not in (SET, ADD, MOVE, LOADOUT, LOADOUTSET):
            if do_output:
                newtokens.append((OUTPUT, None))
                do_output = False

            for k, v in vals.items():
                newtokens.append((ops[k], (k, v)))
            vals.clear()
            ops.clear()

        if token in (SET, ADD):
            offset, val = value
            offset += shift
            # ADD/SET then SET does nothing; remove it
            if offset in ops and token == SET:
                ops.pop(offset)
                vals.pop(offset)
            if offset not in ops:
                ops[offset] = token # SET or ADD
            vals[offset] = vals.get(offset, 0) + val
        elif token == MULCOPY:
            src, dest, mul = value
            newtokens.append((MULCOPY, (src+shift, dest+shift, mul)))
        # XXX Deal with shift in if
        elif token == IF:
            offset = shift + value
            newtokens.append((IF, offset))
        elif token == ENDIF:
            newtokens.append((ENDIF, None))
        elif token == MOVE:
            shift += value
        elif token == OUTPUT:
            do_output = True
        elif token == LOADOUT:
            offset, add = value
            offset += shift
            if ops.get(offset) == SET:
                newtokens.append((LOADOUTSET, vals[offset] + add))
            elif ops.get(offset) == ADD:
                newtokens.append((LOADOUT, (offset, vals[offset] + add)))
            else:
                newtokens.append((LOADOUT, (offset, add)))
        elif token == LOADOUTSET:
            newtokens.append((token, value))
        elif token in (LOOP, ENDLOOP, INPUT, SCAN):
            if shift:
                newtokens.append((MOVE, shift))
                shift = 0
            newtokens.append((token, value))
        else:
            raise ValueError('What is this ' + str(token) + ' doing here?')

    # Any remaining add/set/shift is ignored, as it would have no effect
    if do_output:
        newtokens.append((OUTPUT, None))

    newtokens2 = []

    i = 0
    while i < len(newtokens):
        optimized = False

        # Optimize scan loop
        if (i < len(newtokens)-2 and
             newtokens[i][0] == LOOP and
             newtokens[i+1][0] == MOVE and
             newtokens[i+2][0] == ENDLOOP):

            offset = newtokens[i+1][1]
            newtokens2.append((SCAN, offset))
            optimized = True
            i += 2

        # Optimize out clear loop / multiply move loop
        if not optimized and newtokens[i][0] == LOOP:
            j = i + 1
            adds = {}
            sets = {}
            while j < len(newtokens) and newtokens[j][0] != ENDLOOP:
                if newtokens[j][0] == ADD:
                    offset, add = newtokens[j][1]
                    if offset in sets:
                        sets[offset] += add
                    else:
                        adds[offset] = adds.get(offset, 0) + add
                elif newtokens[j][0] == SET and newtokens[j][1][0] != 0:
                    offset, val = newtokens[j][1]
                    if offset in adds:
                        del adds[offset]
                    sets[offset] = val
                else:
                    break
                j += 1
            else:
                if 0 not in adds:
                    pass
                    # print("Warning: Infinite loop detected.")
                elif len(adds) == 1:
                    if sets:
                        newtokens2.append((IF, 0))
                        for offset, val in sets.items():
                            newtokens2.append((SET, (offset, val)))
                    newtokens2.append((SET, (0, 0)))
                    if sets:
                        newtokens2.append((ENDIF, None))
                    i = j
                    optimized = True
                elif adds[0] == -1:
                    if sets:
                        newtokens2.append((IF, 0))
                        for offset, val in sets.items():
                            newtokens2.append((SET, (offset, val)))
                    for k, v in adds.items():
                        if k != 0:
                            newtokens2.append((MULCOPY, (0, k, v)))
                    if sets:
                        newtokens2.append((ENDIF, None))
                    newtokens2.append((SET, (0, 0)))
                    i = j
                    optimized = True

        # SET + MULCOPY = SET + ADD
        if (not optimized and
             i < len(newtokens)-1 and
             newtokens[i][0] == SET and
             newtokens[i+1][0] == MULCOPY and
             newtokens[i][1][0] == newtokens[i+1][1][0]):

            offset, value = newtokens[i][1]
            j = i+1
            while (j < len(newtokens)-1 and
                   newtokens[j][0] == MULCOPY and
                   newtokens[j][1][0] == offset):
                src, dest, mul = newtokens[j][1]
                newtokens2.append((ADD, (dest, value*mul)))
                j += 1

            newtokens2.append((SET, (offset, value)))
            i = j - 1
            optimized = True

        if not optimized:
            newtokens2.append(newtokens[i])

        i += 1

    # Optimize recursively
    if newtokens2 != tokens:
        return optimize(newtokens2)

    return newtokens2
<|endoftext|>


<commit_before># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The TFLite Support Library.

Install the pip package:

```
pip install tflite-support
```

To learn more about metadata, flatbuffers and TensorFlow Lite models, check out
the [metadata section](https://www.tensorflow.org/lite/convert/metadata) of the
TF Lite guide.
"""

# In the original project structure, all python targets are accessed by paths
# like tensorflow_lite_support.metadata.metadata.MetadataDisplayer, which is
# verbose and deep. This file provides some shortcuts. It's also compatible
# with our first version Pip package.

# In pip build, this file will be renamed as tflite_support/__init__.py.

import flatbuffers
from tensorflow_lite_support.metadata import metadata_schema_py_generated
from tensorflow_lite_support.metadata import schema_py_generated
from tensorflow_lite_support.metadata.python import metadata
from tflite_support import metadata_writers
from tflite_support import task
<commit_msg>Remove Task Library from Pypi __init__ on Windows.<commit_after># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The TFLite Support Library.

Install the pip package:

```
pip install tflite-support
```

To learn more about metadata, flatbuffers and TensorFlow Lite models, check out
the [metadata section](https://www.tensorflow.org/lite/convert/metadata) of the
TF Lite guide.
"""

# In the original project structure, all python targets are accessed by paths
# like tensorflow_lite_support.metadata.metadata.MetadataDisplayer, which is
# verbose and deep. This file provides some shortcuts. It's also compatible
# with our first version Pip package.

# In pip build, this file will be renamed as tflite_support/__init__.py.

import flatbuffers
import platform

from tensorflow_lite_support.metadata import metadata_schema_py_generated
from tensorflow_lite_support.metadata import schema_py_generated
from tensorflow_lite_support.metadata.python import metadata
from tflite_support import metadata_writers

if platform.system() != 'Windows':
  # Task Library is not supported on Windows yet.
  from tflite_support import task
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import logging
import logging.handlers
import os
import platform
import pprint
from . import release
import sys
import threading
import time

import psycopg2

import odoo
from . import sql_db
from . import tools

_logger = logging.getLogger(__name__)

def log(logger, level, prefix, msg, depth=None):
    indent=''
    indent_after=' '*len(prefix)
    for line in (prefix + pprint.pformat(msg, depth=depth)).split('\n'):
        logger.log(level, indent+line)
        indent=indent_after

path_prefix = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))

class PostgreSQLHandler(logging.Handler):
    """ PostgreSQL Logging Handler will store logs in the database, by default
    the current database, can be set using --log-db=DBNAME
    """
    def emit(self, record):
        ct = threading.current_thread()
        ct_db = getattr(ct, 'dbname', None)
        dbname = tools.config['log_db'] if tools.config['log_db'] and tools.config['log_db'] != '%d' else ct_db
        if not dbname:
            return
        with tools.ignore(Exception), tools.mute_logger('odoo.sql_db'), sql_db.db_connect(dbname, allow_uri=True).cursor() as cr:
            # preclude risks of deadlocks
            cr.execute("SET LOCAL statement_timeout = 1000")
            msg = tools.ustr(record.msg)
            if record.args:
                msg = msg % record.args
            traceback = getattr(record, 'exc_text', '')
            if traceback:
                msg = "%s\n%s" % (msg, traceback)
            # we do not use record.levelname because it may have been changed by ColoredFormatter.
            levelname = logging.getLevelName(record.levelno)

            val = ('server', ct_db, record.name, levelname, msg, record.pathname[len(path_prefix)+1:], record.lineno, record.funcName)
            cr.execute("""
                INSERT INTO ir_logging(create_date, type, dbname, name, level, message, path, line, func)
                VALUES (NOW() at time zone 'UTC', %s, %s, %s, %s, %s, %s, %s, %s)
            """, val)

BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, _NOTHING, DEFAULT = range(10)
#The background is set with 40 plus the number of the color, and the foreground with 30
#These are the sequences need to get colored ouput
RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
BOLD_SEQ = "\033[1m"
COLOR_PATTERN = "%s%s%%s%s" % (COLOR_SEQ, COLOR_SEQ, RESET_SEQ)
LEVEL_COLOR_MAPPING = {
    logging.DEBUG: (BLUE, DEFAULT),
    logging.INFO: (GREEN, DEFAULT),
    logging.WARNING: (YELLOW, DEFAULT),
    logging.ERROR: (RED, DEFAULT),
    logging.CRITICAL: (WHITE, RED),
}

class PerfFilter(logging.Filter):
    def format_perf(self, query_count, query_time, remaining_time):
        return ("%d" % query_count, "%.3f" % query_time, "%.3f" % remaining_time)

    def filter(self, record):
        if hasattr(threading.current_thread(), "query_count"):
            query_count = threading.current_thread().query_count
            query_time = threading.current_thread().query_time
            perf_t0 = threading.current_thread().perf_t0
            remaining_time = time.time() - perf_t0 - query_time
            record.perf_info = '%s %s %s' % self.format_perf(query_count, query_time, remaining_time)
            delattr(threading.current_thread(), "query_count")
        else:
            record.perf_info = "- - -"
        return True

class ColoredPerfFilter(PerfFilter):
    def format_perf(self, query_count, query_time, remaining_time):
        def colorize_time(time, format, low=1, high=5):
            if time > high:
                return COLOR_PATTERN % (30 + RED, 40 + DEFAULT, format % time)
            if time > low:
                return COLOR_PATTERN % (30 + YELLOW, 40 + DEFAULT, format % time)
            return format % time
        return (
            colorize_time(query_count, "%d", 100, 1000),
            colorize_time(query_time, "%.3f", 0.1, 3),
            colorize_time(remaining_time, "%.3f", 1, 5)
            )

class DBFormatter(logging.Formatter):
    def format(self, record):
        record.pid = os.getpid()
        record.dbname = getattr(threading.current_thread(), 'dbname', '?')
        return logging.Formatter.format(self, record)

class ColoredFormatter(DBFormatter):
    def format(self, record):
        fg_color, bg_color = LEVEL_COLOR_MAPPING.get(record.levelno, (GREEN, DEFAULT))
        record.levelname = COLOR_PATTERN % (30 + fg_color, 40 + bg_color, record.levelname)
        return DBFormatter.format(self, record)

_logger_init = False
def init_logger():
    global _logger_init
    if _logger_init:
        return
    _logger_init = True

    old_factory = logging.getLogRecordFactory()
    def record_factory(*args, **kwargs):
        record = old_factory(*args, **kwargs)
        record.perf_info = ""
        return record
    logging.setLogRecordFactory(record_factory)

    logging.addLevelName(25, "INFO")
    logging.captureWarnings(True)

    from .tools.translate import resetlocale
    resetlocale()

    # create a format for log messages and dates
    format = '%(asctime)s %(pid)s %(levelname)s %(dbname)s %(name)s: %(message)s %(perf_info)s'
    # Normal Handler on stderr
    handler = logging.StreamHandler()

    if tools.config['syslog']:
        # SysLog Handler
        if os.name == 'nt':
            handler = logging.handlers.NTEventLogHandler("%s %s" % (release.description, release.version))
        elif platform.system() == 'Darwin':
            handler = logging.handlers.SysLogHandler('/var/run/log')
        else:
            handler = logging.handlers.SysLogHandler('/dev/log')
        format = '%s %s' % (release.description, release.version) \
                + ':%(dbname)s:%(levelname)s:%(name)s:%(message)s'

    elif tools.config['logfile']:
        # LogFile Handler
        logf = tools.config['logfile']
        try:
            # We check we have the right location for the log files
            dirname = os.path.dirname(logf)
            if dirname and not os.path.isdir(dirname):
                os.makedirs(dirname)
            if os.name == 'posix':
                handler = logging.handlers.WatchedFileHandler(logf)
            else:
                handler = logging.FileHandler(logf)
        except Exception:
            sys.stderr.write("ERROR: couldn't create the logfile directory. Logging to the standard output.\n")

    # Check that handler.stream has a fileno() method: when running OpenERP
    # behind Apache with mod_wsgi, handler.stream will have type mod_wsgi.Log,
    # which has no fileno() method. (mod_wsgi.Log is what is being bound to
    # sys.stderr when the logging.StreamHandler is being constructed above.)
    def is_a_tty(stream):
        return hasattr(stream, 'fileno') and os.isatty(stream.fileno())

    if os.name == 'posix' and isinstance(handler, logging.StreamHandler) and is_a_tty(handler.stream):
        formatter = ColoredFormatter(format)
        perf_filter = ColoredPerfFilter()
    else:
        formatter = DBFormatter(format)
        perf_filter = PerfFilter()
    handler.setFormatter(formatter)
    logging.getLogger().addHandler(handler)
    logging.getLogger('werkzeug').addFilter(perf_filter)

    if tools.config['log_db']:
        db_levels = {
            'debug': logging.DEBUG,
            'info': logging.INFO,
            'warning': logging.WARNING,
            'error': logging.ERROR,
            'critical': logging.CRITICAL,
        }
        postgresqlHandler = PostgreSQLHandler()
        postgresqlHandler.setLevel(int(db_levels.get(tools.config['log_db_level'], tools.config['log_db_level'])))
        logging.getLogger().addHandler(postgresqlHandler)

    # Configure loggers levels
    pseudo_config = PSEUDOCONFIG_MAPPER.get(tools.config['log_level'], [])

    logconfig = tools.config['log_handler']

    logging_configurations = DEFAULT_LOG_CONFIGURATION + pseudo_config + logconfig
    for logconfig_item in logging_configurations:
        loggername, level = logconfig_item.split(':')
        level = getattr(logging, level, logging.INFO)
        logger = logging.getLogger(loggername)
        logger.setLevel(level)

    for logconfig_item in logging_configurations:
        _logger.debug('logger level set: "%s"', logconfig_item)

DEFAULT_LOG_CONFIGURATION = [
    'odoo.http.rpc.request:INFO',
    'odoo.http.rpc.response:INFO',
    ':INFO',
]
PSEUDOCONFIG_MAPPER = {
    'debug_rpc_answer': [':DEBUG', 'odoo.sql_db:INFO', 'odoo.http.rpc:DEBUG'],
    'debug_rpc': [':DEBUG', 'odoo.sql_db:INFO', 'odoo.http.rpc.request:DEBUG'],
    'debug': [':DEBUG', 'odoo.sql_db:INFO'],
    'debug_sql': ['odoo.sql_db:DEBUG'],
    'info': [],
    'warn': [':WARNING'],
    'error': [':ERROR'],
    'critical': [':CRITICAL'],
}
<commit_msg>[REV] core: logger configuration part of 6044782aeffc61955b956068b0a4d58fa33303f0<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import logging
import logging.handlers
import os
import platform
import pprint
from . import release
import sys
import threading
import time

import psycopg2

import odoo
from . import sql_db
from . import tools

_logger = logging.getLogger(__name__)

def log(logger, level, prefix, msg, depth=None):
    indent=''
    indent_after=' '*len(prefix)
    for line in (prefix + pprint.pformat(msg, depth=depth)).split('\n'):
        logger.log(level, indent+line)
        indent=indent_after

path_prefix = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))

class PostgreSQLHandler(logging.Handler):
    """ PostgreSQL Logging Handler will store logs in the database, by default
    the current database, can be set using --log-db=DBNAME
    """
    def emit(self, record):
        ct = threading.current_thread()
        ct_db = getattr(ct, 'dbname', None)
        dbname = tools.config['log_db'] if tools.config['log_db'] and tools.config['log_db'] != '%d' else ct_db
        if not dbname:
            return
        with tools.ignore(Exception), tools.mute_logger('odoo.sql_db'), sql_db.db_connect(dbname, allow_uri=True).cursor() as cr:
            # preclude risks of deadlocks
            cr.execute("SET LOCAL statement_timeout = 1000")
            msg = tools.ustr(record.msg)
            if record.args:
                msg = msg % record.args
            traceback = getattr(record, 'exc_text', '')
            if traceback:
                msg = "%s\n%s" % (msg, traceback)
            # we do not use record.levelname because it may have been changed by ColoredFormatter.
            levelname = logging.getLevelName(record.levelno)

            val = ('server', ct_db, record.name, levelname, msg, record.pathname[len(path_prefix)+1:], record.lineno, record.funcName)
            cr.execute("""
                INSERT INTO ir_logging(create_date, type, dbname, name, level, message, path, line, func)
                VALUES (NOW() at time zone 'UTC', %s, %s, %s, %s, %s, %s, %s, %s)
            """, val)

BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, _NOTHING, DEFAULT = range(10)
#The background is set with 40 plus the number of the color, and the foreground with 30
#These are the sequences need to get colored ouput
RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
BOLD_SEQ = "\033[1m"
COLOR_PATTERN = "%s%s%%s%s" % (COLOR_SEQ, COLOR_SEQ, RESET_SEQ)
LEVEL_COLOR_MAPPING = {
    logging.DEBUG: (BLUE, DEFAULT),
    logging.INFO: (GREEN, DEFAULT),
    logging.WARNING: (YELLOW, DEFAULT),
    logging.ERROR: (RED, DEFAULT),
    logging.CRITICAL: (WHITE, RED),
}

class PerfFilter(logging.Filter):
    def format_perf(self, query_count, query_time, remaining_time):
        return ("%d" % query_count, "%.3f" % query_time, "%.3f" % remaining_time)

    def filter(self, record):
        if hasattr(threading.current_thread(), "query_count"):
            query_count = threading.current_thread().query_count
            query_time = threading.current_thread().query_time
            perf_t0 = threading.current_thread().perf_t0
            remaining_time = time.time() - perf_t0 - query_time
            record.perf_info = '%s %s %s' % self.format_perf(query_count, query_time, remaining_time)
            delattr(threading.current_thread(), "query_count")
        else:
            record.perf_info = "- - -"
        return True

class ColoredPerfFilter(PerfFilter):
    def format_perf(self, query_count, query_time, remaining_time):
        def colorize_time(time, format, low=1, high=5):
            if time > high:
                return COLOR_PATTERN % (30 + RED, 40 + DEFAULT, format % time)
            if time > low:
                return COLOR_PATTERN % (30 + YELLOW, 40 + DEFAULT, format % time)
            return format % time
        return (
            colorize_time(query_count, "%d", 100, 1000),
            colorize_time(query_time, "%.3f", 0.1, 3),
            colorize_time(remaining_time, "%.3f", 1, 5)
            )

class DBFormatter(logging.Formatter):
    def format(self, record):
        record.pid = os.getpid()
        record.dbname = getattr(threading.current_thread(), 'dbname', '?')
        return logging.Formatter.format(self, record)

class ColoredFormatter(DBFormatter):
    def format(self, record):
        fg_color, bg_color = LEVEL_COLOR_MAPPING.get(record.levelno, (GREEN, DEFAULT))
        record.levelname = COLOR_PATTERN % (30 + fg_color, 40 + bg_color, record.levelname)
        return DBFormatter.format(self, record)

_logger_init = False
def init_logger():
    global _logger_init
    if _logger_init:
        return
    _logger_init = True

    old_factory = logging.getLogRecordFactory()
    def record_factory(*args, **kwargs):
        record = old_factory(*args, **kwargs)
        record.perf_info = ""
        return record
    logging.setLogRecordFactory(record_factory)

    logging.addLevelName(25, "INFO")
    logging.captureWarnings(True)

    from .tools.translate import resetlocale
    resetlocale()

    # create a format for log messages and dates
    format = '%(asctime)s %(pid)s %(levelname)s %(dbname)s %(name)s: %(message)s %(perf_info)s'
    # Normal Handler on stderr
    handler = logging.StreamHandler()

    if tools.config['syslog']:
        # SysLog Handler
        if os.name == 'nt':
            handler = logging.handlers.NTEventLogHandler("%s %s" % (release.description, release.version))
        elif platform.system() == 'Darwin':
            handler = logging.handlers.SysLogHandler('/var/run/log')
        else:
            handler = logging.handlers.SysLogHandler('/dev/log')
        format = '%s %s' % (release.description, release.version) \
                + ':%(dbname)s:%(levelname)s:%(name)s:%(message)s'

    elif tools.config['logfile']:
        # LogFile Handler
        logf = tools.config['logfile']
        try:
            # We check we have the right location for the log files
            dirname = os.path.dirname(logf)
            if dirname and not os.path.isdir(dirname):
                os.makedirs(dirname)
            if os.name == 'posix':
                handler = logging.handlers.WatchedFileHandler(logf)
            else:
                handler = logging.FileHandler(logf)
        except Exception:
            sys.stderr.write("ERROR: couldn't create the logfile directory. Logging to the standard output.\n")

    # Check that handler.stream has a fileno() method: when running OpenERP
    # behind Apache with mod_wsgi, handler.stream will have type mod_wsgi.Log,
    # which has no fileno() method. (mod_wsgi.Log is what is being bound to
    # sys.stderr when the logging.StreamHandler is being constructed above.)
    def is_a_tty(stream):
        return hasattr(stream, 'fileno') and os.isatty(stream.fileno())

    if os.name == 'posix' and isinstance(handler, logging.StreamHandler) and is_a_tty(handler.stream):
        formatter = ColoredFormatter(format)
        perf_filter = ColoredPerfFilter()
    else:
        formatter = DBFormatter(format)
        perf_filter = PerfFilter()
    handler.setFormatter(formatter)
    logging.getLogger().addHandler(handler)
    logging.getLogger('werkzeug').addFilter(perf_filter)

    if tools.config['log_db']:
        db_levels = {
            'debug': logging.DEBUG,
            'info': logging.INFO,
            'warning': logging.WARNING,
            'error': logging.ERROR,
            'critical': logging.CRITICAL,
        }
        postgresqlHandler = PostgreSQLHandler()
        postgresqlHandler.setLevel(int(db_levels.get(tools.config['log_db_level'], tools.config['log_db_level'])))
        logging.getLogger().addHandler(postgresqlHandler)

    # Configure loggers levels
    pseudo_config = PSEUDOCONFIG_MAPPER.get(tools.config['log_level'], [])

    logconfig = tools.config['log_handler']

    logging_configurations = DEFAULT_LOG_CONFIGURATION + pseudo_config + logconfig
    for logconfig_item in logging_configurations:
        loggername, level = logconfig_item.split(':')
        level = getattr(logging, level, logging.INFO)
        logger = logging.getLogger(loggername)
        logger.setLevel(level)

    for logconfig_item in logging_configurations:
        _logger.debug('logger level set: "%s"', logconfig_item)

DEFAULT_LOG_CONFIGURATION = [
    'odoo.http.rpc.request:INFO',
    'odoo.http.rpc.response:INFO',
    ':INFO',
]
PSEUDOCONFIG_MAPPER = {
    'debug_rpc_answer': ['odoo:DEBUG', 'odoo.sql_db:INFO', 'odoo.http.rpc:DEBUG'],
    'debug_rpc': ['odoo:DEBUG', 'odoo.sql_db:INFO', 'odoo.http.rpc.request:DEBUG'],
    'debug': ['odoo:DEBUG', 'odoo.sql_db:INFO'],
    'debug_sql': ['odoo.sql_db:DEBUG'],
    'info': [],
    'warn': ['odoo:WARNING', 'werkzeug:WARNING'],
    'error': ['odoo:ERROR', 'werkzeug:ERROR'],
    'critical': ['odoo:CRITICAL', 'werkzeug:CRITICAL'],
}
<|endoftext|>


<commit_before>"""Module for rover states."""

__author__ = 'Salman Hashmi'
__license__ = 'BSD License'

import numpy as np

class FindWall():
    """Create a class to represent FindWall state."""

    def __init__(self):
        """Initialize a FindWall instance."""
        self.name = 'Find Wall'

    def execute(self, Rover):
        """Execute the FindWall state action."""
        Rover.throttle = 0
        Rover.brake = 0
        Rover.steer = 15

class FollowWall():
    """Create a class to represent FollowWall state."""

    def __init__(self):
        """Initialize a FollowWall instance."""
        self.throttle_setting = 0.8
        self.wall_angle_offset = 9.2
        self.name = 'Follow Wall'

    def execute(self, Rover):
        """Execute the FollowWall state action."""
        if Rover.vel < Rover.max_vel:
            Rover.throttle = self.throttle_setting
        else:
            Rover.throttle = 0

        Rover.brake = 0
        Rover.steer = np.clip(
            np.mean(Rover.nav_angles_left * 180 / np.pi),
            -15, 15) - self.wall_angle_offset

class TurnToWall():
    """Create a class to represent TurnToWall state."""

    def __init__(self):
        """Initialize a TurnToWall instance."""
        self.brake_setting = 10
        self.name = 'Turn To Wall'

    def execute(self, Rover):
        """Execute the TurnToWall state action."""
        # Stop before turning
        if Rover.vel > 0.2:
            Rover.throttle = 0
            Rover.brake = self.brake_setting
            Rover.steer = 0

        elif Rover.vel <= 0.2:
            Rover.throttle = 0
            Rover.brake = 0
            Rover.steer = 15  # turn left towards wall

class AvoidWall():
    """Create a class to represent AvoidWall state."""

    def __init__(self):
        """Initialize a AvoidWall instance."""
        self.brake_setting = 10
        self.name = 'Avoid Wall'

    def execute(self, Rover):
        """Execute the AvoidWall state action."""
        # Stop before turning away from wall
        if Rover.vel > 0.2:
            Rover.throttle = 0
            Rover.brake = self.brake_setting
            Rover.steer = 0

        elif Rover.vel <= 0.2:
            Rover.throttle = 0
            Rover.brake = 0
            Rover.steer = -15  # turn right away from wall

class AvoidObstacles():
    """Create a class to represent AvoidObstacles state."""

    def __init__(self):
        """Initialize a AvoidObstacles instance."""
        self.brake_setting = 10
        self.name = 'Avoid Obstacles'

    def execute(self, Rover):
        """Execute the AvoidObstacles state action."""
        # Stop before avoiding obstacles
        if Rover.vel > 0.2:
            Rover.throttle = 0
            Rover.brake = self.brake_setting
            Rover.steer = 0

        elif Rover.vel <= 0.2:
            Rover.throttle = 0
            Rover.brake = 0
            # Turn left or right depending on availability
            # of nav terrain
            if np.mean(Rover.nav_angles) < -0.3:
                Rover.steer = -15
            elif np.mean(Rover.nav_angles) > 0.3:
                Rover.steer = 15
            else:  # e.g. if nav_angles are NaN
                Rover.steer = 0
                Rover.throttle = -1.0

class GoToSample():
    """Create a class to represent GoToSample state."""

    def __init__(self):
        """Initialize a GoToSample instance."""
        self.name = 'Go to Sample'

    def execute(self, Rover):
        """Execute the GoToSample state action."""
        pass

class InitiatePickup():
    """Create a class to represent InitiatePickup state."""

    def __init__(self):
        """Initialize a InitiatePickup instance."""
        self.name = 'Initiate Pickup'

    def execute(self, Rover):
        """Execute the InitiatePickup state action."""
        pass

class WaitForPickupInitiate():
    """Create a class to represent WaitForPickupInitiate state."""

    def __init__(self):
        """Initialize a WaitForPickupInitiate instance."""
        self.name = 'Wait..'

    def execute(self, Rover):
        """Execute the WaitForPickupInitiate state action."""
        pass

class WaitForPickupFinish():
    """Create a class to represent WaitForPickupFinish state."""

    def __init__(self):
        """Initialize a WaitForPickupFinish instance."""
        self.name = 'Pickup Sample.'

    def execute(self, Rover):
        """Execute the WaitForPickupFinish state action."""
        pass

class ReturnHome():
    """Create a class to represent ReturnHome state."""

    def __init__(self):
        """Initialize a ReturnHome instance."""
        self.name = 'Return Home'

    def execute(self, Rover):
        """Execute the ReturnHome state action."""
        pass

class Stop():
    """Create a class to represent Stop state."""

    def __init__(self):
        """Initialize a Stop instance."""
        self.name = 'Stop'

    def execute(self, Rover):
        """Execute the Stop state action."""
        pass

class FullStop():
    """Create a class to represent FullStop state."""

    def __init__(self):
        """Initialize a FullStop instance."""
        self.name = 'Full Stop'

    def execute(self, Rover):
        """Execute the FullStop state action."""
        pass
<commit_msg>implement GoToSample state action<commit_after>"""Module for rover states."""

__author__ = 'Salman Hashmi'
__license__ = 'BSD License'

import numpy as np

class FindWall():
    """Create a class to represent FindWall state."""

    def __init__(self):
        """Initialize a FindWall instance."""
        self.name = 'Find Wall'

    def execute(self, Rover):
        """Execute the FindWall state action."""
        Rover.throttle = 0
        Rover.brake = 0
        Rover.steer = 15

class FollowWall():
    """Create a class to represent FollowWall state."""

    def __init__(self):
        """Initialize a FollowWall instance."""
        self.throttle_setting = 0.8
        self.wall_angle_offset = 9.2
        self.name = 'Follow Wall'

    def execute(self, Rover):
        """Execute the FollowWall state action."""
        if Rover.vel < Rover.max_vel:
            Rover.throttle = self.throttle_setting
        else:
            Rover.throttle = 0

        Rover.brake = 0
        Rover.steer = np.clip(
            np.mean(Rover.nav_angles_left * 180 / np.pi),
            -15, 15) - self.wall_angle_offset

class TurnToWall():
    """Create a class to represent TurnToWall state."""

    def __init__(self):
        """Initialize a TurnToWall instance."""
        self.brake_setting = 10
        self.name = 'Turn To Wall'

    def execute(self, Rover):
        """Execute the TurnToWall state action."""
        # Stop before turning
        if Rover.vel > 0.2:
            Rover.throttle = 0
            Rover.brake = self.brake_setting
            Rover.steer = 0

        elif Rover.vel <= 0.2:
            Rover.throttle = 0
            Rover.brake = 0
            Rover.steer = 15  # turn left towards wall

class AvoidWall():
    """Create a class to represent AvoidWall state."""

    def __init__(self):
        """Initialize a AvoidWall instance."""
        self.brake_setting = 10
        self.name = 'Avoid Wall'

    def execute(self, Rover):
        """Execute the AvoidWall state action."""
        # Stop before turning away from wall
        if Rover.vel > 0.2:
            Rover.throttle = 0
            Rover.brake = self.brake_setting
            Rover.steer = 0

        elif Rover.vel <= 0.2:
            Rover.throttle = 0
            Rover.brake = 0
            Rover.steer = -15  # turn right away from wall

class AvoidObstacles():
    """Create a class to represent AvoidObstacles state."""

    def __init__(self):
        """Initialize a AvoidObstacles instance."""
        self.brake_setting = 10
        self.name = 'Avoid Obstacles'

    def execute(self, Rover):
        """Execute the AvoidObstacles state action."""
        # Stop before avoiding obstacles
        if Rover.vel > 0.2:
            Rover.throttle = 0
            Rover.brake = self.brake_setting
            Rover.steer = 0

        elif Rover.vel <= 0.2:
            Rover.throttle = 0
            Rover.brake = 0
            # Turn left or right depending on availability
            # of nav terrain
            if np.mean(Rover.nav_angles) < -0.3:
                Rover.steer = -15
            elif np.mean(Rover.nav_angles) > 0.3:
                Rover.steer = 15
            else:  # e.g. if nav_angles are NaN
                Rover.steer = 0
                Rover.throttle = -1.0

class GoToSample():
    """Create a class to represent GoToSample state."""

    def __init__(self):
        """Initialize a GoToSample instance."""
        self.throttle_setting = 0.39
        self.approach_vel = 1.0
        self.approach_angle_bias = -0.05
        self.drive_angle_offset = -3
        self.brake_setting = 10
        self.yaw_left_setting = 15
        self.yaw_right_setting = -15
        self.name = 'Go to Sample'

    def execute(self, Rover):
        """Execute the GoToSample state action."""
        mean_rock_angle = np.mean(Rover.rock_angles) + self.approach_angle_bias
        # stop before going to sample
        if Rover.vel > self.approach_vel:
            Rover.throttle = 0
            Rover.brake = self.brake_setting
            Rover.steer = 0

        elif(Rover.vel <= self.approach_vel):
            # yaw left if sample to left more than 0.4 rads
            if mean_rock_angle >= 0.4:
                Rover.throttle = 0
                Rover.brake = 0
                Rover.steer = self.yaw_left_setting
            # yaw right if sample to right more than -0.4 rads
            elif mean_rock_angle <= -0.4:
                Rover.throttle = 0
                Rover.brake = 0
                Rover.steer = self.yaw_right_setting
            # otherwise drive in the direction of mean_rock_angle
            elif ((mean_rock_angle < 0.4 and mean_rock_angle > -0.4) or
                  math.isnan(mean_rock_angle)):
                Rover.brake = 0
                Rover.throttle = self.throttle_setting

                mean_rock_angle_deg = np.mean(
                    Rover.rock_angles * 180 / np.pi) + self.drive_angle_offset

                Rover.steer = np.clip(mean_rock_angle_deg,
                                      self.yaw_right_setting,
                                      self.yaw_left_setting)

                Rover.curr_rock_angle = mean_rock_angle_deg

class InitiatePickup():
    """Create a class to represent InitiatePickup state."""

    def __init__(self):
        """Initialize a InitiatePickup instance."""
        self.name = 'Initiate Pickup'

    def execute(self, Rover):
        """Execute the InitiatePickup state action."""
        pass

class WaitForPickupInitiate():
    """Create a class to represent WaitForPickupInitiate state."""

    def __init__(self):
        """Initialize a WaitForPickupInitiate instance."""
        self.name = 'Wait..'

    def execute(self, Rover):
        """Execute the WaitForPickupInitiate state action."""
        pass

class WaitForPickupFinish():
    """Create a class to represent WaitForPickupFinish state."""

    def __init__(self):
        """Initialize a WaitForPickupFinish instance."""
        self.name = 'Pickup Sample.'

    def execute(self, Rover):
        """Execute the WaitForPickupFinish state action."""
        pass

class ReturnHome():
    """Create a class to represent ReturnHome state."""

    def __init__(self):
        """Initialize a ReturnHome instance."""
        self.name = 'Return Home'

    def execute(self, Rover):
        """Execute the ReturnHome state action."""
        pass

class Stop():
    """Create a class to represent Stop state."""

    def __init__(self):
        """Initialize a Stop instance."""
        self.name = 'Stop'

    def execute(self, Rover):
        """Execute the Stop state action."""
        pass

class FullStop():
    """Create a class to represent FullStop state."""

    def __init__(self):
        """Initialize a FullStop instance."""
        self.name = 'Full Stop'

    def execute(self, Rover):
        """Execute the FullStop state action."""
        pass
<|endoftext|>


<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import logging
import os
import time

from tracing.metrics import metric_runner

from telemetry.timeline import chrome_trace_category_filter
from telemetry.timeline import model as model_module
from telemetry.timeline import tracing_config
from telemetry.value import trace
from telemetry.value import common_value_helpers
from telemetry.web_perf.metrics import timeline_based_metric
from telemetry.web_perf import story_test
from telemetry.web_perf import timeline_interaction_record as tir_module

# TimelineBasedMeasurement considers all instrumentation as producing a single
# timeline. But, depending on the amount of instrumentation that is enabled,
# overhead increases. The user of the measurement must therefore chose between
# a few levels of instrumentation.
LOW_OVERHEAD_LEVEL = 'low-overhead'
DEFAULT_OVERHEAD_LEVEL = 'default-overhead'
DEBUG_OVERHEAD_LEVEL = 'debug-overhead'

ALL_OVERHEAD_LEVELS = [
    LOW_OVERHEAD_LEVEL,
    DEFAULT_OVERHEAD_LEVEL,
    DEBUG_OVERHEAD_LEVEL,
]

class InvalidInteractions(Exception):
  pass

# TODO(nednguyen): Get rid of this results wrapper hack after we add interaction
# record to telemetry value system (crbug.com/453109)
class ResultsWrapperInterface(object):
  def __init__(self):
    self._tir_label = None
    self._results = None

  def SetResults(self, results):
    self._results = results

  def SetTirLabel(self, tir_label):
    self._tir_label = tir_label

  @property
  def current_page(self):
    return self._results.current_page

  def AddValue(self, value):
    raise NotImplementedError

class _TBMResultWrapper(ResultsWrapperInterface):
  def AddValue(self, value):
    assert self._tir_label
    if value.tir_label:
      assert value.tir_label == self._tir_label
    else:
      value.tir_label = self._tir_label
    self._results.AddValue(value)

def _GetRendererThreadsToInteractionRecordsMap(model):
  threads_to_records_map = collections.defaultdict(list)
  interaction_labels_of_previous_threads = set()
  for curr_thread in model.GetAllThreads():
    for event in curr_thread.async_slices:
      # TODO(nduca): Add support for page-load interaction record.
      if tir_module.IsTimelineInteractionRecord(event.name):
        interaction = tir_module.TimelineInteractionRecord.FromAsyncEvent(event)
        threads_to_records_map[curr_thread].append(interaction)
        if interaction.label in interaction_labels_of_previous_threads:
          raise InvalidInteractions(
              'Interaction record label %s is duplicated on different '
              'threads' % interaction.label)
    if curr_thread in threads_to_records_map:
      interaction_labels_of_previous_threads.update(
          r.label for r in threads_to_records_map[curr_thread])

  return threads_to_records_map

class _TimelineBasedMetrics(object):
  def __init__(self, model, renderer_thread, interaction_records,
               results_wrapper, metrics):
    self._model = model
    self._renderer_thread = renderer_thread
    self._interaction_records = interaction_records
    self._results_wrapper = results_wrapper
    self._all_metrics = metrics

  def AddResults(self, results):
    interactions_by_label = collections.defaultdict(list)
    for i in self._interaction_records:
      interactions_by_label[i.label].append(i)

    for label, interactions in interactions_by_label.iteritems():
      are_repeatable = [i.repeatable for i in interactions]
      if not all(are_repeatable) and len(interactions) > 1:
        raise InvalidInteractions('Duplicate unrepeatable interaction records '
                                  'on the page')
      self._results_wrapper.SetResults(results)
      self._results_wrapper.SetTirLabel(label)
      self.UpdateResultsByMetric(interactions, self._results_wrapper)

  def UpdateResultsByMetric(self, interactions, wrapped_results):
    if not interactions:
      return

    for metric in self._all_metrics:
      metric.AddResults(self._model, self._renderer_thread,
                        interactions, wrapped_results)

class Options(object):
  """A class to be used to configure TimelineBasedMeasurement.

  This is created and returned by
  Benchmark.CreateCoreTimelineBasedMeasurementOptions.

  """

  def __init__(self, overhead_level=LOW_OVERHEAD_LEVEL):
    """As the amount of instrumentation increases, so does the overhead.
    The user of the measurement chooses the overhead level that is appropriate,
    and the tracing is filtered accordingly.

    overhead_level: Can either be a custom ChromeTraceCategoryFilter object or
        one of LOW_OVERHEAD_LEVEL, DEFAULT_OVERHEAD_LEVEL or
        DEBUG_OVERHEAD_LEVEL.
    """
    self._config = tracing_config.TracingConfig()
    self._config.enable_chrome_trace = True
    self._config.enable_platform_display_trace = False

    if isinstance(overhead_level,
                  chrome_trace_category_filter.ChromeTraceCategoryFilter):
      self._config.chrome_trace_config.SetCategoryFilter(overhead_level)
    elif overhead_level in ALL_OVERHEAD_LEVELS:
      if overhead_level == LOW_OVERHEAD_LEVEL:
        self._config.chrome_trace_config.SetLowOverheadFilter()
      elif overhead_level == DEFAULT_OVERHEAD_LEVEL:
        self._config.chrome_trace_config.SetDefaultOverheadFilter()
      else:
        self._config.chrome_trace_config.SetDebugOverheadFilter()
    else:
      raise Exception("Overhead level must be a ChromeTraceCategoryFilter "
                      "object or valid overhead level string. Given overhead "
                      "level: %s" % overhead_level)

    self._timeline_based_metrics = None
    self._legacy_timeline_based_metrics = []

  def ExtendTraceCategoryFilter(self, filters):
    for category_filter in filters:
      self.AddTraceCategoryFilter(category_filter)

  def AddTraceCategoryFilter(self, category_filter):
    self._config.chrome_trace_config.category_filter.AddFilter(category_filter)

  @property
  def category_filter(self):
    return self._config.chrome_trace_config.category_filter

  @property
  def config(self):
    return self._config

  def ExtendTimelineBasedMetric(self, metrics):
    for metric in metrics:
      self.AddTimelineBasedMetric(metric)

  def AddTimelineBasedMetric(self, metric):
    assert isinstance(metric, basestring)
    if self._timeline_based_metrics is None:
      self._timeline_based_metrics = []
    self._timeline_based_metrics.append(metric)

  def SetTimelineBasedMetrics(self, metrics):
    """Sets the new-style (TBMv2) metrics to run.

    Metrics are assumed to live in //tracing/tracing/metrics, so the path you
    pass in should be relative to that. For example, to specify
    sample_metric.html, you should pass in ['sample_metric.html'].

    Args:
      metrics: A list of strings giving metric paths under
          //tracing/tracing/metrics.
    """
    assert isinstance(metrics, list)
    for metric in metrics:
      assert isinstance(metric, basestring)
    self._timeline_based_metrics = metrics

  def GetTimelineBasedMetrics(self):
    return self._timeline_based_metrics

  def SetLegacyTimelineBasedMetrics(self, metrics):
    assert isinstance(metrics, collections.Iterable)
    for m in metrics:
      assert isinstance(m, timeline_based_metric.TimelineBasedMetric)
    self._legacy_timeline_based_metrics = metrics

  def GetLegacyTimelineBasedMetrics(self):
    return self._legacy_timeline_based_metrics

class TimelineBasedMeasurement(story_test.StoryTest):
  """Collects multiple metrics based on their interaction records.

  A timeline based measurement shifts the burden of what metrics to collect onto
  the story under test. Instead of the measurement
  having a fixed set of values it collects, the story being tested
  issues (via javascript) an Interaction record into the user timing API that
  describing what is happening at that time, as well as a standardized set
  of flags describing the semantics of the work being done. The
  TimelineBasedMeasurement object collects a trace that includes both these
  interaction records, and a user-chosen amount of performance data using
  Telemetry's various timeline-producing APIs, tracing especially.

  It then passes the recorded timeline to different TimelineBasedMetrics based
  on those flags. As an example, this allows a single story run to produce
  load timing data, smoothness data, critical jank information and overall cpu
  usage information.

  For information on how to mark up a page to work with
  TimelineBasedMeasurement, refer to the
  perf.metrics.timeline_interaction_record module.

  Args:
      options: an instance of timeline_based_measurement.Options.
      results_wrapper: A class that has the __init__ method takes in
        the page_test_results object and the interaction record label. This
        class follows the ResultsWrapperInterface. Note: this class is not
        supported long term and to be removed when crbug.com/453109 is resolved.
  """
  def __init__(self, options, results_wrapper=None):
    self._tbm_options = options
    self._results_wrapper = results_wrapper or _TBMResultWrapper()

  def WillRunStory(self, platform):
    """Configure and start tracing."""
    if not platform.tracing_controller.IsChromeTracingSupported():
      raise Exception('Not supported')
    if self._tbm_options.config.enable_chrome_trace:
      # Always enable 'blink.console' category for:
      # 1) Backward compat of chrome clock sync (crbug.com/646925)
      # 2) Allows users to add trace event through javascript.
      # Note that blink.console is extremely low-overhead, so this doesn't
      # affect the tracing overhead budget much.
      chrome_config = self._tbm_options.config.chrome_trace_config
      chrome_config.category_filter.AddIncludedCategory('blink.console')
    platform.tracing_controller.StartTracing(self._tbm_options.config)

  def Measure(self, platform, results):
    """Collect all possible metrics and added them to results."""
    platform.tracing_controller.telemetry_info = results.telemetry_info
    trace_result, _ = platform.tracing_controller.StopTracing()
    trace_value = trace.TraceValue(
        results.current_page, trace_result,
        file_path=results.telemetry_info.trace_local_path,
        remote_path=results.telemetry_info.trace_remote_path,
        upload_bucket=results.telemetry_info.upload_bucket,
        cloud_url=results.telemetry_info.trace_remote_url)
    results.AddValue(trace_value)

    try:
      if self._tbm_options.GetTimelineBasedMetrics():
        assert not self._tbm_options.GetLegacyTimelineBasedMetrics(), (
            'Specifying both TBMv1 and TBMv2 metrics is not allowed.')
        self._ComputeTimelineBasedMetrics(results, trace_value)
      else:
        # Run all TBMv1 metrics if no other metric is specified
        # (legacy behavior)
        if not self._tbm_options.GetLegacyTimelineBasedMetrics():
          raise Exception(
              'Please specify the TBMv1 metrics you are interested in '
              'explicitly.')
        self._ComputeLegacyTimelineBasedMetrics(results, trace_result)
    finally:
      trace_result.CleanUpAllTraces()

  def DidRunStory(self, platform, results):
    """Clean up after running the story."""
    if platform.tracing_controller.is_tracing_running:
      trace_result, _ = platform.tracing_controller.StopTracing()
      trace_value = trace.TraceValue(
          results.current_page, trace_result,
          file_path=results.telemetry_info.trace_local_path,
          remote_path=results.telemetry_info.trace_remote_path,
          upload_bucket=results.telemetry_info.upload_bucket,
          cloud_url=results.telemetry_info.trace_remote_url)
      results.AddValue(trace_value)

  def _ComputeTimelineBasedMetrics(self, results, trace_value):
    metrics = self._tbm_options.GetTimelineBasedMetrics()
    extra_import_options = {
        'trackDetailedModelStats': True
    }
    trace_size_in_mib = os.path.getsize(trace_value.filename) / (2 ** 20)
    # Bails out on trace that are too big. See crbug.com/812631 for more
    # details.
    if trace_size_in_mib > 400:
      results.Fail('Trace size is too big: %s MiB' % trace_size_in_mib)
      return

    logging.info('Starting to compute metrics on trace')
    start = time.time()
    mre_result = metric_runner.RunMetric(
        trace_value.filename, metrics, extra_import_options,
        report_progress=False, canonical_url=results.telemetry_info.trace_url)
    logging.info('Processing resulting traces took %.3f seconds' % (
        time.time() - start))
    page = results.current_page

    for f in mre_result.failures:
      results.Fail(f.stack)

    histogram_dicts = mre_result.pairs.get('histograms', [])
    results.ImportHistogramDicts(histogram_dicts)

    for d in mre_result.pairs.get('scalars', []):
      results.AddValue(common_value_helpers.TranslateScalarValue(d, page))

  def _ComputeLegacyTimelineBasedMetrics(self, results, trace_result):
    model = model_module.TimelineModel(trace_result)
    threads_to_records_map = _GetRendererThreadsToInteractionRecordsMap(model)
    if (len(threads_to_records_map.values()) == 0 and
        self._tbm_options.config.enable_chrome_trace):
      logging.warning(
          'No timeline interaction records were recorded in the trace. '
          'This could be caused by console.time() & console.timeEnd() execution'
          ' failure or the tracing category specified doesn\'t include '
          'blink.console categories.')

    all_metrics = self._tbm_options.GetLegacyTimelineBasedMetrics()

    for renderer_thread, interaction_records in (
        threads_to_records_map.iteritems()):
      meta_metrics = _TimelineBasedMetrics(
          model, renderer_thread, interaction_records, self._results_wrapper,
          all_metrics)
      meta_metrics.AddResults(results)

    for metric in all_metrics:
      metric.AddWholeTraceResults(model, results)

  @property
  def tbm_options(self):
    return self._tbm_options
<commit_msg>Add v8.console to the default categories.<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import logging
import os
import time

from tracing.metrics import metric_runner

from telemetry.timeline import chrome_trace_category_filter
from telemetry.timeline import model as model_module
from telemetry.timeline import tracing_config
from telemetry.value import trace
from telemetry.value import common_value_helpers
from telemetry.web_perf.metrics import timeline_based_metric
from telemetry.web_perf import story_test
from telemetry.web_perf import timeline_interaction_record as tir_module

# TimelineBasedMeasurement considers all instrumentation as producing a single
# timeline. But, depending on the amount of instrumentation that is enabled,
# overhead increases. The user of the measurement must therefore chose between
# a few levels of instrumentation.
LOW_OVERHEAD_LEVEL = 'low-overhead'
DEFAULT_OVERHEAD_LEVEL = 'default-overhead'
DEBUG_OVERHEAD_LEVEL = 'debug-overhead'

ALL_OVERHEAD_LEVELS = [
    LOW_OVERHEAD_LEVEL,
    DEFAULT_OVERHEAD_LEVEL,
    DEBUG_OVERHEAD_LEVEL,
]

class InvalidInteractions(Exception):
  pass

# TODO(nednguyen): Get rid of this results wrapper hack after we add interaction
# record to telemetry value system (crbug.com/453109)
class ResultsWrapperInterface(object):
  def __init__(self):
    self._tir_label = None
    self._results = None

  def SetResults(self, results):
    self._results = results

  def SetTirLabel(self, tir_label):
    self._tir_label = tir_label

  @property
  def current_page(self):
    return self._results.current_page

  def AddValue(self, value):
    raise NotImplementedError

class _TBMResultWrapper(ResultsWrapperInterface):
  def AddValue(self, value):
    assert self._tir_label
    if value.tir_label:
      assert value.tir_label == self._tir_label
    else:
      value.tir_label = self._tir_label
    self._results.AddValue(value)

def _GetRendererThreadsToInteractionRecordsMap(model):
  threads_to_records_map = collections.defaultdict(list)
  interaction_labels_of_previous_threads = set()
  for curr_thread in model.GetAllThreads():
    for event in curr_thread.async_slices:
      # TODO(nduca): Add support for page-load interaction record.
      if tir_module.IsTimelineInteractionRecord(event.name):
        interaction = tir_module.TimelineInteractionRecord.FromAsyncEvent(event)
        threads_to_records_map[curr_thread].append(interaction)
        if interaction.label in interaction_labels_of_previous_threads:
          raise InvalidInteractions(
              'Interaction record label %s is duplicated on different '
              'threads' % interaction.label)
    if curr_thread in threads_to_records_map:
      interaction_labels_of_previous_threads.update(
          r.label for r in threads_to_records_map[curr_thread])

  return threads_to_records_map

class _TimelineBasedMetrics(object):
  def __init__(self, model, renderer_thread, interaction_records,
               results_wrapper, metrics):
    self._model = model
    self._renderer_thread = renderer_thread
    self._interaction_records = interaction_records
    self._results_wrapper = results_wrapper
    self._all_metrics = metrics

  def AddResults(self, results):
    interactions_by_label = collections.defaultdict(list)
    for i in self._interaction_records:
      interactions_by_label[i.label].append(i)

    for label, interactions in interactions_by_label.iteritems():
      are_repeatable = [i.repeatable for i in interactions]
      if not all(are_repeatable) and len(interactions) > 1:
        raise InvalidInteractions('Duplicate unrepeatable interaction records '
                                  'on the page')
      self._results_wrapper.SetResults(results)
      self._results_wrapper.SetTirLabel(label)
      self.UpdateResultsByMetric(interactions, self._results_wrapper)

  def UpdateResultsByMetric(self, interactions, wrapped_results):
    if not interactions:
      return

    for metric in self._all_metrics:
      metric.AddResults(self._model, self._renderer_thread,
                        interactions, wrapped_results)

class Options(object):
  """A class to be used to configure TimelineBasedMeasurement.

  This is created and returned by
  Benchmark.CreateCoreTimelineBasedMeasurementOptions.

  """

  def __init__(self, overhead_level=LOW_OVERHEAD_LEVEL):
    """As the amount of instrumentation increases, so does the overhead.
    The user of the measurement chooses the overhead level that is appropriate,
    and the tracing is filtered accordingly.

    overhead_level: Can either be a custom ChromeTraceCategoryFilter object or
        one of LOW_OVERHEAD_LEVEL, DEFAULT_OVERHEAD_LEVEL or
        DEBUG_OVERHEAD_LEVEL.
    """
    self._config = tracing_config.TracingConfig()
    self._config.enable_chrome_trace = True
    self._config.enable_platform_display_trace = False

    if isinstance(overhead_level,
                  chrome_trace_category_filter.ChromeTraceCategoryFilter):
      self._config.chrome_trace_config.SetCategoryFilter(overhead_level)
    elif overhead_level in ALL_OVERHEAD_LEVELS:
      if overhead_level == LOW_OVERHEAD_LEVEL:
        self._config.chrome_trace_config.SetLowOverheadFilter()
      elif overhead_level == DEFAULT_OVERHEAD_LEVEL:
        self._config.chrome_trace_config.SetDefaultOverheadFilter()
      else:
        self._config.chrome_trace_config.SetDebugOverheadFilter()
    else:
      raise Exception("Overhead level must be a ChromeTraceCategoryFilter "
                      "object or valid overhead level string. Given overhead "
                      "level: %s" % overhead_level)

    self._timeline_based_metrics = None
    self._legacy_timeline_based_metrics = []

  def ExtendTraceCategoryFilter(self, filters):
    for category_filter in filters:
      self.AddTraceCategoryFilter(category_filter)

  def AddTraceCategoryFilter(self, category_filter):
    self._config.chrome_trace_config.category_filter.AddFilter(category_filter)

  @property
  def category_filter(self):
    return self._config.chrome_trace_config.category_filter

  @property
  def config(self):
    return self._config

  def ExtendTimelineBasedMetric(self, metrics):
    for metric in metrics:
      self.AddTimelineBasedMetric(metric)

  def AddTimelineBasedMetric(self, metric):
    assert isinstance(metric, basestring)
    if self._timeline_based_metrics is None:
      self._timeline_based_metrics = []
    self._timeline_based_metrics.append(metric)

  def SetTimelineBasedMetrics(self, metrics):
    """Sets the new-style (TBMv2) metrics to run.

    Metrics are assumed to live in //tracing/tracing/metrics, so the path you
    pass in should be relative to that. For example, to specify
    sample_metric.html, you should pass in ['sample_metric.html'].

    Args:
      metrics: A list of strings giving metric paths under
          //tracing/tracing/metrics.
    """
    assert isinstance(metrics, list)
    for metric in metrics:
      assert isinstance(metric, basestring)
    self._timeline_based_metrics = metrics

  def GetTimelineBasedMetrics(self):
    return self._timeline_based_metrics

  def SetLegacyTimelineBasedMetrics(self, metrics):
    assert isinstance(metrics, collections.Iterable)
    for m in metrics:
      assert isinstance(m, timeline_based_metric.TimelineBasedMetric)
    self._legacy_timeline_based_metrics = metrics

  def GetLegacyTimelineBasedMetrics(self):
    return self._legacy_timeline_based_metrics

class TimelineBasedMeasurement(story_test.StoryTest):
  """Collects multiple metrics based on their interaction records.

  A timeline based measurement shifts the burden of what metrics to collect onto
  the story under test. Instead of the measurement
  having a fixed set of values it collects, the story being tested
  issues (via javascript) an Interaction record into the user timing API that
  describing what is happening at that time, as well as a standardized set
  of flags describing the semantics of the work being done. The
  TimelineBasedMeasurement object collects a trace that includes both these
  interaction records, and a user-chosen amount of performance data using
  Telemetry's various timeline-producing APIs, tracing especially.

  It then passes the recorded timeline to different TimelineBasedMetrics based
  on those flags. As an example, this allows a single story run to produce
  load timing data, smoothness data, critical jank information and overall cpu
  usage information.

  For information on how to mark up a page to work with
  TimelineBasedMeasurement, refer to the
  perf.metrics.timeline_interaction_record module.

  Args:
      options: an instance of timeline_based_measurement.Options.
      results_wrapper: A class that has the __init__ method takes in
        the page_test_results object and the interaction record label. This
        class follows the ResultsWrapperInterface. Note: this class is not
        supported long term and to be removed when crbug.com/453109 is resolved.
  """
  def __init__(self, options, results_wrapper=None):
    self._tbm_options = options
    self._results_wrapper = results_wrapper or _TBMResultWrapper()

  def WillRunStory(self, platform):
    """Configure and start tracing."""
    if not platform.tracing_controller.IsChromeTracingSupported():
      raise Exception('Not supported')
    if self._tbm_options.config.enable_chrome_trace:
      # Always enable 'blink.console' and 'v8.console' categories for:
      # 1) Backward compat of chrome clock sync (https://crbug.com/646925).
      # 2) Allows users to add trace event through javascript.
      # 3) For the console error metric (https://crbug.com/880432).
      # Note that these categories are extremely low-overhead, so this doesn't
      # affect the tracing overhead budget much.
      chrome_config = self._tbm_options.config.chrome_trace_config
      chrome_config.category_filter.AddIncludedCategory('blink.console')
      chrome_config.category_filter.AddIncludedCategory('v8.console')
    platform.tracing_controller.StartTracing(self._tbm_options.config)

  def Measure(self, platform, results):
    """Collect all possible metrics and added them to results."""
    platform.tracing_controller.telemetry_info = results.telemetry_info
    trace_result, _ = platform.tracing_controller.StopTracing()
    trace_value = trace.TraceValue(
        results.current_page, trace_result,
        file_path=results.telemetry_info.trace_local_path,
        remote_path=results.telemetry_info.trace_remote_path,
        upload_bucket=results.telemetry_info.upload_bucket,
        cloud_url=results.telemetry_info.trace_remote_url)
    results.AddValue(trace_value)

    try:
      if self._tbm_options.GetTimelineBasedMetrics():
        assert not self._tbm_options.GetLegacyTimelineBasedMetrics(), (
            'Specifying both TBMv1 and TBMv2 metrics is not allowed.')
        self._ComputeTimelineBasedMetrics(results, trace_value)
      else:
        # Run all TBMv1 metrics if no other metric is specified
        # (legacy behavior)
        if not self._tbm_options.GetLegacyTimelineBasedMetrics():
          raise Exception(
              'Please specify the TBMv1 metrics you are interested in '
              'explicitly.')
        self._ComputeLegacyTimelineBasedMetrics(results, trace_result)
    finally:
      trace_result.CleanUpAllTraces()

  def DidRunStory(self, platform, results):
    """Clean up after running the story."""
    if platform.tracing_controller.is_tracing_running:
      trace_result, _ = platform.tracing_controller.StopTracing()
      trace_value = trace.TraceValue(
          results.current_page, trace_result,
          file_path=results.telemetry_info.trace_local_path,
          remote_path=results.telemetry_info.trace_remote_path,
          upload_bucket=results.telemetry_info.upload_bucket,
          cloud_url=results.telemetry_info.trace_remote_url)
      results.AddValue(trace_value)

  def _ComputeTimelineBasedMetrics(self, results, trace_value):
    metrics = self._tbm_options.GetTimelineBasedMetrics()
    extra_import_options = {
        'trackDetailedModelStats': True
    }
    trace_size_in_mib = os.path.getsize(trace_value.filename) / (2 ** 20)
    # Bails out on trace that are too big. See crbug.com/812631 for more
    # details.
    if trace_size_in_mib > 400:
      results.Fail('Trace size is too big: %s MiB' % trace_size_in_mib)
      return

    logging.info('Starting to compute metrics on trace')
    start = time.time()
    mre_result = metric_runner.RunMetric(
        trace_value.filename, metrics, extra_import_options,
        report_progress=False, canonical_url=results.telemetry_info.trace_url)
    logging.info('Processing resulting traces took %.3f seconds' % (
        time.time() - start))
    page = results.current_page

    for f in mre_result.failures:
      results.Fail(f.stack)

    histogram_dicts = mre_result.pairs.get('histograms', [])
    results.ImportHistogramDicts(histogram_dicts)

    for d in mre_result.pairs.get('scalars', []):
      results.AddValue(common_value_helpers.TranslateScalarValue(d, page))

  def _ComputeLegacyTimelineBasedMetrics(self, results, trace_result):
    model = model_module.TimelineModel(trace_result)
    threads_to_records_map = _GetRendererThreadsToInteractionRecordsMap(model)
    if (len(threads_to_records_map.values()) == 0 and
        self._tbm_options.config.enable_chrome_trace):
      logging.warning(
          'No timeline interaction records were recorded in the trace. '
          'This could be caused by console.time() & console.timeEnd() execution'
          ' failure or the tracing category specified doesn\'t include '
          'blink.console categories.')

    all_metrics = self._tbm_options.GetLegacyTimelineBasedMetrics()

    for renderer_thread, interaction_records in (
        threads_to_records_map.iteritems()):
      meta_metrics = _TimelineBasedMetrics(
          model, renderer_thread, interaction_records, self._results_wrapper,
          all_metrics)
      meta_metrics.AddResults(results)

    for metric in all_metrics:
      metric.AddWholeTraceResults(model, results)

  @property
  def tbm_options(self):
    return self._tbm_options
<|endoftext|>


<commit_before>import tensorflow.compat.v2 as tf

from tensorflow_probability.python import bijectors as tfb
from tensorflow_probability.python import util
from tensorflow_probability.python.experimental.bijectors import scalar_function_with_inferred_inverse
from tensorflow_probability.python.internal import cache_util
from tensorflow_probability.python.internal import samplers

def build_highway_flow_layer(width, residual_fraction_initial_value=0.5, activation_fn=None, seed=None):
    # TODO: add control that residual_fraction_initial_value is between 0 and 1
    residual_fraction_initial_value = tf.convert_to_tensor(residual_fraction_initial_value,
                                                           dtype_hint=tf.float32,
                                                           name='residual_fraction_initial_value')
    dtype = residual_fraction_initial_value.dtype

    bias_seed, upper_seed, lower_seed, diagonal_seed = samplers.split_seed(seed, n=4)

    return HighwayFlow(
        width=width,
        residual_fraction=util.TransformedVariable(
            initial_value=tf.convert_to_tensor(residual_fraction_initial_value),
            bijector=tfb.Sigmoid(),
            dtype=dtype),
        activation_fn=activation_fn,
        bias=tf.Variable(samplers.normal((width,), 0., 0.01, seed=bias_seed), dtype=dtype),
        upper_diagonal_weights_matrix=util.TransformedVariable(
            initial_value=tf.experimental.numpy.tril(samplers.normal((width, width), 0., 1., seed=upper_seed),
                                                     -1) + tf.linalg.diag(
                samplers.uniform((width,), minval=0., maxval=1., seed=diagonal_seed)),
            bijector=tfb.FillScaleTriL(diag_bijector=tfb.Softplus(), diag_shift=None),
            dtype=dtype),
        lower_diagonal_weights_matrix=util.TransformedVariable(
            initial_value=samplers.normal((width, width), 0., 1., seed=lower_seed),
            bijector=tfb.Chain([tfb.TransformDiagonal(diag_bijector=tfb.Shift(1.)),
                                tfb.Pad(paddings=[(1, 0), (0, 1)]),
                                tfb.FillTriangular()]),
            dtype=dtype)
    )

class HighwayFlow(tfb.Bijector):
    # todo: update comments?
    # HighWay Flow simultaneously computes `forward` and `fldj` (and `inverse`/`ildj`),
    # so we override the bijector cache to update the LDJ entries of attrs on
    # forward/inverse inverse calls (instead of updating them only when the LDJ
    # methods themselves are called).
    _cache = cache_util.BijectorCacheWithGreedyAttrs(
        forward_name='_augmented_forward',
        inverse_name='_augmented_inverse')

    def __init__(self, width, residual_fraction, activation_fn, bias, upper_diagonal_weights_matrix,
                 lower_diagonal_weights_matrix, validate_args=False,
                 name='highway_flow'):
        super(HighwayFlow, self).__init__(
            validate_args=validate_args,
            forward_min_event_ndims=1,  # FIXME: should this also be an argument of HighwayFlow __init__?
            name=name)

        self.width = width

        self.bias = bias

        self.residual_fraction = residual_fraction

        # still lower triangular, transposed is done in matvec.
        # TODO: transpose directly here or in the definition of TransformedVariable?
        self.upper_diagonal_weights_matrix = upper_diagonal_weights_matrix

        self.lower_diagonal_weights_matrix = lower_diagonal_weights_matrix

        self.activation_fn = activation_fn

    def df(self, x):
        # derivative of activation
        return self.residual_fraction + (1. - self.residual_fraction) * tf.math.sigmoid(x) * (1. - tf.math.sigmoid(x))

    def _convex_update(self, weights_matrix):
        # convex update
        # same as in the paper, but probably easier to invert
        return self.residual_fraction * tf.eye(self.width) + (1. - self.residual_fraction) * weights_matrix

    def inv_f(self, y, N=20):
        # inverse with Newton iteration
        x = tf.ones(y.shape)
        for _ in range(N):
            x = x - (self.residual_fraction * x + (1. - self.residual_fraction) * tf.math.softplus(x) - y) / (
                self.df(x))
        return x

    def _augmented_forward(self, x):
        # upper mmatrix jacobian
        fldj = tf.zeros(x.shape[0]) + tf.reduce_sum(
            tf.math.log(self.residual_fraction + (1. - self.residual_fraction) * tf.linalg.diag_part(
                self.upper_diagonal_weights_matrix)))  # jacobian from upper matrix
        # jacobian from lower matrix is 0

        # todo: see how to optimize _convex_update
        x = tf.linalg.matvec(self._convex_update(self.lower_diagonal_weights_matrix), x)
        x = tf.linalg.matvec(tf.transpose(self._convex_update(self.upper_diagonal_weights_matrix)),
                             x) + self.bias  # in the implementation there was only one bias
        if self.activation_fn:
            # fldj += tf.reduce_sum(tf.math.log(self.df(x)))
            # x = self.residual_fraction * x + (1. - self.residual_fraction) * self.activation_fn(x)
            activation_layer = scalar_function_with_inferred_inverse.ScalarFunctionWithInferredInverse(
                lambda x: self.residual_fraction * x + (1. - self.residual_fraction) * self.activation_fn(x))
            fldj += activation_layer.forward_log_det_jacobian(x, self.forward_min_event_ndims)
            x = activation_layer.forward(x)
        return x, {'ildj': -fldj, 'fldj': fldj}

    def _augmented_inverse(self, y):
        ildj = tf.zeros(y.shape[0]) - tf.reduce_sum(
            tf.math.log(self.residual_fraction + (1. - self.residual_fraction) * tf.linalg.diag_part(
                self.upper_diagonal_weights_matrix)))
        if self.activation_fn:
            # y = self.inv_f(y)
            # ildj += tf.reduce_sum(tf.math.log(self.df(y)))
            residual_fraction = tf.identity(self.residual_fraction)
            # FIXME: this way of using inverse activation does not seem to work because residual_fraction is a variable
            activation_layer = scalar_function_with_inferred_inverse.ScalarFunctionWithInferredInverse(
                lambda x: residual_fraction * x + (1. - residual_fraction) * self.activation_fn(x))
            ildj += activation_layer.inverse_log_det_jacobian(y, self.forward_min_event_ndims)
            y = activation_layer.inverse(y)

        # this works with y having shape [BATCH x WIDTH], don't know how well it generalizes
        y = tf.linalg.triangular_solve(tf.transpose(self._convex_update(self.upper_diagonal_weights_matrix)),
                                       tf.linalg.matrix_transpose(y - self.bias), lower=False)
        y = tf.linalg.triangular_solve(self._convex_update(self.lower_diagonal_weights_matrix), y)
        return tf.linalg.matrix_transpose(y), {'ildj': ildj, 'fldj': -ildj}

    def _forward(self, x):
        y, _ = self._augmented_forward(x)
        return y

    def _inverse(self, y):
        x, _ = self._augmented_inverse(y)
        return x

    def _forward_log_det_jacobian(self, x):
        cached = self._cache.forward_attributes(x)
        # If LDJ isn't in the cache, call forward once.
        if 'fldj' not in cached:
            _, attrs = self._augmented_forward(x)
            cached.update(attrs)
        return cached['fldj']

    def _inverse_log_det_jacobian(self, y):
        cached = self._cache.inverse_attributes(y)
        # If LDJ isn't in the cache, call inverse once.
        if 'ildj' not in cached:
            _, attrs = self._augmented_inverse(y)
            cached.update(attrs)
        return cached['ildj']
<commit_msg>added gating to bias<commit_after>import tensorflow.compat.v2 as tf

from tensorflow_probability.python import bijectors as tfb
from tensorflow_probability.python import util
from tensorflow_probability.python.experimental.bijectors import scalar_function_with_inferred_inverse
from tensorflow_probability.python.internal import cache_util
from tensorflow_probability.python.internal import samplers

def build_highway_flow_layer(width, residual_fraction_initial_value=0.5, activation_fn=None, seed=None):
    # TODO: add control that residual_fraction_initial_value is between 0 and 1
    residual_fraction_initial_value = tf.convert_to_tensor(residual_fraction_initial_value,
                                                           dtype_hint=tf.float32,
                                                           name='residual_fraction_initial_value')
    dtype = residual_fraction_initial_value.dtype

    bias_seed, upper_seed, lower_seed, diagonal_seed = samplers.split_seed(seed, n=4)

    return HighwayFlow(
        width=width,
        residual_fraction=util.TransformedVariable(
            initial_value=tf.convert_to_tensor(residual_fraction_initial_value),
            bijector=tfb.Sigmoid(),
            dtype=dtype),
        activation_fn=activation_fn,
        bias=tf.Variable(samplers.normal((width,), 0., 0.01, seed=bias_seed), dtype=dtype),
        upper_diagonal_weights_matrix=util.TransformedVariable(
            initial_value=tf.experimental.numpy.tril(samplers.normal((width, width), 0., 1., seed=upper_seed),
                                                     -1) + tf.linalg.diag(
                samplers.uniform((width,), minval=0., maxval=1., seed=diagonal_seed)),
            bijector=tfb.FillScaleTriL(diag_bijector=tfb.Softplus(), diag_shift=None),
            dtype=dtype),
        lower_diagonal_weights_matrix=util.TransformedVariable(
            initial_value=samplers.normal((width, width), 0., 1., seed=lower_seed),
            bijector=tfb.Chain([tfb.TransformDiagonal(diag_bijector=tfb.Shift(1.)),
                                tfb.Pad(paddings=[(1, 0), (0, 1)]),
                                tfb.FillTriangular()]),
            dtype=dtype)
    )

class HighwayFlow(tfb.Bijector):
    # todo: update comments?
    # HighWay Flow simultaneously computes `forward` and `fldj` (and `inverse`/`ildj`),
    # so we override the bijector cache to update the LDJ entries of attrs on
    # forward/inverse inverse calls (instead of updating them only when the LDJ
    # methods themselves are called).
    _cache = cache_util.BijectorCacheWithGreedyAttrs(
        forward_name='_augmented_forward',
        inverse_name='_augmented_inverse')

    def __init__(self, width, residual_fraction, activation_fn, bias, upper_diagonal_weights_matrix,
                 lower_diagonal_weights_matrix, validate_args=False,
                 name='highway_flow'):
        super(HighwayFlow, self).__init__(
            validate_args=validate_args,
            forward_min_event_ndims=1,  # FIXME: should this also be an argument of HighwayFlow __init__?
            name=name)

        self.width = width

        self.bias = bias

        self.residual_fraction = residual_fraction

        # still lower triangular, transposed is done in matvec.
        # TODO: transpose directly here or in the definition of TransformedVariable?
        self.upper_diagonal_weights_matrix = upper_diagonal_weights_matrix

        self.lower_diagonal_weights_matrix = lower_diagonal_weights_matrix

        self.activation_fn = activation_fn

    def df(self, x):
        # derivative of activation
        return self.residual_fraction + (1. - self.residual_fraction) * tf.math.sigmoid(x) * (1. - tf.math.sigmoid(x))

    def _convex_update(self, weights_matrix):
        # convex update
        # same as in the paper, but probably easier to invert
        return self.residual_fraction * tf.eye(self.width) + (1. - self.residual_fraction) * weights_matrix

    def inv_f(self, y, N=20):
        # inverse with Newton iteration
        x = tf.ones(y.shape)
        for _ in range(N):
            x = x - (self.residual_fraction * x + (1. - self.residual_fraction) * tf.math.softplus(x) - y) / (
                self.df(x))
        return x

    def _augmented_forward(self, x):
        # upper mmatrix jacobian
        fldj = tf.zeros(x.shape[0]) + tf.reduce_sum(
            tf.math.log(self.residual_fraction + (1. - self.residual_fraction) * tf.linalg.diag_part(
                self.upper_diagonal_weights_matrix)))  # jacobian from upper matrix
        # jacobian from lower matrix is 0

        # todo: see how to optimize _convex_update
        x = tf.linalg.matvec(self._convex_update(self.lower_diagonal_weights_matrix), x)
        x = tf.linalg.matvec(tf.transpose(self._convex_update(self.upper_diagonal_weights_matrix)),
                             x) + (1-self.residual_fraction)*self.bias  # in the implementation there was only one bias
        if self.activation_fn:
            # fldj += tf.reduce_sum(tf.math.log(self.df(x)))
            # x = self.residual_fraction * x + (1. - self.residual_fraction) * self.activation_fn(x)
            activation_layer = scalar_function_with_inferred_inverse.ScalarFunctionWithInferredInverse(
                lambda x: self.residual_fraction * x + (1. - self.residual_fraction) * self.activation_fn(x))
            fldj += activation_layer.forward_log_det_jacobian(x, self.forward_min_event_ndims)
            x = activation_layer.forward(x)
        return x, {'ildj': -fldj, 'fldj': fldj}

    def _augmented_inverse(self, y):
        ildj = tf.zeros(y.shape[0]) - tf.reduce_sum(
            tf.math.log(self.residual_fraction + (1. - self.residual_fraction) * tf.linalg.diag_part(
                self.upper_diagonal_weights_matrix)))
        if self.activation_fn:
            # y = self.inv_f(y)
            # ildj += tf.reduce_sum(tf.math.log(self.df(y)))
            residual_fraction = tf.identity(self.residual_fraction)
            # FIXME: this way of using inverse activation does not seem to work because residual_fraction is a variable
            activation_layer = scalar_function_with_inferred_inverse.ScalarFunctionWithInferredInverse(
                lambda x: residual_fraction * x + (1. - residual_fraction) * self.activation_fn(x))
            ildj += activation_layer.inverse_log_det_jacobian(y, self.forward_min_event_ndims)
            y = activation_layer.inverse(y)

        # this works with y having shape [BATCH x WIDTH], don't know how well it generalizes
        y = tf.linalg.triangular_solve(tf.transpose(self._convex_update(self.upper_diagonal_weights_matrix)),
                                       tf.linalg.matrix_transpose(y - (1-self.residual_fraction)*self.bias), lower=False)
        y = tf.linalg.triangular_solve(self._convex_update(self.lower_diagonal_weights_matrix), y)
        return tf.linalg.matrix_transpose(y), {'ildj': ildj, 'fldj': -ildj}

    def _forward(self, x):
        y, _ = self._augmented_forward(x)
        return y

    def _inverse(self, y):
        x, _ = self._augmented_inverse(y)
        return x

    def _forward_log_det_jacobian(self, x):
        cached = self._cache.forward_attributes(x)
        # If LDJ isn't in the cache, call forward once.
        if 'fldj' not in cached:
            _, attrs = self._augmented_forward(x)
            cached.update(attrs)
        return cached['fldj']

    def _inverse_log_det_jacobian(self, y):
        cached = self._cache.inverse_attributes(y)
        # If LDJ isn't in the cache, call inverse once.
        if 'ildj' not in cached:
            _, attrs = self._augmented_inverse(y)
            cached.update(attrs)
        return cached['ildj']
<|endoftext|>