<commit_before># Copyright (c) OpenMMLab. All rights reserved.
import warnings

import mmcv
import numpy as np
import torch
from mmcv.ops import RoIPool
from mmcv.parallel import collate, scatter
from mmcv.runner import load_checkpoint

from mmdet.core import get_classes
from mmdet.datasets import replace_ImageToTensor
from mmdet.datasets.pipelines import Compose
from mmdet.models import build_detector

def init_detector(config, checkpoint=None, device='cuda:0', cfg_options=None):
    """Initialize a detector from config file.

    Args:
        config (str or :obj:`mmcv.Config`): Config file path or the config
            object.
        checkpoint (str, optional): Checkpoint path. If left as None, the model
            will not load any weights.
        cfg_options (dict): Options to override some settings in the used
            config.

    Returns:
        nn.Module: The constructed detector.
    """
    if isinstance(config, str):
        config = mmcv.Config.fromfile(config)
    elif not isinstance(config, mmcv.Config):
        raise TypeError('config must be a filename or Config object, '
                        f'but got {type(config)}')
    if cfg_options is not None:
        config.merge_from_dict(cfg_options)
    config.model.pretrained = None
    config.model.train_cfg = None
    model = build_detector(config.model, test_cfg=config.get('test_cfg'))
    if checkpoint is not None:
        map_loc = 'cpu' if device == 'cpu' else None
        checkpoint = load_checkpoint(model, checkpoint, map_location=map_loc)
        if 'CLASSES' in checkpoint.get('meta', {}):
            model.CLASSES = checkpoint['meta']['CLASSES']
        else:
            warnings.simplefilter('once')
            warnings.warn('Class names are not saved in the checkpoint\'s '
                          'meta data, use COCO classes by default.')
            model.CLASSES = get_classes('coco')
    model.cfg = config  # save the config in the model for convenience
    model.to(device)
    model.eval()
    return model

class LoadImage:
    """Deprecated.

    A simple pipeline to load image.
    """

    def __call__(self, results):
        """Call function to load images into results.

        Args:
            results (dict): A result dict contains the file name
                of the image to be read.
        Returns:
            dict: ``results`` will be returned containing loaded image.
        """
        warnings.simplefilter('once')
        warnings.warn('`LoadImage` is deprecated and will be removed in '
                      'future releases. You may use `LoadImageFromWebcam` '
                      'from `mmdet.datasets.pipelines.` instead.')
        if isinstance(results['img'], str):
            results['filename'] = results['img']
            results['ori_filename'] = results['img']
        else:
            results['filename'] = None
            results['ori_filename'] = None
        img = mmcv.imread(results['img'])
        results['img'] = img
        results['img_fields'] = ['img']
        results['img_shape'] = img.shape
        results['ori_shape'] = img.shape
        return results

def inference_detector(model, imgs):
    """Inference image(s) with the detector.

    Args:
        model (nn.Module): The loaded detector.
        imgs (str/ndarray or list[str/ndarray] or tuple[str/ndarray]):
           Either image files or loaded images.

    Returns:
        If imgs is a list or tuple, the same length list type results
        will be returned, otherwise return the detection results directly.
    """

    if isinstance(imgs, (list, tuple)):
        is_batch = True
    else:
        imgs = [imgs]
        is_batch = False

    cfg = model.cfg
    device = next(model.parameters()).device  # model device

    if isinstance(imgs[0], np.ndarray):
        cfg = cfg.copy()
        # set loading pipeline type
        cfg.data.test.pipeline[0].type = 'LoadImageFromWebcam'

    cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline)
    test_pipeline = Compose(cfg.data.test.pipeline)

    datas = []
    for img in imgs:
        # prepare data
        if isinstance(img, np.ndarray):
            # directly add img
            data = dict(img=img)
        else:
            # add information into dict
            data = dict(img_info=dict(filename=img), img_prefix=None)
        # build the data pipeline
        data = test_pipeline(data)
        datas.append(data)

    data = collate(datas, samples_per_gpu=len(imgs))
    # just get the actual data from DataContainer
    data['img_metas'] = [img_metas.data[0] for img_metas in data['img_metas']]
    data['img'] = [img.data[0] for img in data['img']]
    if next(model.parameters()).is_cuda:
        # scatter to specified GPU
        data = scatter(data, [device])[0]
    else:
        for m in model.modules():
            assert not isinstance(
                m, RoIPool
            ), 'CPU inference with RoIPool is not supported currently.'

    # forward the model
    with torch.no_grad():
        results = model(return_loss=False, rescale=True, **data)

    if not is_batch:
        return results[0]
    else:
        return results

async def async_inference_detector(model, imgs):
    """Async inference image(s) with the detector.

    Args:
        model (nn.Module): The loaded detector.
        img (str | ndarray): Either image files or loaded images.

    Returns:
        Awaitable detection results.
    """
    if not isinstance(imgs, (list, tuple)):
        imgs = [imgs]

    cfg = model.cfg
    device = next(model.parameters()).device  # model device

    if isinstance(imgs[0], np.ndarray):
        cfg = cfg.copy()
        # set loading pipeline type
        cfg.data.test.pipeline[0].type = 'LoadImageFromWebcam'

    cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline)
    test_pipeline = Compose(cfg.data.test.pipeline)

    datas = []
    for img in imgs:
        # prepare data
        if isinstance(img, np.ndarray):
            # directly add img
            data = dict(img=img)
        else:
            # add information into dict
            data = dict(img_info=dict(filename=img), img_prefix=None)
        # build the data pipeline
        data = test_pipeline(data)
        datas.append(data)

    data = collate(datas, samples_per_gpu=len(imgs))
    # just get the actual data from DataContainer
    data['img_metas'] = [img_metas.data[0] for img_metas in data['img_metas']]
    data['img'] = [img.data[0] for img in data['img']]
    if next(model.parameters()).is_cuda:
        # scatter to specified GPU
        data = scatter(data, [device])[0]
    else:
        for m in model.modules():
            assert not isinstance(
                m, RoIPool
            ), 'CPU inference with RoIPool is not supported currently.'

    # We don't restore `torch.is_grad_enabled()` value during concurrent
    # inference since execution can overlap
    torch.set_grad_enabled(False)
    results = await model.aforward_test(rescale=True, **data)
    return results

def show_result_pyplot(model,
                       img,
                       result,
                       score_thr=0.3,
                       title='result',
                       wait_time=0):
    """Visualize the detection results on the image.

    Args:
        model (nn.Module): The loaded detector.
        img (str or np.ndarray): Image filename or loaded image.
        result (tuple[list] or list): The detection result, can be either
            (bbox, segm) or just bbox.
        score_thr (float): The threshold to visualize the bboxes and masks.
        title (str): Title of the pyplot figure.
        wait_time (float): Value of waitKey param.
                Default: 0.
    """
    if hasattr(model, 'module'):
        model = model.module
    model.show_result(
        img,
        result,
        score_thr=score_thr,
        show=True,
        wait_time=wait_time,
        win_name=title,
        bbox_color=(72, 101, 241),
        text_color=(72, 101, 241))
<commit_msg>Always map location to cpu when load checkpoint (#6405)<commit_after># Copyright (c) OpenMMLab. All rights reserved.
import warnings

import mmcv
import numpy as np
import torch
from mmcv.ops import RoIPool
from mmcv.parallel import collate, scatter
from mmcv.runner import load_checkpoint

from mmdet.core import get_classes
from mmdet.datasets import replace_ImageToTensor
from mmdet.datasets.pipelines import Compose
from mmdet.models import build_detector

def init_detector(config, checkpoint=None, device='cuda:0', cfg_options=None):
    """Initialize a detector from config file.

    Args:
        config (str or :obj:`mmcv.Config`): Config file path or the config
            object.
        checkpoint (str, optional): Checkpoint path. If left as None, the model
            will not load any weights.
        cfg_options (dict): Options to override some settings in the used
            config.

    Returns:
        nn.Module: The constructed detector.
    """
    if isinstance(config, str):
        config = mmcv.Config.fromfile(config)
    elif not isinstance(config, mmcv.Config):
        raise TypeError('config must be a filename or Config object, '
                        f'but got {type(config)}')
    if cfg_options is not None:
        config.merge_from_dict(cfg_options)
    config.model.pretrained = None
    config.model.train_cfg = None
    model = build_detector(config.model, test_cfg=config.get('test_cfg'))
    if checkpoint is not None:
        checkpoint = load_checkpoint(model, checkpoint, map_location='cpu')
        if 'CLASSES' in checkpoint.get('meta', {}):
            model.CLASSES = checkpoint['meta']['CLASSES']
        else:
            warnings.simplefilter('once')
            warnings.warn('Class names are not saved in the checkpoint\'s '
                          'meta data, use COCO classes by default.')
            model.CLASSES = get_classes('coco')
    model.cfg = config  # save the config in the model for convenience
    model.to(device)
    model.eval()
    return model

class LoadImage:
    """Deprecated.

    A simple pipeline to load image.
    """

    def __call__(self, results):
        """Call function to load images into results.

        Args:
            results (dict): A result dict contains the file name
                of the image to be read.
        Returns:
            dict: ``results`` will be returned containing loaded image.
        """
        warnings.simplefilter('once')
        warnings.warn('`LoadImage` is deprecated and will be removed in '
                      'future releases. You may use `LoadImageFromWebcam` '
                      'from `mmdet.datasets.pipelines.` instead.')
        if isinstance(results['img'], str):
            results['filename'] = results['img']
            results['ori_filename'] = results['img']
        else:
            results['filename'] = None
            results['ori_filename'] = None
        img = mmcv.imread(results['img'])
        results['img'] = img
        results['img_fields'] = ['img']
        results['img_shape'] = img.shape
        results['ori_shape'] = img.shape
        return results

def inference_detector(model, imgs):
    """Inference image(s) with the detector.

    Args:
        model (nn.Module): The loaded detector.
        imgs (str/ndarray or list[str/ndarray] or tuple[str/ndarray]):
           Either image files or loaded images.

    Returns:
        If imgs is a list or tuple, the same length list type results
        will be returned, otherwise return the detection results directly.
    """

    if isinstance(imgs, (list, tuple)):
        is_batch = True
    else:
        imgs = [imgs]
        is_batch = False

    cfg = model.cfg
    device = next(model.parameters()).device  # model device

    if isinstance(imgs[0], np.ndarray):
        cfg = cfg.copy()
        # set loading pipeline type
        cfg.data.test.pipeline[0].type = 'LoadImageFromWebcam'

    cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline)
    test_pipeline = Compose(cfg.data.test.pipeline)

    datas = []
    for img in imgs:
        # prepare data
        if isinstance(img, np.ndarray):
            # directly add img
            data = dict(img=img)
        else:
            # add information into dict
            data = dict(img_info=dict(filename=img), img_prefix=None)
        # build the data pipeline
        data = test_pipeline(data)
        datas.append(data)

    data = collate(datas, samples_per_gpu=len(imgs))
    # just get the actual data from DataContainer
    data['img_metas'] = [img_metas.data[0] for img_metas in data['img_metas']]
    data['img'] = [img.data[0] for img in data['img']]
    if next(model.parameters()).is_cuda:
        # scatter to specified GPU
        data = scatter(data, [device])[0]
    else:
        for m in model.modules():
            assert not isinstance(
                m, RoIPool
            ), 'CPU inference with RoIPool is not supported currently.'

    # forward the model
    with torch.no_grad():
        results = model(return_loss=False, rescale=True, **data)

    if not is_batch:
        return results[0]
    else:
        return results

async def async_inference_detector(model, imgs):
    """Async inference image(s) with the detector.

    Args:
        model (nn.Module): The loaded detector.
        img (str | ndarray): Either image files or loaded images.

    Returns:
        Awaitable detection results.
    """
    if not isinstance(imgs, (list, tuple)):
        imgs = [imgs]

    cfg = model.cfg
    device = next(model.parameters()).device  # model device

    if isinstance(imgs[0], np.ndarray):
        cfg = cfg.copy()
        # set loading pipeline type
        cfg.data.test.pipeline[0].type = 'LoadImageFromWebcam'

    cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline)
    test_pipeline = Compose(cfg.data.test.pipeline)

    datas = []
    for img in imgs:
        # prepare data
        if isinstance(img, np.ndarray):
            # directly add img
            data = dict(img=img)
        else:
            # add information into dict
            data = dict(img_info=dict(filename=img), img_prefix=None)
        # build the data pipeline
        data = test_pipeline(data)
        datas.append(data)

    data = collate(datas, samples_per_gpu=len(imgs))
    # just get the actual data from DataContainer
    data['img_metas'] = [img_metas.data[0] for img_metas in data['img_metas']]
    data['img'] = [img.data[0] for img in data['img']]
    if next(model.parameters()).is_cuda:
        # scatter to specified GPU
        data = scatter(data, [device])[0]
    else:
        for m in model.modules():
            assert not isinstance(
                m, RoIPool
            ), 'CPU inference with RoIPool is not supported currently.'

    # We don't restore `torch.is_grad_enabled()` value during concurrent
    # inference since execution can overlap
    torch.set_grad_enabled(False)
    results = await model.aforward_test(rescale=True, **data)
    return results

def show_result_pyplot(model,
                       img,
                       result,
                       score_thr=0.3,
                       title='result',
                       wait_time=0):
    """Visualize the detection results on the image.

    Args:
        model (nn.Module): The loaded detector.
        img (str or np.ndarray): Image filename or loaded image.
        result (tuple[list] or list): The detection result, can be either
            (bbox, segm) or just bbox.
        score_thr (float): The threshold to visualize the bboxes and masks.
        title (str): Title of the pyplot figure.
        wait_time (float): Value of waitKey param.
                Default: 0.
    """
    if hasattr(model, 'module'):
        model = model.module
    model.show_result(
        img,
        result,
        score_thr=score_thr,
        show=True,
        wait_time=wait_time,
        win_name=title,
        bbox_color=(72, 101, 241),
        text_color=(72, 101, 241))
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from optparse import OptionParser

import struct
import binascii
import time

class LogFileHeader(object):
    def __init__(self, log):
        s = struct.Struct('4s')
        self.MAGIC = int(binascii.hexlify(s.pack('ZKLG')), 16)
        s = struct.Struct('>i i q')
        (self.magic, self.version, self.dbid) = s.unpack(log.read(s.size))
    def isvalid(self):
        return self.magic == self.MAGIC

CREATE = 1
DELETE = 2
SETDATA = 5
SETACL = 7
SESSIONCREATE = -10
SESSIONCLOSE = -11
opcodes = { CREATE:'create', DELETE:'delete', SETDATA:'setdata', SETACL:'setacl',
            SESSIONCREATE:'sessioncreate', SESSIONCLOSE:'sessionclose', }

# endofstream
class EOS(Exception):
    pass

class Txn(object):
    def __init__(self, log):
        s = struct.Struct('>q i')
        (self.crc, self.txn_len) = s.unpack(log.read(s.size))

        if self.txn_len == 0: raise EOS()

        self.header = h = TxnHeader(log)
        if h.type == CREATE:
            self.entry = TxnCreate(log)
        elif h.type == DELETE:
            self.entry = TxnDelete(log)
        elif h.type == SETDATA:
            self.entry = TxnSetData(log)
        elif h.type == SETACL:
            self.entry = TxnSetAcl(log)
        elif h.type == SESSIONCREATE:
            self.entry = TxnSessionCreate(log)
        elif h.type == SESSIONCLOSE:
            self.entry = TxnSessionClose(log)

        eor = log.read(1)
    def __str__(self):
        return "%s -- %s" % (self.header, self.entry)

class TxnHeader(object):
    def __init__(self, log):
        s = struct.Struct('>q i q q i')
        (self.client_id, self.cxid, self.zxid, self.time,
         self.type) = s.unpack(log.read(s.size))
    def op2type(self, type):
        return opcodes[type]
    def __str__(self):
        return "%s (%3dms) sessionid 0x%x zxid 0x%x cxid 0x%x %s" % (
                  time.ctime(self.time/1000), self.time % 1000,
                  self.client_id, self.zxid, self.cxid,
                  self.op2type(self.type))

class TxnEntry(object):
    def readString(self, log):
        _len = self.readInt(log)
        s = struct.Struct(str(_len) + 's')
        _str = s.unpack(log.read(s.size))
        return _str

    def readData(self, log):
        _len = self.readInt(log)
        return log.read(_len)

    def readAcls(self, log):
        count = self.readInt(log)
        return [self.readAcl(log) for i in xrange(count)]

    def readAcl(self, log):
        return Acl(log)

    def readInt(self, log):
        s = struct.Struct('>i')
        (_int,) = s.unpack(log.read(s.size))
        return _int

    def readBool(self, log):
        s = struct.Struct('B')
        _bool = s.unpack(log.read(s.size))
        return _bool == 0

class Acl(TxnEntry):
    def __init__(self, log):
        self.perms = self.readInt(log)
        self.scheme = self.readString(log)
        self.id = self.readString(log)
    def __str__(self):
        return "Acl %s %s %x" % (self.scheme, self.id, self.perms)

class TxnCreate(TxnEntry):
    def __init__(self, log):
        self.path = self.readString(log)
        self.data = self.readData(log)
        self.acls = self.readAcls(log)
        self.ephemeral = self.readBool(log)
    def __str__(self):
        return "Create path %s data '%s' acls %s ephemeral %i" % (
                 self.path, self.data, self.acls, self.ephemeral)

class TxnDelete(TxnEntry):
    def __init__(self, log):
        path = self.readString(log)
    def __str__(self):
        return "Delete path %s" % self.path

class TxnSetData(TxnEntry):
    def __init__(self, log):
        self.path = self.readString(log)
        self.data = self.readData(log)
        self.version = self.readInt(log)
    def __str__(self):
        return "SetData path %s data '%s' version %i" % (self.path, self.data, self.version)

class TxnSetAcl(TxnEntry):
    def __init__(self, log):
        self.path = self.readString(log)
        self.acls = self.readAcls(log)
        self.version = self.readInt(log)
    def __str__(self):
        return "SetAcl path %s acls %s version %i" % (self.path, self.acls, self.version)

class TxnSessionCreate(TxnEntry):
    def __init__(self, log):
        self.timeout = self.readInt(log)
    def __str__(self):
        return "SessionCreate timeout %ims" % (self.timeout)

class TxnSessionClose(TxnEntry):
    def __init__(self, log):
        pass
    def __str__(self):
        return "SessionClose"

if __name__ == '__main__':
    usage = "usage: %prog [options] zookeeper_txnlog"
    parser = OptionParser(usage=usage)

    #parser.add_option("-c", "--count", dest="count", type="int",
    #                  default=3, help="ensemble size (default 3)")

    (options, args) = parser.parse_args()

    if len(args) != 1:
        parser.error("need log file")

    log = open(args[0], 'rb')
    log_header = LogFileHeader(log)
    if not log_header.isvalid():
        parser.error("Not a valid ZooKeeper transaction log")

    start = None
    try:
        while(True):
            txn = Txn(log)
            if not start:
                start = txn.header.time
                print("Log starts at %s and %ims" % (time.ctime(start/1000), start % 1000))
            diff = txn.header.time - start
            print("%09i,%03i %s" % (diff/1000, diff%1000, str(txn)[33:]))
    except Exception as e:
        print(e)

    log.close()
<commit_msg>Added more txn types supported<commit_after>#!/usr/bin/env python

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from optparse import OptionParser

import struct
import binascii
import time

class LogFileHeader(object):
    def __init__(self, log):
        s = struct.Struct('4s')
        self.MAGIC = int(binascii.hexlify(s.pack('ZKLG')), 16)
        s = struct.Struct('>i i q')
        (self.magic, self.version, self.dbid) = s.unpack(log.read(s.size))
    def isvalid(self):
        return self.magic == self.MAGIC

SESSIONCLOSE = -11
SESSIONCREATE = -10
ERROR = -1
NOTIFICATION = 0
CREATE = 1
DELETE = 2
EXISTS = 3
GETDATA = 4
SETDATA = 5
GETACL = 6
SETACL = 7
GETCHILDREN = 8
SYNC = 9
PING = 11
GETCHILDREN2 = 12
CHECK = 13
MULTI = 14
AUTH = 100
SETWATCHES = 101
SASL = 102

opcodes = {
    SESSIONCLOSE:'sessionclose',
    SESSIONCREATE:'sessioncreate',
    ERROR:'error',
    NOTIFICATION:'notification',
    CREATE:'create',
    DELETE:'delete',
    EXISTS:'exists',
    GETDATA:'getdata',
    SETDATA:'setdata',
    GETACL:'getacl',
    SETACL:'setacl',
    GETCHILDREN:'getchildren',
    SYNC:'sync',
    PING:'ping',
    GETCHILDREN2:'getchildren2',
    CHECK:'check',
    MULTI:'multi',
    AUTH:'auth',
    SETWATCHES:'setwatches',
    SASL:'sasl',
}

# endofstream
class EOS(Exception):
    pass

class UnknownType(Exception):
    def __init__(self, type):
        self.type = type
    def __str__(self):
        return "Unknown type %d" % self.type

class Txn(object):
    def __init__(self, log):
        s = struct.Struct('>q i')
        (self.crc, self.txn_len) = s.unpack(log.read(s.size))

        if self.txn_len == 0: raise EOS()

        self.header = h = TxnHeader(log)
        if h.type == CREATE:
            self.entry = TxnCreate(log)
        elif h.type == DELETE:
            self.entry = TxnDelete(log)
        elif h.type == SETDATA:
            self.entry = TxnSetData(log)
        elif h.type == SETACL:
            self.entry = TxnSetAcl(log)
        elif h.type == SESSIONCREATE:
            self.entry = TxnSessionCreate(log)
        elif h.type == SESSIONCLOSE:
            self.entry = TxnSessionClose(log)
        elif h.type == ERROR:
            self.entry = TxnError(log)
        else:
            raise(UnknownType(h.type))

        eor = log.read(1)
    def __str__(self):
        return "%s -- %s" % (self.header, self.entry)

class TxnHeader(object):
    def __init__(self, log):
        s = struct.Struct('>Q I Q Q i')
        (self.client_id, self.cxid, self.zxid, self.time,
         self.type) = s.unpack(log.read(s.size))
    def op2type(self, type):
        return opcodes[type]
    def __str__(self):
        return "%s (%3dms) sessionid 0x%x zxid 0x%x cxid 0x%x %s" % (
                  time.ctime(self.time/1000), self.time % 1000,
                  self.client_id, self.zxid, self.cxid,
                  self.op2type(self.type))

class TxnEntry(object):
    def readString(self, log):
        _len = self.readInt(log)
        s = struct.Struct(str(_len) + 's')
        _str = s.unpack(log.read(s.size))
        return _str

    def readData(self, log):
        _len = self.readInt(log)
        return log.read(_len)

    def readAcls(self, log):
        count = self.readInt(log)
        return [self.readAcl(log) for i in xrange(count)]

    def readAcl(self, log):
        return Acl(log)

    def readInt(self, log):
        s = struct.Struct('>i')
        (_int,) = s.unpack(log.read(s.size))
        return _int

    def readBool(self, log):
        s = struct.Struct('B')
        _bool = s.unpack(log.read(s.size))
        return _bool == 0

class Acl(TxnEntry):
    def __init__(self, log):
        self.perms = self.readInt(log)
        self.scheme = self.readString(log)
        self.id = self.readString(log)
    def __str__(self):
        return "Acl %s %s %x" % (self.scheme, self.id, self.perms)

class TxnError(TxnEntry):
    Ok = 0
    SystemError = -1
    RuntimeInconsistency = -2
    DataInconsistency = -3
    ConnectionLoss = -4
    MarshallingError = -5
    Unimplemented = -6
    OperationTimeout = -7
    BadArguments = -8
    APIError = -100
    NoNode = -101
    NoAuth = -102
    BadVersion = -103
    NoChildrenForEphemerals = -108
    NodeExists = -110
    NotEmpty = -111
    SessionExpired = -112
    InvalidCallback = -113
    InvalidACL = -114
    AuthFailed = -115
    SessionMoved = -118

    errorcodes = {
        Ok:'ok',
        SystemError:'systemerror',
        RuntimeInconsistency:'runtimeinconsistency',
        DataInconsistency:'datainconsistency',
        ConnectionLoss:'connectionloss',
        MarshallingError:'marshallingerror',
        Unimplemented:'unimplemented',
        OperationTimeout:'operationtimeout',
        BadArguments:'badarguments',
        APIError:'apierror',
        NoNode:'nonode',
        NoAuth:'noauth',
        BadVersion:'badversion',
        NoChildrenForEphemerals:'nochildrenforephemerals',
        NodeExists:'nodeexists',
        NotEmpty:'notempty',
        SessionExpired:'sessionexpired',
        InvalidCallback:'invalidcallback',
        InvalidACL:'invalidacl',
        AuthFailed:'authfailed',
        SessionMoved:'sessionmoved',
    }

    def __init__(self, log):
        self.err = self.readInt(log)
    def __str__(self):
        return "Error %s" % self.errorcodes[self.err]

class TxnCreate(TxnEntry):
    def __init__(self, log):
        self.path = self.readString(log)
        self.data = self.readData(log)
        self.acls = self.readAcls(log)
        self.ephemeral = self.readBool(log)
    def __str__(self):
        return "Create path %s data '%s' acls %s ephemeral %i" % (
                 self.path, self.data, self.acls, self.ephemeral)

class TxnDelete(TxnEntry):
    def __init__(self, log):
        self.path = self.readString(log)
    def __str__(self):
        return "Delete path %s" % self.path

class TxnSetData(TxnEntry):
    def __init__(self, log):
        self.path = self.readString(log)
        self.data = self.readData(log)
        self.version = self.readInt(log)
    def __str__(self):
        return "SetData path %s data '%s' version %i" % (self.path, self.data, self.version)

class TxnSetAcl(TxnEntry):
    def __init__(self, log):
        self.path = self.readString(log)
        self.acls = self.readAcls(log)
        self.version = self.readInt(log)
    def __str__(self):
        return "SetAcl path %s acls %s version %i" % (self.path, self.acls, self.version)

class TxnSessionCreate(TxnEntry):
    def __init__(self, log):
        self.timeout = self.readInt(log)
    def __str__(self):
        return "SessionCreate timeout %ims" % (self.timeout)

class TxnSessionClose(TxnEntry):
    def __init__(self, log):
        pass
    def __str__(self):
        return "SessionClose"

if __name__ == '__main__':
    usage = "usage: %prog [options] zookeeper_txnlog"
    parser = OptionParser(usage=usage)

    #parser.add_option("-c", "--count", dest="count", type="int",
    #                  default=3, help="ensemble size (default 3)")

    (options, args) = parser.parse_args()

    if len(args) != 1:
        parser.error("need log file")

    log = open(args[0], 'rb')
    log_header = LogFileHeader(log)
    if not log_header.isvalid():
        parser.error("Not a valid ZooKeeper transaction log")

    start = None
    try:
        while(True):
            txn = Txn(log)
            if not start:
                start = txn.header.time
                print("Log starts at %s and %ims" % (time.ctime(start/1000), start % 1000))
            diff = txn.header.time - start
            print("%09i,%03i %s" % (diff/1000, diff%1000, str(txn)[33:]))
    except Exception as e:
        print(e)

    log.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import time

import configfetcher
import messagebuilder
import rqueue
import channelfilter

class Redis2Stdout(object):
    def __init__(self, conf, builder, channelfilter):
        """
        :type conf: configfetcher.ConfigFetcher
        :type builder: messagebuilder.IRCMessageBuilder
        :type channelfilter: channelfilter.ChannelFilter
        """
        self.rqueue = rqueue.RedisQueue(
            conf.get('REDIS_QUEUE_NAME'),
            conf.get('REDIS_HOST')
        )
        self.conf = conf
        self.join_channels = conf.get('CHANNELS').values()
        self.builder = builder
        self.connected = False
        self.channelfilter = channelfilter

    def get_channels_for_projects(self, projects):
        """
        :param projects: List of human readable project names
        :type projects: list
        """
        channels = set()
        for proj in projects:
            proj_channels = self.channelfilter.channels_for(proj)
            if proj_channels:
                channels.union(proj_channels)

        return channels

    def start(self):
        while 1:
            time.sleep(0.1)
            useful_info = self.rqueue.get()
            print(useful_info)
            if useful_info:
                text = self.builder.build_message(useful_info)
                channels = self.get_channels_for_projects(useful_info['projects'])
                print(','.join(channels) + ': ' + text)

if __name__ == '__main__':
    bot = Redis2Stdout(
        configfetcher.ConfigFetcher(),
        messagebuilder.IRCMessageBuilder(),
        channelfilter.ChannelFilter()
    )
    bot.start()
<commit_msg>Unbreak redis2stdout<commit_after>#!/usr/bin/env python

import time

import configfetcher
import messagebuilder
import rqueue
import channelfilter

class Redis2Stdout(object):
    def __init__(self, conf, builder, channelfilter):
        """
        :type conf: configfetcher.ConfigFetcher
        :type builder: messagebuilder.IRCMessageBuilder
        :type channelfilter: channelfilter.ChannelFilter
        """
        self.rqueue = rqueue.RedisQueue(
            conf.get('REDIS_QUEUE_NAME'),
            conf.get('REDIS_HOST')
        )
        self.conf = conf
        self.builder = builder
        self.connected = False
        self.channelfilter = channelfilter

    def get_channels_for_projects(self, projects):
        """
        :param projects: List of human readable project names
        :type projects: list
        """
        channels = set()
        for proj in projects:
            proj_channels = self.channelfilter.channels_for(proj)
            if proj_channels:
                channels.union(proj_channels)

        return channels

    def start(self):
        while 1:
            time.sleep(0.1)
            useful_info = self.rqueue.get()
            print(useful_info)
            if useful_info:
                text = self.builder.build_message(useful_info)
                channels = self.get_channels_for_projects(useful_info['projects'])
                print(','.join(channels) + ': ' + text)

if __name__ == '__main__':
    bot = Redis2Stdout(
        configfetcher.ConfigFetcher(),
        messagebuilder.IRCMessageBuilder(),
        channelfilter.ChannelFilter()
    )
    bot.start()
<|endoftext|>


<commit_before>""" Collection of core plotting objects, which can be represented in the
Javascript layer.  The object graph formed by composing the objects in
this module can be stored as a backbone.js model graph, and stored in a
plot server or serialized into JS for embedding in HTML or an IPython
notebook.
"""
from __future__ import absolute_import

import warnings
import logging
logger = logging.getLogger(__file__)

from .properties import (HasProps, Dict, Enum, Either, Float, Instance, Int,
    List, String, Color, Include, Bool, Tuple, Any)
from .mixins import FillProps, LineProps, TextProps
from .enums import Units, Orientation, Dimension, BorderSymmetry
from .plotobject import PlotObject
from .glyphs import BaseGlyph

class DataSource(PlotObject):
    """ Base class for data sources """
    # List of names of the fields of each tuple in self.data
    # ordering is incoporated here
    column_names = List(String)
    selected = List(String) # index of selected points

    def columns(self, *columns):
        """ Returns a ColumnsRef object that points to a column or set of
        columns on this data source
        """
        return ColumnsRef(source=self, columns=list(columns))

class ColumnsRef(HasProps):
    source = Instance(DataSource, has_ref=True)
    columns = List(String)

class ColumnDataSource(DataSource):
    # Maps names of columns to sequences or arrays
    data = Dict()

    # Maps field/column name to a DataRange or FactorRange object. If the
    # field is not in the dict, then a range is created automatically.
    cont_ranges = Dict()
    discrete_ranges = Dict()

    def __init__(self, *args, **kw):
        """ Modify the basic DataSource/PlotObj constructor so that if we
        are called with a single argument that is a dict, then we treat
        that implicitly as our "data" attribute.
        """
        if len(args) == 1 and "data" not in kw:
            kw["data"] = args[0]
        raw_data = kw.get("data", {})
        if not isinstance(raw_data, dict):
            import pandas as pd
            if isinstance(raw_data, pd.DataFrame):
                new_data = {}
                for colname in raw_data:
                    new_data[colname] = raw_data[colname].tolist()
                raw_data = new_data
        for name, data in raw_data.items():
            self.add(data, name)
        super(ColumnDataSource, self).__init__(**kw)

    def add(self, data, name=None):
        """ Appends the data to the list of columns.  Returns the name
        that was inserted.
        """
        if name is None:
            n = len(self.data)
            while "Series %d"%n in self.data:
                n += 1
            name = "Series %d"%n
        self.column_names.append(name)
        self.data[name] = data
        return name

    def remove(self, name):
        try:
            self.column_names.remove(name)
            del self.data[name]
        except (ValueError, KeyError):
            warnings.warn("Unable to find column '%s' in datasource" % name)

class PandasDataSource(DataSource):
    """ Represents serverside data.  This gets stored into the plot server's
    database, but it does not have any client side representation.  Instead,
    a PandasPlotSource needs to be created and pointed at it.
    """

    data = Dict()

class Range(PlotObject):
    pass

class Range1d(Range):
    start = Float()
    end = Float()

class DataRange(Range):
    sources = List(Instance(ColumnsRef), has_ref=True)

    def finalize(self, models):
        super(DataRange, self).finalize(models)
        for idx, source in enumerate(self.sources):
            if isinstance(source, dict):
                self.sources[idx] = ColumnsRef(
                    source=source['ref'],
                    columns=source['columns'])

    def references(self):
        return [x.source for x in self.sources]

class DataRange1d(DataRange):
    """ Represents a range in a scalar dimension """
    sources = List(Instance(ColumnsRef), has_ref=True)
    rangepadding = Float(0.1)
    start = Float
    end = Float

class FactorRange(Range):
    """ Represents a range in a categorical dimension """
    factors = List(Any)

class Renderer(PlotObject):
    pass

class Glyph(Renderer):
    data_source = Instance(DataSource, has_ref=True)
    xdata_range = Instance(Range, has_ref=True)
    ydata_range = Instance(Range, has_ref=True)

    # How to intepret the values in the data_source
    units = Enum(Units)

    glyph = Instance(BaseGlyph)

    # Optional glyph used when data is selected.
    selection_glyph = Instance(BaseGlyph)
    # Optional glyph used when data is unselected.
    nonselection_glyph = Instance(BaseGlyph)

    def vm_serialize(self):
        # Glyphs need to serialize their state a little differently,
        # because the internal glyph instance is turned into a glyphspec
        data =  {"id" : self._id,
                 "data_source": self.data_source,
                 "xdata_range": self.xdata_range,
                 "ydata_range": self.ydata_range,
                 "glyphspec": self.glyph.to_glyphspec()
                 }
        if self.selection_glyph:
            data['selection_glyphspec'] = self.selection_glyph.to_glyphspec()
        if self.nonselection_glyph:
            data['nonselection_glyphspec'] = self.nonselection_glyph.to_glyphspec()
        return data

    def finalize(self, models):
        super(Glyph, self).finalize(models)

        ## FIXME: we shouldn't have to do this i think..

        if hasattr(self, 'glyphspec'):
            glyphspec = self.glyphspec
            del self.glyphspec
            glyph_type = glyphspec.pop('type')
            cls = PlotObject.get_class(glyph_type)
            self.glyph = cls(**glyphspec)
        else:
            self.glyph = None

        if hasattr(self, 'selection_glyphspec'):
            selection_glyphspec = self.selection_glyphspec
            del self.selection_glyphspec
            glyph_type = selection_glyphspec.pop('type')
            cls = PlotObject.get_class(glyph_type)
            self.selection_glyph = cls(**selection_glyphspec)
        else:
            self.selection_glyph = None

        if hasattr(self, 'nonselection_glyphspec'):
            nonselection_glyphspec = self.nonselection_glyphspec
            del self.nonselection_glyphspec
            glyph_type = nonselection_glyphspec.pop('type')
            cls = PlotObject.get_class(glyph_type)
            self.nonselection_glyph = cls(**nonselection_glyphspec)
        else:
            self.nonselection_glyph = None

class Plot(PlotObject):
    """ Object representing a plot, containing glyphs, guides, annotations.
    """

    data_sources = List(Instance(DataSource), has_ref=True)
    title = String("Bokeh Plot")

    x_range = Instance(Range, has_ref=True)
    y_range = Instance(Range, has_ref=True)
    png = String('')
    title = String('')
    outline_props = Include(LineProps, prefix="outline")

    # A list of all renderers on this plot; this includes guides as well
    # as glyph renderers
    renderers = List(Instance(Renderer), has_ref=True)
    tools = List(Instance(".objects.Tool"), has_ref=True)

    # TODO: These don't appear in the CS source, but are created by mpl.py, so
    # I'm leaving them here for initial compatibility testing.
    # axes = List(has_ref=True)

    # TODO: How do we want to handle syncing of the different layers?
    # image = List()
    # underlay = List()
    # glyph = List()
    #
    # annotation = List()

    height = Int(600)
    width = Int(600)

    background_fill = Color("white")
    border_fill = Color("white")
    canvas_width = Int(400)
    canvas_height = Int(400)
    outer_width = Int(400)
    outer_height = Int(400)
    min_border_top = Int(50)
    min_border_bottom = Int(50)
    min_border_left = Int(50)
    min_border_right = Int(50)
    min_border = Int(50)
    border_symmetry = Enum(BorderSymmetry)
    script_inject_snippet = String("")

    def vm_props(self, *args, **kw):
        # FIXME: We need to duplicate the height and width into canvas and
        # outer height/width.  This is a quick fix for the gorpiness, but this
        # needs to be fixed more structurally on the JS side, and then this
        # should be revisited on the Python side.
        if hasattr(self.session, "root_url"):
            self.script_inject_snippet = self.create_html_snippet(server=True)
        if "canvas_width" not in self._changed_vars:
            self.canvas_width = self.width
        if "outer_width" not in self._changed_vars:
            self.outer_width = self.width
        if "canvas_height" not in self._changed_vars:
            self.canvas_height = self.height
        if "outer_height" not in self._changed_vars:
            self.outer_height = self.height
        return super(Plot, self).vm_props(*args, **kw)

class MapOptions(HasProps):
    lat = Float
    lng = Float
    zoom = Int(12)

class GMapPlot(Plot):
    map_options = Instance(MapOptions)

    def vm_serialize(self):
        data = super(GMapPlot, self).vm_serialize()
        self._session.raw_js_snippets(self)
        return data

    def get_raw_js(self):
        return '<script src="https://maps.googleapis.com/maps/api/js?sensor=false"></script>'

class GridPlot(Plot):
    """ A 2D grid of plots """

    children = List(List(Instance(Plot), has_ref=True), has_ref=True)
    border_space = Int(0)

class GuideRenderer(Renderer):
    plot = Instance(Plot, has_ref=True)

    def __init__(self, **kwargs):
        super(GuideRenderer, self).__init__(**kwargs)

        if self.plot is not None:
            if self not in self.plot.renderers:
                self.plot.renderers.append(self)

class Axis(GuideRenderer):
    type = String("axis")

    dimension = Int(0)
    location = Either(String('min'), Float)
    bounds = Either(Enum('auto'), Tuple) # XXX: Tuple(Float, Float)

    axis_label = String
    axis_label_standoff = Int
    axis_label_props = Include(TextProps, prefix="axis_label")

    major_label_standoff = Int
    major_label_orientation = Either(Enum("horizontal", "vertical"), Int)
    major_label_props = Include(TextProps, prefix="major_label")

    # Line props
    axis_props = Include(LineProps, prefix="axis")
    tick_props = Include(LineProps, prefix="major_tick")

    major_tick_in = Int
    major_tick_out = Int

class LinearAxis(Axis):
    type = String("linear_axis")

class CategoricalAxis(Axis):
    type = String("categorical_axis")

class DatetimeAxis(LinearAxis):
    type = String("datetime_axis")
    axis_label = String("date")
    scale = String("time")
    num_labels = Int(8)
    char_width = Int(10)
    fill_ratio = Float(0.3)
    formats = Dict({"days": ["%m/%d/%Y"]})

class Grid(GuideRenderer):
    """ 1D Grid component """
    type = String("grid")

    dimension = Int(0)
    bounds = String('auto')

    is_datetime = Bool(False)

    # Line props
    grid_props = Include(LineProps, prefix="grid")

class Tool(PlotObject):
    plot = Instance(Plot, has_ref=True)

class PanTool(Tool):
    dimensions = List(Enum(Dimension))

class WheelZoomTool(Tool):
    dimensions = List(Enum(Dimension))

class PreviewSaveTool(Tool):
    pass

class EmbedTool(Tool):
    pass

class ResetTool(Tool):
    pass

class ResizeTool(Tool):
    pass

class CrosshairTool(Tool):
    pass

class BoxZoomTool(Tool):
    pass

class BoxSelectTool(Tool):
    renderers = List(Instance(Renderer), has_ref=True)
    select_every_mousemove = Bool(True)

class BoxSelectionOverlay(Renderer):
    __view_model__ = 'BoxSelection'
    tool = Instance(Tool, has_ref=True)

class HoverTool(Tool):
    renderers = List(Instance(Renderer), has_ref=True)
    tooltips = Dict()

class ObjectExplorerTool(Tool):
    pass

class Legend(PlotObject):
    plot = Instance(Plot, has_ref=True)
    orientation = Enum(Orientation)
    border = Include(LineProps, prefix="border")

    label_props = Include(TextProps, prefix="label")
    label_standoff = Int(15)
    label_height = Int(20)
    label_width = Int(50)
    legend_padding = Int(10)

    glyph_height = Int(20)
    glyph_width = Int(20)
    legend_spacing = Int(3)
    legends = Dict()

class DataSlider(PlotObject):
    plot = Instance(Plot, has_ref=True)
    data_source = Instance(DataSource, has_ref=True)
    field = String()

class DataRangeBoxSelectTool(PlotObject):
    plot = Instance(Plot, has_ref=True)
    xselect = List(Instance(Range), has_ref=True)
    yselect = List(Instance(Range), has_ref=True)

class PlotContext(PlotObject):
    children = List(List(Instance(Plot), has_ref=True), has_ref=True)

class PlotList(PlotContext):
    # just like plot context, except plot context has special meaning
    # everywhere, so plotlist is the generic one
    pass
<commit_msg>Don't override 'sources' in DataRange1d<commit_after>""" Collection of core plotting objects, which can be represented in the
Javascript layer.  The object graph formed by composing the objects in
this module can be stored as a backbone.js model graph, and stored in a
plot server or serialized into JS for embedding in HTML or an IPython
notebook.
"""
from __future__ import absolute_import

import warnings
import logging
logger = logging.getLogger(__file__)

from .properties import (HasProps, Dict, Enum, Either, Float, Instance, Int,
    List, String, Color, Include, Bool, Tuple, Any)
from .mixins import FillProps, LineProps, TextProps
from .enums import Units, Orientation, Dimension, BorderSymmetry
from .plotobject import PlotObject
from .glyphs import BaseGlyph

class DataSource(PlotObject):
    """ Base class for data sources """
    # List of names of the fields of each tuple in self.data
    # ordering is incoporated here
    column_names = List(String)
    selected = List(String) # index of selected points

    def columns(self, *columns):
        """ Returns a ColumnsRef object that points to a column or set of
        columns on this data source
        """
        return ColumnsRef(source=self, columns=list(columns))

class ColumnsRef(HasProps):
    source = Instance(DataSource, has_ref=True)
    columns = List(String)

class ColumnDataSource(DataSource):
    # Maps names of columns to sequences or arrays
    data = Dict()

    # Maps field/column name to a DataRange or FactorRange object. If the
    # field is not in the dict, then a range is created automatically.
    cont_ranges = Dict()
    discrete_ranges = Dict()

    def __init__(self, *args, **kw):
        """ Modify the basic DataSource/PlotObj constructor so that if we
        are called with a single argument that is a dict, then we treat
        that implicitly as our "data" attribute.
        """
        if len(args) == 1 and "data" not in kw:
            kw["data"] = args[0]
        raw_data = kw.get("data", {})
        if not isinstance(raw_data, dict):
            import pandas as pd
            if isinstance(raw_data, pd.DataFrame):
                new_data = {}
                for colname in raw_data:
                    new_data[colname] = raw_data[colname].tolist()
                raw_data = new_data
        for name, data in raw_data.items():
            self.add(data, name)
        super(ColumnDataSource, self).__init__(**kw)

    def add(self, data, name=None):
        """ Appends the data to the list of columns.  Returns the name
        that was inserted.
        """
        if name is None:
            n = len(self.data)
            while "Series %d"%n in self.data:
                n += 1
            name = "Series %d"%n
        self.column_names.append(name)
        self.data[name] = data
        return name

    def remove(self, name):
        try:
            self.column_names.remove(name)
            del self.data[name]
        except (ValueError, KeyError):
            warnings.warn("Unable to find column '%s' in datasource" % name)

class PandasDataSource(DataSource):
    """ Represents serverside data.  This gets stored into the plot server's
    database, but it does not have any client side representation.  Instead,
    a PandasPlotSource needs to be created and pointed at it.
    """

    data = Dict()

class Range(PlotObject):
    pass

class Range1d(Range):
    start = Float()
    end = Float()

class DataRange(Range):
    sources = List(Instance(ColumnsRef), has_ref=True)

    def finalize(self, models):
        super(DataRange, self).finalize(models)
        for idx, source in enumerate(self.sources):
            if isinstance(source, dict):
                self.sources[idx] = ColumnsRef(
                    source=source['ref'],
                    columns=source['columns'])

    def references(self):
        return [x.source for x in self.sources]

class DataRange1d(DataRange):
    """ Represents a range in a scalar dimension """
    rangepadding = Float(0.1)
    start = Float
    end = Float

class FactorRange(Range):
    """ Represents a range in a categorical dimension """
    factors = List(Any)

class Renderer(PlotObject):
    pass

class Glyph(Renderer):
    data_source = Instance(DataSource, has_ref=True)
    xdata_range = Instance(Range, has_ref=True)
    ydata_range = Instance(Range, has_ref=True)

    # How to intepret the values in the data_source
    units = Enum(Units)

    glyph = Instance(BaseGlyph)

    # Optional glyph used when data is selected.
    selection_glyph = Instance(BaseGlyph)
    # Optional glyph used when data is unselected.
    nonselection_glyph = Instance(BaseGlyph)

    def vm_serialize(self):
        # Glyphs need to serialize their state a little differently,
        # because the internal glyph instance is turned into a glyphspec
        data =  {"id" : self._id,
                 "data_source": self.data_source,
                 "xdata_range": self.xdata_range,
                 "ydata_range": self.ydata_range,
                 "glyphspec": self.glyph.to_glyphspec()
                 }
        if self.selection_glyph:
            data['selection_glyphspec'] = self.selection_glyph.to_glyphspec()
        if self.nonselection_glyph:
            data['nonselection_glyphspec'] = self.nonselection_glyph.to_glyphspec()
        return data

    def finalize(self, models):
        super(Glyph, self).finalize(models)

        ## FIXME: we shouldn't have to do this i think..

        if hasattr(self, 'glyphspec'):
            glyphspec = self.glyphspec
            del self.glyphspec
            glyph_type = glyphspec.pop('type')
            cls = PlotObject.get_class(glyph_type)
            self.glyph = cls(**glyphspec)
        else:
            self.glyph = None

        if hasattr(self, 'selection_glyphspec'):
            selection_glyphspec = self.selection_glyphspec
            del self.selection_glyphspec
            glyph_type = selection_glyphspec.pop('type')
            cls = PlotObject.get_class(glyph_type)
            self.selection_glyph = cls(**selection_glyphspec)
        else:
            self.selection_glyph = None

        if hasattr(self, 'nonselection_glyphspec'):
            nonselection_glyphspec = self.nonselection_glyphspec
            del self.nonselection_glyphspec
            glyph_type = nonselection_glyphspec.pop('type')
            cls = PlotObject.get_class(glyph_type)
            self.nonselection_glyph = cls(**nonselection_glyphspec)
        else:
            self.nonselection_glyph = None

class Plot(PlotObject):
    """ Object representing a plot, containing glyphs, guides, annotations.
    """

    data_sources = List(Instance(DataSource), has_ref=True)
    title = String("Bokeh Plot")

    x_range = Instance(Range, has_ref=True)
    y_range = Instance(Range, has_ref=True)
    png = String('')
    title = String('')
    outline_props = Include(LineProps, prefix="outline")

    # A list of all renderers on this plot; this includes guides as well
    # as glyph renderers
    renderers = List(Instance(Renderer), has_ref=True)
    tools = List(Instance(".objects.Tool"), has_ref=True)

    # TODO: These don't appear in the CS source, but are created by mpl.py, so
    # I'm leaving them here for initial compatibility testing.
    # axes = List(has_ref=True)

    # TODO: How do we want to handle syncing of the different layers?
    # image = List()
    # underlay = List()
    # glyph = List()
    #
    # annotation = List()

    height = Int(600)
    width = Int(600)

    background_fill = Color("white")
    border_fill = Color("white")
    canvas_width = Int(400)
    canvas_height = Int(400)
    outer_width = Int(400)
    outer_height = Int(400)
    min_border_top = Int(50)
    min_border_bottom = Int(50)
    min_border_left = Int(50)
    min_border_right = Int(50)
    min_border = Int(50)
    border_symmetry = Enum(BorderSymmetry)
    script_inject_snippet = String("")

    def vm_props(self, *args, **kw):
        # FIXME: We need to duplicate the height and width into canvas and
        # outer height/width.  This is a quick fix for the gorpiness, but this
        # needs to be fixed more structurally on the JS side, and then this
        # should be revisited on the Python side.
        if hasattr(self.session, "root_url"):
            self.script_inject_snippet = self.create_html_snippet(server=True)
        if "canvas_width" not in self._changed_vars:
            self.canvas_width = self.width
        if "outer_width" not in self._changed_vars:
            self.outer_width = self.width
        if "canvas_height" not in self._changed_vars:
            self.canvas_height = self.height
        if "outer_height" not in self._changed_vars:
            self.outer_height = self.height
        return super(Plot, self).vm_props(*args, **kw)

class MapOptions(HasProps):
    lat = Float
    lng = Float
    zoom = Int(12)

class GMapPlot(Plot):
    map_options = Instance(MapOptions)

    def vm_serialize(self):
        data = super(GMapPlot, self).vm_serialize()
        self._session.raw_js_snippets(self)
        return data

    def get_raw_js(self):
        return '<script src="https://maps.googleapis.com/maps/api/js?sensor=false"></script>'

class GridPlot(Plot):
    """ A 2D grid of plots """

    children = List(List(Instance(Plot), has_ref=True), has_ref=True)
    border_space = Int(0)

class GuideRenderer(Renderer):
    plot = Instance(Plot, has_ref=True)

    def __init__(self, **kwargs):
        super(GuideRenderer, self).__init__(**kwargs)

        if self.plot is not None:
            if self not in self.plot.renderers:
                self.plot.renderers.append(self)

class Axis(GuideRenderer):
    type = String("axis")

    dimension = Int(0)
    location = Either(String('min'), Float)
    bounds = Either(Enum('auto'), Tuple) # XXX: Tuple(Float, Float)

    axis_label = String
    axis_label_standoff = Int
    axis_label_props = Include(TextProps, prefix="axis_label")

    major_label_standoff = Int
    major_label_orientation = Either(Enum("horizontal", "vertical"), Int)
    major_label_props = Include(TextProps, prefix="major_label")

    # Line props
    axis_props = Include(LineProps, prefix="axis")
    tick_props = Include(LineProps, prefix="major_tick")

    major_tick_in = Int
    major_tick_out = Int

class LinearAxis(Axis):
    type = String("linear_axis")

class CategoricalAxis(Axis):
    type = String("categorical_axis")

class DatetimeAxis(LinearAxis):
    type = String("datetime_axis")
    axis_label = String("date")
    scale = String("time")
    num_labels = Int(8)
    char_width = Int(10)
    fill_ratio = Float(0.3)
    formats = Dict({"days": ["%m/%d/%Y"]})

class Grid(GuideRenderer):
    """ 1D Grid component """
    type = String("grid")

    dimension = Int(0)
    bounds = String('auto')

    is_datetime = Bool(False)

    # Line props
    grid_props = Include(LineProps, prefix="grid")

class Tool(PlotObject):
    plot = Instance(Plot, has_ref=True)

class PanTool(Tool):
    dimensions = List(Enum(Dimension))

class WheelZoomTool(Tool):
    dimensions = List(Enum(Dimension))

class PreviewSaveTool(Tool):
    pass

class EmbedTool(Tool):
    pass

class ResetTool(Tool):
    pass

class ResizeTool(Tool):
    pass

class CrosshairTool(Tool):
    pass

class BoxZoomTool(Tool):
    pass

class BoxSelectTool(Tool):
    renderers = List(Instance(Renderer), has_ref=True)
    select_every_mousemove = Bool(True)

class BoxSelectionOverlay(Renderer):
    __view_model__ = 'BoxSelection'
    tool = Instance(Tool, has_ref=True)

class HoverTool(Tool):
    renderers = List(Instance(Renderer), has_ref=True)
    tooltips = Dict()

class ObjectExplorerTool(Tool):
    pass

class Legend(PlotObject):
    plot = Instance(Plot, has_ref=True)
    orientation = Enum(Orientation)
    border = Include(LineProps, prefix="border")

    label_props = Include(TextProps, prefix="label")
    label_standoff = Int(15)
    label_height = Int(20)
    label_width = Int(50)
    legend_padding = Int(10)

    glyph_height = Int(20)
    glyph_width = Int(20)
    legend_spacing = Int(3)
    legends = Dict()

class DataSlider(PlotObject):
    plot = Instance(Plot, has_ref=True)
    data_source = Instance(DataSource, has_ref=True)
    field = String()

class DataRangeBoxSelectTool(PlotObject):
    plot = Instance(Plot, has_ref=True)
    xselect = List(Instance(Range), has_ref=True)
    yselect = List(Instance(Range), has_ref=True)

class PlotContext(PlotObject):
    children = List(List(Instance(Plot), has_ref=True), has_ref=True)

class PlotList(PlotContext):
    # just like plot context, except plot context has special meaning
    # everywhere, so plotlist is the generic one
    pass
<|endoftext|>


<commit_before>from datetime import datetime, timedelta
from dateutil import parser as date_parser
import pytest
import pytz
from web_test_base import *

class TestIATIDashboard(WebTestBase):
    requests_to_load = {
        'Dashboard Homepage': {
            'url': 'http://dashboard.iatistandard.org/'
        }
    }

    def test_contains_links(self, loaded_request):
        """
        Test that each page contains links to the defined URLs.
        """
        result = utility.get_links_from_page(loaded_request)

        assert "https://github.com/IATI/IATI-Dashboard/" in result

    def test_recently_generated(self, loaded_request):
        """
        Tests that the dashboard was generated in the past 7 days.
        """
        max_delay = timedelta(days=2)
        generation_time_xpath = '//*[@id="footer"]/div/p/em[1]'
        data_time_xpath = '//*[@id="footer"]/div/p/em[2]'

        generation_time_arr = utility.get_text_from_xpath(loaded_request, generation_time_xpath)
        data_time_arr = utility.get_text_from_xpath(loaded_request, data_time_xpath)

        generation_time = date_parser.parse(generation_time_arr[0])
        data_time = date_parser.parse(data_time_arr[0])
        now = datetime.now(pytz.utc)

        assert len(generation_time_arr) == 1
        assert len(data_time_arr) == 1
        assert (now - max_delay) < generation_time
        assert (now - max_delay) < data_time
<commit_msg>Mark failing dashboard test as xfail<commit_after>from datetime import datetime, timedelta
from dateutil import parser as date_parser
import pytest
import pytz
from web_test_base import *

class TestIATIDashboard(WebTestBase):
    requests_to_load = {
        'Dashboard Homepage': {
            'url': 'http://dashboard.iatistandard.org/'
        }
    }

    def test_contains_links(self, loaded_request):
        """
        Test that each page contains links to the defined URLs.
        """
        result = utility.get_links_from_page(loaded_request)

        assert "https://github.com/IATI/IATI-Dashboard/" in result

    @pytest.mark.xfail
    def test_recently_generated(self, loaded_request):
        """
        Tests that the dashboard was generated in the past 7 days.
        """
        max_delay = timedelta(days=2)
        generation_time_xpath = '//*[@id="footer"]/div/p/em[1]'
        data_time_xpath = '//*[@id="footer"]/div/p/em[2]'

        generation_time_arr = utility.get_text_from_xpath(loaded_request, generation_time_xpath)
        data_time_arr = utility.get_text_from_xpath(loaded_request, data_time_xpath)

        generation_time = date_parser.parse(generation_time_arr[0])
        data_time = date_parser.parse(data_time_arr[0])
        now = datetime.now(pytz.utc)

        assert len(generation_time_arr) == 1
        assert len(data_time_arr) == 1
        assert (now - max_delay) < generation_time
        assert (now - max_delay) < data_time
<|endoftext|>


<commit_before>from glob import glob
from importlib import import_module
from os.path import basename
from os.path import dirname
from os.path import join
from os.path import splitext
import sys

import pytest

EXAMPLES_DIR = join(dirname(dirname(dirname(__file__))), 'examples')

def _example_modules():
    paths = glob(join(EXAMPLES_DIR, "*py"))
    return [splitext(basename(path))[0] for path in paths]

@pytest.fixture
def example(request):
    sys.path.insert(0, EXAMPLES_DIR)
    request.addfinalizer(lambda: sys.path.remove(EXAMPLES_DIR))

@pytest.mark.slow
@pytest.mark.parametrize("module_name", _example_modules())
def test_example(example, module_name):
    try:
        main = getattr(import_module(module_name), 'main')
        main(1)  # run the example for one iteration
    except ImportError as e:  # some examples require pylearn2
        if "pylearn2" in str(e):
            pytest.skip(e)
        else:
            raise
<commit_msg>Ignore errors caused by cudnn absence in tests.<commit_after>from glob import glob
from importlib import import_module
from os.path import basename
from os.path import dirname
from os.path import join
from os.path import splitext
import sys

import pytest

EXAMPLES_DIR = join(dirname(dirname(dirname(__file__))), 'examples')

def _example_modules():
    paths = glob(join(EXAMPLES_DIR, "*py"))
    return [splitext(basename(path))[0] for path in paths]

@pytest.fixture
def example(request):
    sys.path.insert(0, EXAMPLES_DIR)
    request.addfinalizer(lambda: sys.path.remove(EXAMPLES_DIR))

@pytest.mark.slow
@pytest.mark.parametrize("module_name", _example_modules())
def test_example(example, module_name):
    try:
        main = getattr(import_module(module_name), 'main')
        main(1)  # run the example for one iteration
    except ImportError as e:  # some examples require pylearn2
        if "pylearn2" in str(e):
            pytest.skip(e)
        else:
            raise
    except RuntimeError as e: # ignore errors caused by cudnn absence
        if "cudnn is not available" in str(e):
            pytest.skip(e)
        else:
            raise
<|endoftext|>


<commit_before>#  Author:
#  Rudiger Birkner (Networked Systems Group ETH Zurich)

from ryu.ofproto import ether
from ryu.ofproto import inet

from ofdpa20 import OFDPA20

class FlowMod(object):
    def __init__(self, config, origin, flow_mod):
        self.mod_types = ["insert", "remove"]
        self.rule_types = ["inbound", "outbound", "main", "main-in", "main-out", "arp", "load-balancer", "umbrella-edge", "umbrella-core"]

        self.config = config
        self.parser = None

        self.mod_type = None
        self.rule_type = None

        self.origin = origin
        self.datapath = None
        self.table = None
        self.priority = None
        self.cookie = {}
        self.matches = {}
        self.actions = []

        if self.config.ofdpa:
            self.ofdpa = OFDPA20(config);

        self.validate_flow_mod(flow_mod)

    def get_flow_mod_msg(self):
        return self.flow_mod

    def validate_flow_mod(self, flow_mod):
        if "cookie" in flow_mod:
            if len(flow_mod["cookie"]) > 1:
                self.cookie["cookie"] = int('{0:016b}'.format(int(self.origin))+'{0:016b}'.format(int(flow_mod["cookie"][0])),2)
                self.cookie["mask"] = int('{0:016b}'.format(2**16-1) + '{0:016b}'.format(flow_mod["cookie"][1]),2)
            else:
                self.cookie["cookie"] = int('{0:016b}'.format(int(self.origin))+'{0:016b}'.format(int(flow_mod["cookie"])),2)
                self.cookie["mask"] = 2**32-1
            if ("mod_type" in flow_mod and flow_mod["mod_type"] in self.mod_types):
                self.mod_type = flow_mod["mod_type"]
                if ("rule_type" in flow_mod and flow_mod["rule_type"] in self.rule_types):
                    if flow_mod["rule_type"] in self.config.dp_alias:
                        self.rule_type = self.config.dp_alias[flow_mod["rule_type"]]
                    else:
                        self.rule_type = flow_mod["rule_type"]
                    datapath = flow_mod["datapath"]
                    if datapath in self.config.dpids:
                        self.datapath = datapath
                    if ("priority" in flow_mod):
                        self.priority = flow_mod["priority"]
                        if "match" in flow_mod:
                            self.matches = self.validate_match(flow_mod["match"])
                        if "action" in flow_mod:
                            self.actions = self.validate_action(flow_mod["action"])

    def validate_match(self, matches):
        validated_matches = {}

        for match, value in matches.iteritems():
            if match == "eth_type":
                validated_matches[match] = value
            elif match == "arp_tpa":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_ARP
            elif match == "in_port":
                if isinstance( value, int ) or value.isdigit():
                    validated_matches["in_port"] = value
                else:
                    if self.config.tables:
                        if self.rule_type == "arp" and value in self.config.datapath_ports[self.rule_type]:
                            validated_matches["in_port"] = self.config.datapath_ports[self.rule_type][value]
                        else:
                            validated_matches["in_port"] = self.config.datapath_ports["main"][value]
                    else:
                        if self.rule_type in self.config.datapath_ports and value in self.config.datapath_ports[self.rule_type]:
                            validated_matches["in_port"] = self.config.datapath_ports[self.rule_type][value]
            elif match == "eth_dst":
                if len(value) > 1:
                    validated_matches[match] = value
            elif match == "eth_src":
                if len(value) > 1:
                    validated_matches[match] = value
            elif match == "ipv4_src":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
            elif match == "ipv4_dst":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
            elif match == "tcp_src":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
                if "ip_proto" not in validated_matches:
                    validated_matches["ip_proto"] = inet.IPPROTO_TCP
            elif match == "tcp_dst":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
                if "ip_proto" not in validated_matches:
                    validated_matches["ip_proto"] = inet.IPPROTO_TCP
            elif match == "udp_src":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
                if "ip_proto" not in validated_matches:
                    validated_matches["ip_proto"] = inet.IPPROTO_UDP
            elif match == "udp_dst":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
                if "ip_proto" not in validated_matches:
                    validated_matches["ip_proto"] = inet.IPPROTO_UDP
        return validated_matches

    def make_instructions(self):
        temp_instructions = []
        # merge goto an meta data instructions
        temp_additional_instructions = []
        temp_fwd_actions = []
        temp_actions = []
        temp_meta = []
        for action, value in self.actions.iteritems():
            if action == "fwd":
                if self.config.tables:
                    for port in value:
                        if isinstance( port, int ) or port.isdigit():
                            temp_fwd_actions.append(self.parser.OFPActionOutput(int(port)))
                        elif port in self.config.tables:
                            temp_additional_instructions.append(self.parser.OFPInstructionGotoTable(self.config.tables[port]))
                        elif port in self.config.datapath_ports["main"]:
                            temp_fwd_actions.append(self.parser.OFPActionOutput(self.config.datapath_ports["main"][port]))
                        elif port in self.config.datapath_ports["arp"]:
                            temp_fwd_actions.append(self.parser.OFPActionOutput(self.config.datapath_ports["arp"][port]))
                else:
                    for port in value:
                        if isinstance( port, int ) or port.isdigit():
                            temp_fwd_actions.append(self.parser.OFPActionOutput(int(port)))
                        else:
                            if port in self.config.dp_alias:
                                port = self.config.dp_alias[port]
                            temp_fwd_actions.append(self.parser.OFPActionOutput(self.config.datapath_ports[self.rule_type][port]))
            elif action == "set_eth_src":
                temp_actions.append(self.parser.OFPActionSetField(eth_src=value))
            elif action == "set_eth_dst":
                temp_actions.append(self.parser.OFPActionSetField(eth_dst=value))
            elif action == "meta":
                temp_additional_instructions.append(self.parser.OFPInstructionWriteMetadata(value[0], value[1]))
            elif action == "goto":
                tables = self.config.tables
                umbrella_edge_table = tables[value]
                temp_additional_instructions.append(self.parser.OFPInstructionGotoTable(umbrella_edge_table))

        if temp_fwd_actions:
            temp_actions.extend(temp_fwd_actions)

        if temp_actions:
            temp_instructions = [self.parser.OFPInstructionActions(self.config.ofproto.OFPIT_APPLY_ACTIONS, temp_actions)]

        if len(temp_additional_instructions) > 0:
            temp_instructions.extend(temp_additional_instructions)

        return temp_instructions

    def validate_action(self, actions):
        validated_actions = {}

        for action, value in actions.iteritems():
            if action == "fwd":
                temp_fwds = []
                for val in value:
                    if isinstance( val, int ) or val.isdigit():
                        temp_fwds.append(int(val))
                    elif val in self.rule_types:
                        temp_fwds.append(val)
                validated_actions[action] = temp_fwds
            elif action == "set_eth_src":
                validated_actions[action] = value
            elif action == "set_eth_dst":
                validated_actions[action] = value
            elif action == "meta":
                validated_actions[action] = value
            elif action == "goto":
                validated_actions[action] = value
        return validated_actions

    def get_flow_mod(self, config):
        flow_mod, _ = self.get_flow_and_group_mods(config)
        return flow_mod

    def get_flow_and_group_mods(self, config):
        self.config = config
        self.parser = config.parser
        group_mods = []
        match = self.parser.OFPMatch(**self.matches)
        if self.config.tables:
            if self.rule_type == "arp":
                table_id = 0
                datapath = self.config.datapaths["arp"]
            else:
                if config.isMultiHopMode():
                    datapath = self.config.datapaths[self.datapath]
                    table_id = self.config.tables[self.rule_type]
                else:
                    table_id = self.config.tables[self.rule_type]
                    datapath = self.config.datapaths["main"]
        else:
            datapath = self.config.datapaths[self.rule_type]
            table_id = self.ofdpa.get_table_id() if self.is_ofdpa_datapath(datapath) else 0

        if self.mod_type == "insert":
            if self.is_ofdpa_datapath(datapath):
                instructions, group_mods = self.ofdpa.make_instructions_and_group_mods(self, datapath)
            else:
                instructions = self.make_instructions()
            flow_mod = self.parser.OFPFlowMod(datapath=datapath,
                                          cookie=self.cookie["cookie"], cookie_mask=self.cookie["mask"],
                                          table_id=table_id,
                                          command=self.config.ofproto.OFPFC_ADD,
                                          priority=self.priority,
                                          match=match, instructions=instructions)
        else:
            flow_mod = self.parser.OFPFlowMod(datapath=datapath,
                                          cookie=self.cookie["cookie"], cookie_mask=self.cookie["mask"],
                                          table_id=table_id,
                                          command=self.config.ofproto.OFPFC_DELETE,
                                          out_group=self.config.ofproto.OFPG_ANY,
                                          out_port=self.config.ofproto.OFPP_ANY,
                                          match=match)
        return flow_mod, group_mods

    def get_dst_dp(self):
        if self.config.isMultiHopMode():
            return self.datapath
        #TODO: For now leave previous logic, so we do not need
        # to modify previous code
        if self.config.tables and self.rule_type != "arp":
            return "main"
        return self.rule_type

    def is_ofdpa_datapath(self, datapath):
        return self.config.ofdpa and self.config.dpid_2_name[datapath.id] in self.config.ofdpa
<commit_msg>add debug information for goto table mit "fwd" action<commit_after>#  Author:
#  Rudiger Birkner (Networked Systems Group ETH Zurich)

from ryu.ofproto import ether
from ryu.ofproto import inet

from ofdpa20 import OFDPA20

class FlowMod(object):
    def __init__(self, config, origin, flow_mod):
        self.mod_types = ["insert", "remove"]
        self.rule_types = ["inbound", "outbound", "main", "main-in", "main-out", "arp", "load-balancer", "umbrella-edge", "umbrella-core"]

        self.config = config
        self.parser = None

        self.mod_type = None
        self.rule_type = None

        self.origin = origin
        self.datapath = None
        self.table = None
        self.priority = None
        self.cookie = {}
        self.matches = {}
        self.actions = []

        if self.config.ofdpa:
            self.ofdpa = OFDPA20(config);

        self.validate_flow_mod(flow_mod)

    def get_flow_mod_msg(self):
        return self.flow_mod

    def validate_flow_mod(self, flow_mod):
        if "cookie" in flow_mod:
            if len(flow_mod["cookie"]) > 1:
                self.cookie["cookie"] = int('{0:016b}'.format(int(self.origin))+'{0:016b}'.format(int(flow_mod["cookie"][0])),2)
                self.cookie["mask"] = int('{0:016b}'.format(2**16-1) + '{0:016b}'.format(flow_mod["cookie"][1]),2)
            else:
                self.cookie["cookie"] = int('{0:016b}'.format(int(self.origin))+'{0:016b}'.format(int(flow_mod["cookie"])),2)
                self.cookie["mask"] = 2**32-1
            if ("mod_type" in flow_mod and flow_mod["mod_type"] in self.mod_types):
                self.mod_type = flow_mod["mod_type"]
                if ("rule_type" in flow_mod and flow_mod["rule_type"] in self.rule_types):
                    if flow_mod["rule_type"] in self.config.dp_alias:
                        self.rule_type = self.config.dp_alias[flow_mod["rule_type"]]
                    else:
                        self.rule_type = flow_mod["rule_type"]
                    datapath = flow_mod["datapath"]
                    if datapath in self.config.dpids:
                        self.datapath = datapath
                    if ("priority" in flow_mod):
                        self.priority = flow_mod["priority"]
                        if "match" in flow_mod:
                            self.matches = self.validate_match(flow_mod["match"])
                        if "action" in flow_mod:
                            self.actions = self.validate_action(flow_mod["action"])

    def validate_match(self, matches):
        validated_matches = {}

        for match, value in matches.iteritems():
            if match == "eth_type":
                validated_matches[match] = value
            elif match == "arp_tpa":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_ARP
            elif match == "in_port":
                if isinstance( value, int ) or value.isdigit():
                    validated_matches["in_port"] = value
                else:
                    if self.config.tables:
                        if self.rule_type == "arp" and value in self.config.datapath_ports[self.rule_type]:
                            validated_matches["in_port"] = self.config.datapath_ports[self.rule_type][value]
                        else:
                            validated_matches["in_port"] = self.config.datapath_ports["main"][value]
                    else:
                        if self.rule_type in self.config.datapath_ports and value in self.config.datapath_ports[self.rule_type]:
                            validated_matches["in_port"] = self.config.datapath_ports[self.rule_type][value]
            elif match == "eth_dst":
                if len(value) > 1:
                    validated_matches[match] = value
            elif match == "eth_src":
                if len(value) > 1:
                    validated_matches[match] = value
            elif match == "ipv4_src":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
            elif match == "ipv4_dst":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
            elif match == "tcp_src":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
                if "ip_proto" not in validated_matches:
                    validated_matches["ip_proto"] = inet.IPPROTO_TCP
            elif match == "tcp_dst":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
                if "ip_proto" not in validated_matches:
                    validated_matches["ip_proto"] = inet.IPPROTO_TCP
            elif match == "udp_src":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
                if "ip_proto" not in validated_matches:
                    validated_matches["ip_proto"] = inet.IPPROTO_UDP
            elif match == "udp_dst":
                validated_matches[match] = value
                if "eth_type" not in validated_matches:
                    validated_matches["eth_type"] = ether.ETH_TYPE_IP
                if "ip_proto" not in validated_matches:
                    validated_matches["ip_proto"] = inet.IPPROTO_UDP
        return validated_matches

    def make_instructions(self):
        temp_instructions = []
        # merge goto an meta data instructions
        temp_additional_instructions = []
        temp_fwd_actions = []
        temp_actions = []
        temp_meta = []
        for action, value in self.actions.iteritems():
            if action == "fwd":
                if self.config.tables:
                    for port in value:
                        if isinstance( port, int ) or port.isdigit():
                            temp_fwd_actions.append(self.parser.OFPActionOutput(int(port)))
                        elif port in self.config.tables:
                            temp_additional_instructions.append(self.parser.OFPInstructionGotoTable(self.config.tables[port]))
                            print ("port: %s action: %s value: %s") % (port, action, value)
                        elif port in self.config.datapath_ports["main"]:
                            temp_fwd_actions.append(self.parser.OFPActionOutput(self.config.datapath_ports["main"][port]))
                        elif port in self.config.datapath_ports["arp"]:
                            temp_fwd_actions.append(self.parser.OFPActionOutput(self.config.datapath_ports["arp"][port]))
                else:
                    for port in value:
                        if isinstance( port, int ) or port.isdigit():
                            temp_fwd_actions.append(self.parser.OFPActionOutput(int(port)))
                        else:
                            if port in self.config.dp_alias:
                                port = self.config.dp_alias[port]
                            temp_fwd_actions.append(self.parser.OFPActionOutput(self.config.datapath_ports[self.rule_type][port]))
            elif action == "set_eth_src":
                temp_actions.append(self.parser.OFPActionSetField(eth_src=value))
            elif action == "set_eth_dst":
                temp_actions.append(self.parser.OFPActionSetField(eth_dst=value))
            elif action == "meta":
                temp_additional_instructions.append(self.parser.OFPInstructionWriteMetadata(value[0], value[1]))
            elif action == "goto":
                tables = self.config.tables
                umbrella_edge_table = tables[value]
                temp_additional_instructions.append(self.parser.OFPInstructionGotoTable(umbrella_edge_table))

        if temp_fwd_actions:
            temp_actions.extend(temp_fwd_actions)

        if temp_actions:
            temp_instructions = [self.parser.OFPInstructionActions(self.config.ofproto.OFPIT_APPLY_ACTIONS, temp_actions)]

        if len(temp_additional_instructions) > 0:
            temp_instructions.extend(temp_additional_instructions)

        return temp_instructions

    def validate_action(self, actions):
        validated_actions = {}

        for action, value in actions.iteritems():
            if action == "fwd":
                temp_fwds = []
                for val in value:
                    if isinstance( val, int ) or val.isdigit():
                        temp_fwds.append(int(val))
                    elif val in self.rule_types:
                        temp_fwds.append(val)
                validated_actions[action] = temp_fwds
            elif action == "set_eth_src":
                validated_actions[action] = value
            elif action == "set_eth_dst":
                validated_actions[action] = value
            elif action == "meta":
                validated_actions[action] = value
            elif action == "goto":
                validated_actions[action] = value
        return validated_actions

    def get_flow_mod(self, config):
        flow_mod, _ = self.get_flow_and_group_mods(config)
        return flow_mod

    def get_flow_and_group_mods(self, config):
        self.config = config
        self.parser = config.parser
        group_mods = []
        match = self.parser.OFPMatch(**self.matches)
        if self.config.tables:
            if self.rule_type == "arp":
                table_id = 0
                datapath = self.config.datapaths["arp"]
            else:
                if config.isMultiHopMode():
                    datapath = self.config.datapaths[self.datapath]
                    table_id = self.config.tables[self.rule_type]
                else:
                    table_id = self.config.tables[self.rule_type]
                    datapath = self.config.datapaths["main"]
        else:
            datapath = self.config.datapaths[self.rule_type]
            table_id = self.ofdpa.get_table_id() if self.is_ofdpa_datapath(datapath) else 0

        if self.mod_type == "insert":
            if self.is_ofdpa_datapath(datapath):
                instructions, group_mods = self.ofdpa.make_instructions_and_group_mods(self, datapath)
            else:
                instructions = self.make_instructions()
            flow_mod = self.parser.OFPFlowMod(datapath=datapath,
                                          cookie=self.cookie["cookie"], cookie_mask=self.cookie["mask"],
                                          table_id=table_id,
                                          command=self.config.ofproto.OFPFC_ADD,
                                          priority=self.priority,
                                          match=match, instructions=instructions)
        else:
            flow_mod = self.parser.OFPFlowMod(datapath=datapath,
                                          cookie=self.cookie["cookie"], cookie_mask=self.cookie["mask"],
                                          table_id=table_id,
                                          command=self.config.ofproto.OFPFC_DELETE,
                                          out_group=self.config.ofproto.OFPG_ANY,
                                          out_port=self.config.ofproto.OFPP_ANY,
                                          match=match)
        return flow_mod, group_mods

    def get_dst_dp(self):
        if self.config.isMultiHopMode():
            return self.datapath
        #TODO: For now leave previous logic, so we do not need
        # to modify previous code
        if self.config.tables and self.rule_type != "arp":
            return "main"
        return self.rule_type

    def is_ofdpa_datapath(self, datapath):
        return self.config.ofdpa and self.config.dpid_2_name[datapath.id] in self.config.ofdpa
<|endoftext|>