<commit_before># Copyright 2015 Futurewei.  All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import netaddr
import six

from oslo_log import helpers as log_helpers
from oslo_log import log as logging
from oslo_utils import uuidutils

import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.orm import exc

from neutron_lib import constants as const

from neutron.db import common_db_mixin
from neutron.db import model_base
from neutron.db import models_v2

from networking_sfc._i18n import _LI
from networking_sfc.extensions import flowclassifier as fc_ext

LOG = logging.getLogger(__name__)
UUID_LEN = 36

class L7Parameter(model_base.BASEV2):
    """Represents a L7 parameter."""
    __tablename__ = 'sfc_flow_classifier_l7_parameters'
    keyword = sa.Column(sa.String(255), primary_key=True)
    value = sa.Column(sa.String(255))
    classifier_id = sa.Column(
        sa.String(36),
        sa.ForeignKey('sfc_flow_classifiers.id', ondelete='CASCADE'),
        primary_key=True)

class FlowClassifier(model_base.BASEV2, models_v2.HasId,
                     models_v2.HasTenant):
    """Represents a v2 neutron flow classifier."""
    __tablename__ = 'sfc_flow_classifiers'
    name = sa.Column(sa.String(255))
    ethertype = sa.Column(sa.String(40))
    protocol = sa.Column(sa.String(40))
    description = sa.Column(sa.String(255))
    source_port_range_min = sa.Column(sa.Integer)
    source_port_range_max = sa.Column(sa.Integer)
    destination_port_range_min = sa.Column(sa.Integer)
    destination_port_range_max = sa.Column(sa.Integer)
    source_ip_prefix = sa.Column(sa.String(255))
    destination_ip_prefix = sa.Column(sa.String(255))
    logical_source_port = sa.Column(
        sa.String(UUID_LEN),
        sa.ForeignKey('ports.id', ondelete='RESTRICT'))
    logical_destination_port = sa.Column(
        sa.String(UUID_LEN),
        sa.ForeignKey('ports.id', ondelete='RESTRICT'))
    l7_parameters = orm.relationship(
        L7Parameter,
        collection_class=attribute_mapped_collection('keyword'),
        cascade='all, delete-orphan')

class FlowClassifierDbPlugin(fc_ext.FlowClassifierPluginBase,
                             common_db_mixin.CommonDbMixin):

    @classmethod
    def _check_port_range_valid(cls, port_range_min,
                                port_range_max,
                                protocol):
        if (
            port_range_min is not None and
            port_range_max is not None and
            port_range_min > port_range_max
        ):
            raise fc_ext.FlowClassifierInvalidPortRange(
                port_range_min=port_range_min,
                port_range_max=port_range_max
            )
        if port_range_min is not None or port_range_max is not None:
            if protocol not in [const.PROTO_NAME_TCP, const.PROTO_NAME_UDP]:
                raise fc_ext.FlowClassifierProtocolRequiredWithPorts()

    @classmethod
    def _check_ip_prefix_valid(cls, ip_prefix, ethertype):
        if ip_prefix is not None:
            ip = netaddr.IPNetwork(ip_prefix)
            if ethertype == 'IPv4' and ip.version == 4:
                pass
            elif ethertype == 'IPv6' and ip.version == 6:
                pass
            else:
                raise (
                    fc_ext.FlowClassifierIpPrefixFormatConflictWithEthertype(
                        ip_prefix=ip_prefix, ethertype=ethertype
                    )
                )

    @classmethod
    def _logical_port_conflict(cls, first_logical_port, second_logical_port):
        if first_logical_port is None or second_logical_port is None:
            return True
        return first_logical_port == second_logical_port

    @classmethod
    def _ip_prefix_conflict(cls, first_ip_prefix, second_ip_prefix):
        if first_ip_prefix is None or second_ip_prefix is None:
            return True
        first_ipset = netaddr.IPSet([first_ip_prefix])
        second_ipset = netaddr.IPSet([second_ip_prefix])
        return bool(first_ipset & second_ipset)

    @classmethod
    def _port_range_conflict(
        cls, first_port_range_min, first_port_range_max,
        second_port_range_min, second_port_range_max
    ):
        first_conflict = True
        second_conflict = True
        if (
            first_port_range_min is not None and
            second_port_range_max is not None
        ):
            first_conflict = first_port_range_min <= second_port_range_max
        if (
            first_port_range_max is not None and
            second_port_range_min is not None
        ):
            second_conflict = second_port_range_min <= first_port_range_max
        return first_conflict & second_conflict

    @classmethod
    def _protocol_conflict(cls, first_protocol, second_protocol):
        if first_protocol is None or second_protocol is None:
            return True
        return first_protocol == second_protocol

    @classmethod
    def _ethertype_conflict(cls, first_ethertype, second_ethertype):
        return first_ethertype == second_ethertype

    @classmethod
    def flowclassifier_basic_conflict(
        cls, first_flowclassifier, second_flowclassifier
    ):
        return all([
            cls._ethertype_conflict(
                first_flowclassifier['ethertype'],
                second_flowclassifier['ethertype']
            ),
            cls._protocol_conflict(
                first_flowclassifier['protocol'],
                second_flowclassifier['protocol']
            ),
            cls._ip_prefix_conflict(
                first_flowclassifier['source_ip_prefix'],
                second_flowclassifier['source_ip_prefix']
            ),
            cls._ip_prefix_conflict(
                first_flowclassifier['destination_ip_prefix'],
                second_flowclassifier['destination_ip_prefix']
            ),
            cls._port_range_conflict(
                first_flowclassifier['source_port_range_min'],
                first_flowclassifier['source_port_range_max'],
                second_flowclassifier['source_port_range_min'],
                second_flowclassifier['source_port_range_max']
            ),
            cls._port_range_conflict(
                first_flowclassifier['destination_port_range_min'],
                first_flowclassifier['destination_port_range_max'],
                second_flowclassifier['destination_port_range_min'],
                second_flowclassifier['destination_port_range_max']
            )
        ])

    @classmethod
    def flowclassifier_conflict(
        cls, first_flowclassifier, second_flowclassifier
    ):
        return all([
            cls.flowclassifier_basic_conflict(
                first_flowclassifier,
                second_flowclassifier
            ),
            cls._logical_port_conflict(
                first_flowclassifier['logical_source_port'],
                second_flowclassifier['logical_source_port']
            ),
            cls._logical_port_conflict(
                first_flowclassifier['logical_destination_port'],
                second_flowclassifier['logical_destination_port']
            )
        ])

    @log_helpers.log_method_call
    def create_flow_classifier(self, context, flow_classifier):
        fc = flow_classifier['flow_classifier']
        tenant_id = fc['tenant_id']
        l7_parameters = {
            key: L7Parameter(key, val)
            for key, val in six.iteritems(fc['l7_parameters'])}
        ethertype = fc['ethertype']
        protocol = fc['protocol']
        source_port_range_min = fc['source_port_range_min']
        source_port_range_max = fc['source_port_range_max']
        self._check_port_range_valid(source_port_range_min,
                                     source_port_range_max,
                                     protocol)
        destination_port_range_min = fc['destination_port_range_min']
        destination_port_range_max = fc['destination_port_range_max']
        self._check_port_range_valid(destination_port_range_min,
                                     destination_port_range_max,
                                     protocol)
        source_ip_prefix = fc['source_ip_prefix']
        self._check_ip_prefix_valid(source_ip_prefix, ethertype)
        destination_ip_prefix = fc['destination_ip_prefix']
        self._check_ip_prefix_valid(destination_ip_prefix, ethertype)
        logical_source_port = fc['logical_source_port']
        logical_destination_port = fc['logical_destination_port']
        with context.session.begin(subtransactions=True):
            if logical_source_port is not None:
                self._get_port(context, logical_source_port)
            if logical_destination_port is not None:
                self._get_port(context, logical_destination_port)
            query = self._model_query(context, FlowClassifier)
            for flow_classifier_db in query.all():
                if self.flowclassifier_conflict(
                    fc,
                    flow_classifier_db
                ):
                    raise fc_ext.FlowClassifierInConflict(
                        id=flow_classifier_db['id']
                    )
            flow_classifier_db = FlowClassifier(
                id=uuidutils.generate_uuid(),
                tenant_id=tenant_id,
                name=fc['name'],
                description=fc['description'],
                ethertype=ethertype,
                protocol=protocol,
                source_port_range_min=source_port_range_min,
                source_port_range_max=source_port_range_max,
                destination_port_range_min=destination_port_range_min,
                destination_port_range_max=destination_port_range_max,
                source_ip_prefix=source_ip_prefix,
                destination_ip_prefix=destination_ip_prefix,
                logical_source_port=logical_source_port,
                logical_destination_port=logical_destination_port,
                l7_parameters=l7_parameters
            )
            context.session.add(flow_classifier_db)
            return self._make_flow_classifier_dict(flow_classifier_db)

    def _make_flow_classifier_dict(self, flow_classifier, fields=None):
        res = {
            'id': flow_classifier['id'],
            'name': flow_classifier['name'],
            'description': flow_classifier['description'],
            'tenant_id': flow_classifier['tenant_id'],
            'ethertype': flow_classifier['ethertype'],
            'protocol': flow_classifier['protocol'],
            'source_port_range_min': flow_classifier['source_port_range_min'],
            'source_port_range_max': flow_classifier['source_port_range_max'],
            'destination_port_range_min': (
                flow_classifier['destination_port_range_min']),
            'destination_port_range_max': (
                flow_classifier['destination_port_range_max']),
            'source_ip_prefix': flow_classifier['source_ip_prefix'],
            'destination_ip_prefix': flow_classifier[
                'destination_ip_prefix'],
            'logical_source_port': flow_classifier['logical_source_port'],
            'logical_destination_port': flow_classifier[
                'logical_destination_port'],
            'l7_parameters': {
                param['keyword']: param['value']
                for k, param in six.iteritems(flow_classifier.l7_parameters)
            }
        }
        return self._fields(res, fields)

    @log_helpers.log_method_call
    def get_flow_classifiers(self, context, filters=None, fields=None,
                             sorts=None, limit=None, marker=None,
                             page_reverse=False):
        marker_obj = self._get_marker_obj(context, 'flow_classifier',
                                          limit, marker)
        return self._get_collection(context,
                                    FlowClassifier,
                                    self._make_flow_classifier_dict,
                                    filters=filters, fields=fields,
                                    sorts=sorts,
                                    limit=limit, marker_obj=marker_obj,
                                    page_reverse=page_reverse)

    @log_helpers.log_method_call
    def get_flow_classifier(self, context, id, fields=None):
        flow_classifier = self._get_flow_classifier(context, id)
        return self._make_flow_classifier_dict(flow_classifier, fields)

    def _get_flow_classifier(self, context, id):
        try:
            return self._get_by_id(context, FlowClassifier, id)
        except exc.NoResultFound:
            raise fc_ext.FlowClassifierNotFound(id=id)

    def _get_port(self, context, id):
        try:
            return self._get_by_id(context, models_v2.Port, id)
        except exc.NoResultFound:
            raise fc_ext.FlowClassifierPortNotFound(id=id)

    @log_helpers.log_method_call
    def update_flow_classifier(self, context, id, flow_classifier):
        new_fc = flow_classifier['flow_classifier']
        with context.session.begin(subtransactions=True):
            old_fc = self._get_flow_classifier(context, id)
            old_fc.update(new_fc)
            return self._make_flow_classifier_dict(old_fc)

    @log_helpers.log_method_call
    def delete_flow_classifier(self, context, id):
        try:
            with context.session.begin(subtransactions=True):
                fc = self._get_flow_classifier(context, id)
                context.session.delete(fc)
        except AssertionError:
            raise fc_ext.FlowClassifierInUse(id=id)
        except fc_ext.FlowClassifierNotFound:
            LOG.info(_LI("Deleting a non-existing flow classifier."))
<commit_msg>Use neutron-lib model_base<commit_after># Copyright 2015 Futurewei.  All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import netaddr
import six

from oslo_log import helpers as log_helpers
from oslo_log import log as logging
from oslo_utils import uuidutils

import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.orm import exc

from neutron_lib import constants as const
from neutron_lib.db import model_base

from neutron.db import common_db_mixin
from neutron.db import models_v2

from networking_sfc._i18n import _LI
from networking_sfc.extensions import flowclassifier as fc_ext

LOG = logging.getLogger(__name__)
UUID_LEN = 36

class L7Parameter(model_base.BASEV2):
    """Represents a L7 parameter."""
    __tablename__ = 'sfc_flow_classifier_l7_parameters'
    keyword = sa.Column(sa.String(255), primary_key=True)
    value = sa.Column(sa.String(255))
    classifier_id = sa.Column(
        sa.String(36),
        sa.ForeignKey('sfc_flow_classifiers.id', ondelete='CASCADE'),
        primary_key=True)

class FlowClassifier(model_base.BASEV2, models_v2.HasId,
                     models_v2.HasTenant):
    """Represents a v2 neutron flow classifier."""
    __tablename__ = 'sfc_flow_classifiers'
    name = sa.Column(sa.String(255))
    ethertype = sa.Column(sa.String(40))
    protocol = sa.Column(sa.String(40))
    description = sa.Column(sa.String(255))
    source_port_range_min = sa.Column(sa.Integer)
    source_port_range_max = sa.Column(sa.Integer)
    destination_port_range_min = sa.Column(sa.Integer)
    destination_port_range_max = sa.Column(sa.Integer)
    source_ip_prefix = sa.Column(sa.String(255))
    destination_ip_prefix = sa.Column(sa.String(255))
    logical_source_port = sa.Column(
        sa.String(UUID_LEN),
        sa.ForeignKey('ports.id', ondelete='RESTRICT'))
    logical_destination_port = sa.Column(
        sa.String(UUID_LEN),
        sa.ForeignKey('ports.id', ondelete='RESTRICT'))
    l7_parameters = orm.relationship(
        L7Parameter,
        collection_class=attribute_mapped_collection('keyword'),
        cascade='all, delete-orphan')

class FlowClassifierDbPlugin(fc_ext.FlowClassifierPluginBase,
                             common_db_mixin.CommonDbMixin):

    @classmethod
    def _check_port_range_valid(cls, port_range_min,
                                port_range_max,
                                protocol):
        if (
            port_range_min is not None and
            port_range_max is not None and
            port_range_min > port_range_max
        ):
            raise fc_ext.FlowClassifierInvalidPortRange(
                port_range_min=port_range_min,
                port_range_max=port_range_max
            )
        if port_range_min is not None or port_range_max is not None:
            if protocol not in [const.PROTO_NAME_TCP, const.PROTO_NAME_UDP]:
                raise fc_ext.FlowClassifierProtocolRequiredWithPorts()

    @classmethod
    def _check_ip_prefix_valid(cls, ip_prefix, ethertype):
        if ip_prefix is not None:
            ip = netaddr.IPNetwork(ip_prefix)
            if ethertype == 'IPv4' and ip.version == 4:
                pass
            elif ethertype == 'IPv6' and ip.version == 6:
                pass
            else:
                raise (
                    fc_ext.FlowClassifierIpPrefixFormatConflictWithEthertype(
                        ip_prefix=ip_prefix, ethertype=ethertype
                    )
                )

    @classmethod
    def _logical_port_conflict(cls, first_logical_port, second_logical_port):
        if first_logical_port is None or second_logical_port is None:
            return True
        return first_logical_port == second_logical_port

    @classmethod
    def _ip_prefix_conflict(cls, first_ip_prefix, second_ip_prefix):
        if first_ip_prefix is None or second_ip_prefix is None:
            return True
        first_ipset = netaddr.IPSet([first_ip_prefix])
        second_ipset = netaddr.IPSet([second_ip_prefix])
        return bool(first_ipset & second_ipset)

    @classmethod
    def _port_range_conflict(
        cls, first_port_range_min, first_port_range_max,
        second_port_range_min, second_port_range_max
    ):
        first_conflict = True
        second_conflict = True
        if (
            first_port_range_min is not None and
            second_port_range_max is not None
        ):
            first_conflict = first_port_range_min <= second_port_range_max
        if (
            first_port_range_max is not None and
            second_port_range_min is not None
        ):
            second_conflict = second_port_range_min <= first_port_range_max
        return first_conflict & second_conflict

    @classmethod
    def _protocol_conflict(cls, first_protocol, second_protocol):
        if first_protocol is None or second_protocol is None:
            return True
        return first_protocol == second_protocol

    @classmethod
    def _ethertype_conflict(cls, first_ethertype, second_ethertype):
        return first_ethertype == second_ethertype

    @classmethod
    def flowclassifier_basic_conflict(
        cls, first_flowclassifier, second_flowclassifier
    ):
        return all([
            cls._ethertype_conflict(
                first_flowclassifier['ethertype'],
                second_flowclassifier['ethertype']
            ),
            cls._protocol_conflict(
                first_flowclassifier['protocol'],
                second_flowclassifier['protocol']
            ),
            cls._ip_prefix_conflict(
                first_flowclassifier['source_ip_prefix'],
                second_flowclassifier['source_ip_prefix']
            ),
            cls._ip_prefix_conflict(
                first_flowclassifier['destination_ip_prefix'],
                second_flowclassifier['destination_ip_prefix']
            ),
            cls._port_range_conflict(
                first_flowclassifier['source_port_range_min'],
                first_flowclassifier['source_port_range_max'],
                second_flowclassifier['source_port_range_min'],
                second_flowclassifier['source_port_range_max']
            ),
            cls._port_range_conflict(
                first_flowclassifier['destination_port_range_min'],
                first_flowclassifier['destination_port_range_max'],
                second_flowclassifier['destination_port_range_min'],
                second_flowclassifier['destination_port_range_max']
            )
        ])

    @classmethod
    def flowclassifier_conflict(
        cls, first_flowclassifier, second_flowclassifier
    ):
        return all([
            cls.flowclassifier_basic_conflict(
                first_flowclassifier,
                second_flowclassifier
            ),
            cls._logical_port_conflict(
                first_flowclassifier['logical_source_port'],
                second_flowclassifier['logical_source_port']
            ),
            cls._logical_port_conflict(
                first_flowclassifier['logical_destination_port'],
                second_flowclassifier['logical_destination_port']
            )
        ])

    @log_helpers.log_method_call
    def create_flow_classifier(self, context, flow_classifier):
        fc = flow_classifier['flow_classifier']
        tenant_id = fc['tenant_id']
        l7_parameters = {
            key: L7Parameter(key, val)
            for key, val in six.iteritems(fc['l7_parameters'])}
        ethertype = fc['ethertype']
        protocol = fc['protocol']
        source_port_range_min = fc['source_port_range_min']
        source_port_range_max = fc['source_port_range_max']
        self._check_port_range_valid(source_port_range_min,
                                     source_port_range_max,
                                     protocol)
        destination_port_range_min = fc['destination_port_range_min']
        destination_port_range_max = fc['destination_port_range_max']
        self._check_port_range_valid(destination_port_range_min,
                                     destination_port_range_max,
                                     protocol)
        source_ip_prefix = fc['source_ip_prefix']
        self._check_ip_prefix_valid(source_ip_prefix, ethertype)
        destination_ip_prefix = fc['destination_ip_prefix']
        self._check_ip_prefix_valid(destination_ip_prefix, ethertype)
        logical_source_port = fc['logical_source_port']
        logical_destination_port = fc['logical_destination_port']
        with context.session.begin(subtransactions=True):
            if logical_source_port is not None:
                self._get_port(context, logical_source_port)
            if logical_destination_port is not None:
                self._get_port(context, logical_destination_port)
            query = self._model_query(context, FlowClassifier)
            for flow_classifier_db in query.all():
                if self.flowclassifier_conflict(
                    fc,
                    flow_classifier_db
                ):
                    raise fc_ext.FlowClassifierInConflict(
                        id=flow_classifier_db['id']
                    )
            flow_classifier_db = FlowClassifier(
                id=uuidutils.generate_uuid(),
                tenant_id=tenant_id,
                name=fc['name'],
                description=fc['description'],
                ethertype=ethertype,
                protocol=protocol,
                source_port_range_min=source_port_range_min,
                source_port_range_max=source_port_range_max,
                destination_port_range_min=destination_port_range_min,
                destination_port_range_max=destination_port_range_max,
                source_ip_prefix=source_ip_prefix,
                destination_ip_prefix=destination_ip_prefix,
                logical_source_port=logical_source_port,
                logical_destination_port=logical_destination_port,
                l7_parameters=l7_parameters
            )
            context.session.add(flow_classifier_db)
            return self._make_flow_classifier_dict(flow_classifier_db)

    def _make_flow_classifier_dict(self, flow_classifier, fields=None):
        res = {
            'id': flow_classifier['id'],
            'name': flow_classifier['name'],
            'description': flow_classifier['description'],
            'tenant_id': flow_classifier['tenant_id'],
            'ethertype': flow_classifier['ethertype'],
            'protocol': flow_classifier['protocol'],
            'source_port_range_min': flow_classifier['source_port_range_min'],
            'source_port_range_max': flow_classifier['source_port_range_max'],
            'destination_port_range_min': (
                flow_classifier['destination_port_range_min']),
            'destination_port_range_max': (
                flow_classifier['destination_port_range_max']),
            'source_ip_prefix': flow_classifier['source_ip_prefix'],
            'destination_ip_prefix': flow_classifier[
                'destination_ip_prefix'],
            'logical_source_port': flow_classifier['logical_source_port'],
            'logical_destination_port': flow_classifier[
                'logical_destination_port'],
            'l7_parameters': {
                param['keyword']: param['value']
                for k, param in six.iteritems(flow_classifier.l7_parameters)
            }
        }
        return self._fields(res, fields)

    @log_helpers.log_method_call
    def get_flow_classifiers(self, context, filters=None, fields=None,
                             sorts=None, limit=None, marker=None,
                             page_reverse=False):
        marker_obj = self._get_marker_obj(context, 'flow_classifier',
                                          limit, marker)
        return self._get_collection(context,
                                    FlowClassifier,
                                    self._make_flow_classifier_dict,
                                    filters=filters, fields=fields,
                                    sorts=sorts,
                                    limit=limit, marker_obj=marker_obj,
                                    page_reverse=page_reverse)

    @log_helpers.log_method_call
    def get_flow_classifier(self, context, id, fields=None):
        flow_classifier = self._get_flow_classifier(context, id)
        return self._make_flow_classifier_dict(flow_classifier, fields)

    def _get_flow_classifier(self, context, id):
        try:
            return self._get_by_id(context, FlowClassifier, id)
        except exc.NoResultFound:
            raise fc_ext.FlowClassifierNotFound(id=id)

    def _get_port(self, context, id):
        try:
            return self._get_by_id(context, models_v2.Port, id)
        except exc.NoResultFound:
            raise fc_ext.FlowClassifierPortNotFound(id=id)

    @log_helpers.log_method_call
    def update_flow_classifier(self, context, id, flow_classifier):
        new_fc = flow_classifier['flow_classifier']
        with context.session.begin(subtransactions=True):
            old_fc = self._get_flow_classifier(context, id)
            old_fc.update(new_fc)
            return self._make_flow_classifier_dict(old_fc)

    @log_helpers.log_method_call
    def delete_flow_classifier(self, context, id):
        try:
            with context.session.begin(subtransactions=True):
                fc = self._get_flow_classifier(context, id)
                context.session.delete(fc)
        except AssertionError:
            raise fc_ext.FlowClassifierInUse(id=id)
        except fc_ext.FlowClassifierNotFound:
            LOG.info(_LI("Deleting a non-existing flow classifier."))
<|endoftext|>


<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2020 CERN
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the MIT License; see the
# LICENSE file for more details.

from __future__ import print_function, unicode_literals

import atexit
import os
import subprocess
import sys
import time

import pywatchman
from flask.helpers import get_root_path
from werkzeug._reloader import _find_observable_paths

from indico.util.console import cformat

def _patterns_to_terms(patterns):
    return ['anyof'] + [['match', p, 'wholename', {'includedotfiles': True}] for p in patterns]

def _disable_reloader(argv):
    argv = list(argv)  # we usually pass sys.argv, so let's not modify that
    for i, arg in enumerate(argv):
        if arg == '--reloader' and argv[i + 1] == 'watchman':
            argv[i + 1] = 'none'
        elif arg.startswith('--reloader') and '=' in arg:
            argv[i] = '--reloader=none'
    return argv

class Watcher(object):
    def __init__(self, path, patterns):
        self.path = path
        self.name = path.replace('/', '-').strip('-')
        self.patterns = patterns
        self.root_dir = None
        self.triggered = False

    def start(self, client):
        query = {
            'expression': _patterns_to_terms(self.patterns),
            'fields': ['name']
        }
        watch = client.query('watch-project', self.path)
        if 'warning' in watch:
            # no idea when this happens, but the example scripts have it...
            print('watchman warning:', watch['warning'])
        self.root_dir = watch['watch']
        if 'relative_path' in watch:
            query['relative_root'] = watch['relative_path']
        # get the initial clock value so that we only get updates
        query['since'] = client.query('clock', self.root_dir)['clock']
        client.query('subscribe', self.root_dir, self.name, query)

    def consume(self, client):
        data = client.getSubscription(self.name)
        if data:
            files = sorted(f for record in data for f in record.get('files', []))
            print(cformat('%{cyan}Changes found in %{cyan!}{}%{reset}%{cyan}:').format(os.path.relpath(self.path)))
            for f in files:
                print(cformat(' * %{blue!}{}').format(f))
            self.triggered = True

    def check(self):
        triggered = self.triggered
        self.triggered = False
        return triggered

    def __repr__(self):
        return '<Watcher({!r}, {!r})>'.format(self.path, self.patterns)

class Watchman(object):
    def __init__(self):
        self._proc = None
        self._watchers = set()
        self._client = None
        atexit.register(self._terminate)

    def run(self):
        self._client = pywatchman.client(timeout=300)
        self._client.capabilityCheck(required=['wildmatch', 'cmd-watch-project'])
        indico_project_root = os.path.realpath(os.path.join(get_root_path('indico'), '..'))
        paths = sorted({os.path.realpath(p) for p in _find_observable_paths() if os.path.exists(p)})
        for path in paths:
            patterns = ['**/*.py', '**/entry_points.txt']
            if path == indico_project_root:
                patterns += ['indico/indico.conf', 'indico/logging.yaml']
            watcher = Watcher(path, patterns)
            watcher.start(self._client)
            self._watchers.add(watcher)
        self._launch()
        self._monitor()

    def _monitor(self):
        while True:
            self._client.setTimeout(300)
            try:
                self._client.receive()
                for w in self._watchers:
                    w.consume(self._client)

                self._client.setTimeout(0.1)
                settled = False
                while not settled:
                    try:
                        self._client.receive()
                        for w in self._watchers:
                            w.consume(self._client)
                    except pywatchman.SocketTimeout:
                        settled = True
                        break

                triggered = False
                for w in self._watchers:
                    # this cannot be done with any() since all triggered watchers
                    # need to be reset during the check
                    if w.check():
                        triggered = True

                if triggered:
                    self._restart()

            except pywatchman.SocketTimeout:
                # are we still connected?
                try:
                    self._client.query('version')
                except Exception as exc:
                    print('watchman error:', exc)
                    return

            except KeyboardInterrupt:
                print()
                return

    def _launch(self, quiet=False, retry=0):
        assert not self._proc
        if not quiet and not retry:
            print(cformat('%{green!}Launching Indico'))
        try:
            argv = _disable_reloader(sys.argv)
            self._proc = subprocess.Popen(argv)
        except OSError as exc:
            delay = (retry + 1) * 0.5
            print(cformat('%{red!}Could not launch Indico: {}').format(exc))
            print(cformat('%{yellow}Retrying in {}s').format(delay))
            time.sleep(delay)
            self._launch(quiet=quiet, retry=(retry + 1))

    def _terminate(self, quiet=False):
        if not self._proc:
            return
        if not quiet:
            print(cformat('%{red!}Terminating Indico'))
        self._proc.terminate()
        self._proc = None

    def _restart(self):
        print(cformat('%{yellow!}Restarting Indico'))
        self._terminate(quiet=True)
        self._launch(quiet=True)
<commit_msg>Use nicer message when changes are in current dir<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2020 CERN
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the MIT License; see the
# LICENSE file for more details.

from __future__ import print_function, unicode_literals

import atexit
import os
import subprocess
import sys
import time

import pywatchman
from flask.helpers import get_root_path
from werkzeug._reloader import _find_observable_paths

from indico.util.console import cformat

def _patterns_to_terms(patterns):
    return ['anyof'] + [['match', p, 'wholename', {'includedotfiles': True}] for p in patterns]

def _disable_reloader(argv):
    argv = list(argv)  # we usually pass sys.argv, so let's not modify that
    for i, arg in enumerate(argv):
        if arg == '--reloader' and argv[i + 1] == 'watchman':
            argv[i + 1] = 'none'
        elif arg.startswith('--reloader') and '=' in arg:
            argv[i] = '--reloader=none'
    return argv

class Watcher(object):
    def __init__(self, path, patterns):
        self.path = path
        self.name = path.replace('/', '-').strip('-')
        self.patterns = patterns
        self.root_dir = None
        self.triggered = False

    def start(self, client):
        query = {
            'expression': _patterns_to_terms(self.patterns),
            'fields': ['name']
        }
        watch = client.query('watch-project', self.path)
        if 'warning' in watch:
            # no idea when this happens, but the example scripts have it...
            print('watchman warning:', watch['warning'])
        self.root_dir = watch['watch']
        if 'relative_path' in watch:
            query['relative_root'] = watch['relative_path']
        # get the initial clock value so that we only get updates
        query['since'] = client.query('clock', self.root_dir)['clock']
        client.query('subscribe', self.root_dir, self.name, query)

    def consume(self, client):
        data = client.getSubscription(self.name)
        if data:
            files = sorted(f for record in data for f in record.get('files', []))
            relpath = os.path.relpath(self.path)
            if relpath == '.':
                print(cformat('%{cyan}Changes found:').format(relpath))
            else:
                print(cformat('%{cyan}Changes found in %{cyan!}{}%{reset}%{cyan}:').format(relpath))
            for f in files:
                print(cformat(' * %{blue!}{}').format(f))
            self.triggered = True

    def check(self):
        triggered = self.triggered
        self.triggered = False
        return triggered

    def __repr__(self):
        return '<Watcher({!r}, {!r})>'.format(self.path, self.patterns)

class Watchman(object):
    def __init__(self):
        self._proc = None
        self._watchers = set()
        self._client = None
        atexit.register(self._terminate)

    def run(self):
        self._client = pywatchman.client(timeout=300)
        self._client.capabilityCheck(required=['wildmatch', 'cmd-watch-project'])
        indico_project_root = os.path.realpath(os.path.join(get_root_path('indico'), '..'))
        paths = sorted({os.path.realpath(p) for p in _find_observable_paths() if os.path.exists(p)})
        for path in paths:
            patterns = ['**/*.py', '**/entry_points.txt']
            if path == indico_project_root:
                patterns += ['indico/indico.conf', 'indico/logging.yaml']
            watcher = Watcher(path, patterns)
            watcher.start(self._client)
            self._watchers.add(watcher)
        self._launch()
        self._monitor()

    def _monitor(self):
        while True:
            self._client.setTimeout(300)
            try:
                self._client.receive()
                for w in self._watchers:
                    w.consume(self._client)

                self._client.setTimeout(0.1)
                settled = False
                while not settled:
                    try:
                        self._client.receive()
                        for w in self._watchers:
                            w.consume(self._client)
                    except pywatchman.SocketTimeout:
                        settled = True
                        break

                triggered = False
                for w in self._watchers:
                    # this cannot be done with any() since all triggered watchers
                    # need to be reset during the check
                    if w.check():
                        triggered = True

                if triggered:
                    self._restart()

            except pywatchman.SocketTimeout:
                # are we still connected?
                try:
                    self._client.query('version')
                except Exception as exc:
                    print('watchman error:', exc)
                    return

            except KeyboardInterrupt:
                print()
                return

    def _launch(self, quiet=False, retry=0):
        assert not self._proc
        if not quiet and not retry:
            print(cformat('%{green!}Launching Indico'))
        try:
            argv = _disable_reloader(sys.argv)
            self._proc = subprocess.Popen(argv)
        except OSError as exc:
            delay = (retry + 1) * 0.5
            print(cformat('%{red!}Could not launch Indico: {}').format(exc))
            print(cformat('%{yellow}Retrying in {}s').format(delay))
            time.sleep(delay)
            self._launch(quiet=quiet, retry=(retry + 1))

    def _terminate(self, quiet=False):
        if not self._proc:
            return
        if not quiet:
            print(cformat('%{red!}Terminating Indico'))
        self._proc.terminate()
        self._proc = None

    def _restart(self):
        print(cformat('%{yellow!}Restarting Indico'))
        self._terminate(quiet=True)
        self._launch(quiet=True)
<|endoftext|>


<commit_before>"""
Author: Frederik Muller, xmulle20@stud.fit.vutbr.cz
Date:   04/2017
"""

import bcrypt
from flask import request
from bson import json_util, ObjectId
import pymongo

from api import auth, db
from api.module import Module
from api.models.models import Category, CategoryException
from api.role import Role

category = Module('categories', __name__, url_prefix='/categories', no_version=True)

def get_categories():
	res = Category.query.all()
	categories = []
	for category in res:
		if category.parent == None:
			tmp = category.to_dict()
			tmp["children"] = category.children_dict()
			categories.append(tmp)
	return(json_util.dumps(categories))

@auth.required(Role.admin)
def add_category():
	r = request.get_json()
	try:
		category = Category.from_dict(r)
	except Exception as e:
		raise CategoryException(str(e))

	try:
		db.db.session.add(category)
		res = db.db.session.commit()
	except Exception as e:
		db.db.session.rollback()
		print(e)
		raise CategoryException(str(e))

	inserted = Category.query.get_or_404(category.id)

	category = inserted.to_dict()

	return(json_util.dumps(category))

@auth.required(Role.admin)
def remove_category(category_id):
	"""
	Remove the category
	"""

	category = Category.query.get_or_404(category_id)

	try:
		db.db.session.delete(category)
		db.db.session.commit()
	except Exception as e:
		db.db.session.rollback()
		raise CategoryException(str(e))

	## MISSING
	## if category has children, direct descendants will lose the parent

	tmp = category.to_dict()

	return(json_util.dumps(tmp))

@auth.required(Role.admin)
def edit_category(category_id):

	category_dict = request.get_json()
	category = Category.query.get_or_404(category_id)

	# check for all fields to be updated
	if "name" in category_dict and category_dict["name"]!= "":
		category.name = category_dict["name"]

	if "description" in category_dict and category_dict["description"] != "":
		category.description = category_dict["description"]

	if "slug" in category_dict and category_dict["slug"] != "":
		category.slug = category_dict["slug"]

	if "hidden" in category_dict and category_dict["hidden"] != "":
		category.hidden = category_dict["hidden"]

	# Update the category and return updated document
	try:
		db.db.session.commit()
	except Exception as e:
		db.db.session.rollback()
		raise CategoryException(str(e))

	tmp = category.to_dict()

	return(json_util.dumps(tmp))

def get_category(category_id):
	category = Category.query.get_or_404(category_id)
	category = category.to_dict()

	## MISSING
	## get all its products

	return(json_util.dumps(category))

category.add_url_rule('', view_func=get_categories, methods=['GET'])
category.add_url_rule('', view_func=add_category, methods=['POST'])
category.add_url_rule('/<string:category_id>', view_func=get_category, methods=['GET'])
category.add_url_rule('/<string:category_id>', view_func=edit_category, methods=['PUT'])
category.add_url_rule('/<string:category_id>', view_func=remove_category, methods=['DELETE'])
<commit_msg>api/modules/categories: add handling parents<commit_after>"""
Author: Frederik Muller, xmulle20@stud.fit.vutbr.cz
Author: Matej Vido, xvidom00@stud.fit.vutbr.cz
Date:   04/2017
"""

import bcrypt
from flask import request
from bson import json_util, ObjectId
import pymongo

from api import auth, db
from api.module import Module
from api.models.models import Category, CategoryException
from api.role import Role

category = Module('categories', __name__, url_prefix='/categories', no_version=True)

def get_categories():
	res = Category.query.all()
	categories = []
	for category in res:
		if category.parent == None:
			tmp = category.to_dict()
			tmp["children"] = category.children_dict()
			categories.append(tmp)
	return(json_util.dumps(categories))

@auth.required(Role.admin)
def add_category():
	r = request.get_json()
	r.pop("products", [])
	r.pop("children", [])
	parent_dict = r.pop("parent", None)
	parent = None
	if parent_dict != None:
		parent_id = parent_dict.get("id", None)
		if parent_id != None:
			parent = Category.query.get_or_404(parent_id)
	try:
		category = Category.from_dict(r)
	except Exception as e:
		raise CategoryException(str(e))

	try:
		if parent != None:
			parent.children.append(category)
		db.db.session.add(category)
		res = db.db.session.commit()
	except Exception as e:
		db.db.session.rollback()
		print(e)
		raise CategoryException(str(e))

	inserted = Category.query.get_or_404(category.id)

	category = inserted.to_dict()
	category["parent"] = inserted.parent_dict()

	return(json_util.dumps(category))

@auth.required(Role.admin)
def remove_category(category_id):
	"""
	Remove the category
	"""

	category = Category.query.get_or_404(category_id)

	try:
		db.db.session.delete(category)
		db.db.session.commit()
	except Exception as e:
		db.db.session.rollback()
		raise CategoryException(str(e))

	tmp = category.to_dict()

	return(json_util.dumps(tmp))

@auth.required(Role.admin)
def edit_category(category_id):

	category_dict = request.get_json()
	category = Category.query.get_or_404(category_id)

	# check for all fields to be updated
	if "name" in category_dict and category_dict["name"]!= "":
		category.name = category_dict["name"]

	if "description" in category_dict and category_dict["description"] != "":
		category.description = category_dict["description"]

	if "slug" in category_dict and category_dict["slug"] != "":
		category.slug = category_dict["slug"]

	if "hidden" in category_dict and category_dict["hidden"] != "":
		category.hidden = category_dict["hidden"]

	parent_dict = category_dict.get("parent", None)
	if parent_dict != None:
		parent_id = parent_dict.get("id", None)
		if parent_id != None:
			parent = Category.query.get(parent_id)
			if parent != None:
				category.parent = parent

	# Update the category and return updated document
	try:
		db.db.session.commit()
	except Exception as e:
		db.db.session.rollback()
		raise CategoryException(str(e))

	tmp = category.to_dict()
	tmp["parent"] = category.parent_dict()
	tmp["children"] = category.children_dict()

	return(json_util.dumps(tmp))

def get_category(category_id):
	category = Category.query.get_or_404(category_id)
	category_dict = category.to_dict()
	category_dict["products"] = category.products_dict()
	category_dict["parent"] = category.parent_dict()
	category_dict["children"] = category.children_with_products_dict()
	return(json_util.dumps(category_dict))

category.add_url_rule('', view_func=get_categories, methods=['GET'])
category.add_url_rule('', view_func=add_category, methods=['POST'])
category.add_url_rule('/<string:category_id>', view_func=get_category, methods=['GET'])
category.add_url_rule('/<string:category_id>', view_func=edit_category, methods=['PUT'])
category.add_url_rule('/<string:category_id>', view_func=remove_category, methods=['DELETE'])
<|endoftext|>


<commit_before>from .ddt_container import ddt_container
from .ddt_tile import ddt_tile
from .ddt_tile_html import ddt_tile_html

class ddt_container_filterMenuAndChart2dAndTable(ddt_container):
    def make_filterMenuAndChart2dAndTable(self,
        data_filtermenu,
        data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,
        data_svg_keys,data_svg_nestkeys,data_svg_keymap,
        data_table_keys,data_table_nestkeys,data_table_keymap,
        data_svg,
        data_table,
        svgtype,
        tabletype,
        svgx1axislabel='',
        svgy1axislabel='',
        tablekeymap = [],
        svgkeymap = [],
        formtile2datamap=[0],
        tabletile2datamap=[0],
        svgtile2datamap=[0],
        svgfilters=None,
        svgtileheader=None,
        svgparameters_I={},
        tablefilters=None,
        tableheaders=None
        ):
        '''Make a filter menu + n SVGs + Table
        INPUT:
        data_filtermenu = listDict of all data
        data_svg = listDict of all data (single plot with a different data from data 1 or a single plot with data 1/2)
                dictColumn of all data (dictionary of data split into different SVGs, required for multiple plots);
        data_table = listDict of table data
        parameters for filtermenu and table
            data_x_keys
            data_x_nestkeys
            data_x_keymap
        parameters for the svg objects
            data2_keys
            data2_nestkeys
            data2_keymap
        tileheader = title for each of the tiles
        svgtype = type of svg (TODO: add optional input for specifying specific svgs for multiple plots)
        tabletype = type of table
        OPTIONAL INPUT for single plot:
        svgkeymap = default, [data2_keymap],
        svgtile2datamap= default, [0],

        svgparameters_I = additional parameters to add or override default parameters

        '''

        data_filtermenu,data_svg,data_table,add_svg_data,add_table_data = self.parse_data(data_filtermenu,data_svg,data_table);

        data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,\
            data_svg_keys,data_svg_nestkeys,data_svg_keymap,\
            data_table_keys,data_table_nestkeys,data_table_keymap = self.parse_keys(
                data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,
                data_svg_keys,data_svg_nestkeys,data_svg_keymap,
                data_table_keys,data_table_nestkeys,data_table_keymap);

        #initialize data counter
        datacnt = 0;

        #make the form
        form = ddt_tile();
        form.make_tileparameters(
            tileparameters={
            'tileheader':'Filter menu',
            'tiletype':'html',
            'tileid':"filtermenu1",
            'rowid':"row1",
            'colid':"col1",
            'tileclass':"panel panel-default",
            'rowclass':"row",
            'colclass':"col-sm-6"}
            );
        form.make_htmlparameters(
            htmlparameters = {
            'htmlid':'filtermenuform1',
            "htmltype":'form_01',
            "formsubmitbuttonidtext":{'id':'submit1','text':'submit'},
            "formresetbuttonidtext":{'id':'reset1','text':'reset'},
            "formupdatebuttonidtext":{'id':'update1','text':'update'}},
            );
        self.add_parameters(form.get_parameters());
        self.update_tile2datamap("filtermenu1",formtile2datamap);
        self.add_filtermenu(
            {"filtermenuid":"filtermenu1",
            "filtermenuhtmlid":"filtermenuform1",
            "filtermenusubmitbuttonid":"submit1",
            "filtermenuresetbuttonid":"reset1",
            "filtermenuupdatebuttonid":"update1"}
            );

         # filtermenu data:
        self.add_data(
            data_filtermenu,
            data_filtermenu_keys,
            data_filtermenu_nestkeys
            );
        datacnt += 1;

        #make the svg objects
        if type(data_svg) is dict:
            rowcnt = self.make_nChart2dDictList(
                data_svg,
                data_svg_keys,
                data_svg_nestkeys,
                data_svg_keymap,
                svgtype,
                svgtile2datamap,
                svgfilters,
                svgtileheader,
                svgparameters_I=svgparameters_I);
        elif type(svgtile2datamap[0]) is list:
            rowcnt = self.make_nChart2dListDict(
                add_svg_data,
                data_svg,
                data_svg_keys,
                data_svg_nestkeys,
                data_svg_keymap,
                svgtype,
                svgkeymap,
                svgtile2datamap,
                svgfilters,
                svgtileheader,
                svgparameters_I=svgparameters_I);
        else:
            rowcnt = self.make_chart2dListDict(
                add_svg_data,
                data_svg,
                data_svg_keys,
                data_svg_nestkeys,
                data_svg_keymap,
                svgtype,
                svgkeymap,
                svgtile2datamap,
                svgfilters,
                svgtileheader,
                svgparameters_I=svgparameters_I
                );

        # make the table object
        crosstable = ddt_tile();
        crosstable.make_tileparameters(
            tileparameters = {
            'tileheader':'Table',
            'tiletype':'table',
            'tileid':"tabletile1",
            'rowid':"row"+str(rowcnt+1),
            'colid':"col1",
            'tileclass':"panel panel-default",
            'rowclass':"row",
            'colclass':"col-sm-12"
            }
            );
        crosstable.make_tableparameters(
            tableparameters = {
            "tabletype":tabletype,
            'tableid':'table1',
            "tablefilters":tablefilters,
            "tableclass":"table  table-condensed table-hover",
    		'tableformtileid':'tile1',
            "tablekeymap":tablekeymap,
            "tableheaders":tableheaders,}
            );
        self.add_parameters(crosstable.get_parameters());
        self.update_tile2datamap("tabletile1",tabletile2datamap);

        if add_table_data:
            self.add_data(
                data_table,
                data_table_keys,
                data_table_nestkeys
                );
            datacnt += 1;

    def make_nChart2dDictList(self,
            data_svg,
            data_svg_keys,
            data_svg_nestkeys,
            data_svg_keymap,
            svgtype,
            svgtile2datamap,
            svgfilters,
            svgtileheader,
            svgparameters_I={},):
        '''make the SVG objects from a dictList
        USE:
        large data set partitioned into individual data sets
        same svgtype
        same keys, nestkeys, and keymaps'''

        rowcnt = 1;
        colcnt = 1;
        cnt = 0;
        for k,v in data_svg.items():
            svgtileid = "tilesvg"+str(cnt);
            svgid = 'svg'+str(cnt);
            iter=cnt+1; #start at 1
            if (cnt % 2 == 0):
                rowcnt = rowcnt+1;#even
                colcnt = 1;
            else:
                colcnt = colcnt+1;
            # svg
            svg = ddt_tile();
            svg.make_tileparameters(
                tileparameters={
                'tileheader':svgtileheader,
                'tiletype':'svg',
                'tileid':svgtileid,
                'rowid':"row"+str(rowcnt),
                'colid':"col"+str(colcnt),
                'tileclass':"panel panel-default",
                'rowclass':"row",
                'colclass':"col-sm-6"
                });
            svg.make_svgparameters(
                svgparameters={
                "svgtype":svgtype,
                "svgkeymap":[data_svg_keymap],
                'svgid':'svg'+str(cnt),
                "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
                "svgwidth":500,
                "svgheight":350,
                "svgx1axislabel":data_svg_keymap['xdata'],
                "svgy1axislabel":data_svg_keymap['ydata']
                }
                    );
            svg.specificparameters.update(svgparameters_I);
            self.add_parameters(svg.get_parameters());
            self.update_tile2datamap(svgtileid,[iter]);
            self.add_data(
                v,
                data_svg_keys,
                data_svg_nestkeys
                );
            cnt+=1;

        return rowcnt;
    def make_nChart2dListDict(self,
            add_svg_data,
            data_svg,
            data_svg_keys,
            data_svg_nestkeys,
            data_svg_keymap,
            svgtype,
            svgkeymap,
            svgtile2datamap,
            svgfilters,
            svgtileheader,
            svgparameters_I={},):
        '''make the SVG objects from a listDict
        USE:
        multiple views of the same data set
        multiple svgtypes
        multiple keys, nestkeys, and keymaps
        '''
        rowcnt = 1;
        colcnt = 1;
        cnt = 0;
        for i in range(len(svgtile2datamap)):
            svgtileid = "tilesvg"+str(cnt);
            svgid = 'svg'+str(cnt);
            iter=cnt+1; #start at 1
            if (cnt % 2 == 0):
                rowcnt = rowcnt+1;#even
                colcnt = 1;
            else:
                colcnt = colcnt+1;
            # svg
            svg = ddt_tile();
            svg.make_tileparameters(
                tileparameters={
                'tileheader':svgtileheader,
                'tiletype':'svg',
                'tileid':svgtileid,
                'rowid':"row"+str(rowcnt),
                'colid':"col"+str(colcnt),
                'tileclass':"panel panel-default",
                'rowclass':"row",
                'colclass':"col-sm-6"
                });
            svg.make_svgparameters(
                svgparameters={
                "svgtype":svgtype[i],
                "svgkeymap":svgkeymap[i],
                'svgid':'svg'+str(cnt),
                "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
                "svgwidth":500,
                "svgheight":350,
                "svgx1axislabel":data_svg_keymap[i]['xdata'],
                "svgy1axislabel":data_svg_keymap[i]['ydata']
                }
                    );
            svgparameters.update(svgparameters_I);
            self.add_parameters(svg.get_parameters());
            self.update_tile2datamap(svgtileid,svgtile2datamap[i]);
            cnt+=1;
        return rowcnt;

        if add_svg_data:
            self.add_data(
                data_svg,
                data_svg_keys,
                data_svg_nestkeys
                );

    def make_chart2dListDict(self,
            add_svg_data,
            data_svg,
            data_svg_keys,
            data_svg_nestkeys,
            data_svg_keymap,
            svgtype,
            svgkeymap,
            svgtile2datamap,
            svgfilters,
            svgtileheader,
            svgparameters_I={},
            ):
        '''make a single SVG object from a listDict
        '''

        cnt = 0;
        svgtileid = "tilesvg"+str(cnt);
        svgid = 'svg'+str(cnt);
        rowcnt = 2;
        colcnt = 1;
        # make the svg object
        svg = ddt_tile();
        svg.make_tileparameters(
            tileparameters={
            'tileheader':svgtileheader,
            'tiletype':'svg',
            'tileid':svgtileid,
            'rowid':"row"+str(rowcnt),
            'colid':"col"+str(colcnt),
            'tileclass':"panel panel-default",
            'rowclass':"row",
            'colclass':"col-sm-6"
            });
        # make the svg parameters
        svgparameters={
            "svgtype":svgtype,
            "svgkeymap":svgkeymap,
            'svgid':svgid,
            "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
            "svgwidth":350,
            "svgheight":250,
            "svgx1axislabel":data_svg_keymap['xdata'],
            "svgy1axislabel":data_svg_keymap['ydata']
            };
        svgparameters.update(svgparameters_I);
        svg.make_svgparameters(
            svgparameters=svgparameters
                );
        self.add_parameters(svg.get_parameters());
        self.update_tile2datamap(svgtileid,svgtile2datamap);

        #add data 2
        if add_svg_data:
            self.add_data(
                data_svg,
                data_svg_keys,
                data_svg_nestkeys
                );
            cnt+=1;
        return rowcnt;

    def parse_data(self,data_filtermenu,data_svg,data_table):
        '''parse out the data'''

        add_svg_data = True;
        add_table_data = True;
        #parse out the data
        if data_filtermenu is None or not data_filtermenu:
            print('no filtermenu data provided.');
            #exit(-1);
            data_filtermenu = [];
            data_svg = data_filtermenu;
            add_svg_data = False;
            data_table = data_filtermenu;
            add_table_data = False;
        if data_svg is None or not data_svg:
            data_svg = data_filtermenu;
            add_svg_data = False;
        if data_table is None or not data_table:
            data_table = data_filtermenu;
            add_table_data = False;
        return data_filtermenu,data_svg,data_table,add_svg_data,add_table_data;

    def parse_keys(self,
        data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,
        data_svg_keys,data_svg_nestkeys,data_svg_keymap,
        data_table_keys,data_table_nestkeys,data_table_keymap,
        ):
        '''parse out the data keys'''

        if data_filtermenu_keys is None or not data_filtermenu_keys:
            print('no filtermenu keys provided.');
            exit(-1);
        if data_svg_keys is None or not data_svg_keys:
            data_svg_keys = data_filtermenu_keys;
            data_svg_nestkeys = data_filtermenu_nestkeys;
            data_svg_keymap = data_filtermenu_keymap;
        if data_table_keys is None or not data_table_keys:
            data_table_keys = data_filtermenu_keys;
            data_table_nestkeys = data_filtermenu_nestkeys;
            data_table_keymap = data_filtermenu_keymap;

        return data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,\
            data_svg_keys,data_svg_nestkeys,data_svg_keymap,\
            data_table_keys,data_table_nestkeys,data_table_keymap<commit_msg>updates to filterMenu<commit_after>from .ddt_container import ddt_container
from .ddt_tile import ddt_tile
from .ddt_tile_html import ddt_tile_html

class ddt_container_filterMenuAndChart2dAndTable(ddt_container):
    def make_filterMenuAndChart2dAndTable(self,
        data_filtermenu,
        data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,
        data_svg_keys,data_svg_nestkeys,data_svg_keymap,
        data_table_keys,data_table_nestkeys,data_table_keymap,
        data_svg,
        data_table,
        svgtype,
        tabletype,
        svgx1axislabel='',
        svgy1axislabel='',
        tablekeymap = [],
        svgkeymap = [],
        formtile2datamap=[0],
        tabletile2datamap=[0],
        svgtile2datamap=[0],
        svgfilters=None,
        svgtileheader=None,
        svgparameters_I={},
        tablefilters=None,
        tableheaders=None
        ):
        '''Make a filter menu + n SVGs + Table
        INPUT:
        data_filtermenu = listDict of all data
        data_svg = listDict of all data (single plot with a different data from data 1 or a single plot with data 1/2)
                dictColumn of all data (dictionary of data split into different SVGs, required for multiple plots);
        data_table = listDict of table data
        parameters for filtermenu and table
            data_x_keys
            data_x_nestkeys
            data_x_keymap
        parameters for the svg objects
            data2_keys
            data2_nestkeys
            data2_keymap
        tileheader = title for each of the tiles
        svgtype = type of svg (TODO: add optional input for specifying specific svgs for multiple plots)
        tabletype = type of table
        OPTIONAL INPUT for single plot:
        svgkeymap = default, [data2_keymap],
        svgtile2datamap= default, [0],

        svgparameters_I = additional parameters to add or override default parameters

        '''

        data_filtermenu,data_svg,data_table,add_svg_data,add_table_data = self.parse_data(data_filtermenu,data_svg,data_table);

        data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,\
            data_svg_keys,data_svg_nestkeys,data_svg_keymap,\
            data_table_keys,data_table_nestkeys,data_table_keymap = self.parse_keys(
                data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,
                data_svg_keys,data_svg_nestkeys,data_svg_keymap,
                data_table_keys,data_table_nestkeys,data_table_keymap);

        #initialize data counter
        datacnt = 0;

        #make the form
        form = ddt_tile();
        form.make_tileparameters(
            tileparameters={
            'tileheader':'Filter menu',
            'tiletype':'html',
            'tileid':"filtermenu1",
            'rowid':"row1",
            'colid':"col1",
            'tileclass':"panel panel-default",
            'rowclass':"row",
            'colclass':"col-sm-6"}
            );
        form.make_htmlparameters(
            htmlparameters = {
            'htmlid':'filtermenuform1',
            "htmltype":'form_01',
            "formsubmitbuttonidtext":{'id':'submit1','text':'submit'},
            "formresetbuttonidtext":{'id':'reset1','text':'reset'},
            "formupdatebuttonidtext":{'id':'update1','text':'update'}},
            );
        self.add_parameters(form.get_parameters());
        self.update_tile2datamap("filtermenu1",formtile2datamap);
        self.add_filtermenu(
            {"filtermenuid":"filtermenu1",
            "filtermenuhtmlid":"filtermenuform1",
            "filtermenusubmitbuttonid":"submit1",
            "filtermenuresetbuttonid":"reset1",
            "filtermenuupdatebuttonid":"update1"}
            );

         # filtermenu data:
        self.add_data(
            data_filtermenu,
            data_filtermenu_keys,
            data_filtermenu_nestkeys
            );
        datacnt += 1;

        #make the svg objects
        if type(data_svg) is dict:
            rowcnt = self.make_nChart2dDictList(
                data_svg,
                data_svg_keys,
                data_svg_nestkeys,
                data_svg_keymap,
                svgtype,
                svgkeymap,
                svgtile2datamap,
                svgfilters,
                svgtileheader,
                svgparameters_I=svgparameters_I);
        elif type(svgtile2datamap[0]) is list:
            rowcnt = self.make_nChart2dListDict(
                add_svg_data,
                data_svg,
                data_svg_keys,
                data_svg_nestkeys,
                data_svg_keymap,
                svgtype,
                svgkeymap,
                svgtile2datamap,
                svgfilters,
                svgtileheader,
                svgparameters_I=svgparameters_I);
        else:
            rowcnt = self.make_chart2dListDict(
                add_svg_data,
                data_svg,
                data_svg_keys,
                data_svg_nestkeys,
                data_svg_keymap,
                svgtype,
                svgkeymap,
                svgtile2datamap,
                svgfilters,
                svgtileheader,
                svgparameters_I=svgparameters_I
                );

        # make the table object
        crosstable = ddt_tile();
        crosstable.make_tileparameters(
            tileparameters = {
            'tileheader':'Table',
            'tiletype':'table',
            'tileid':"tabletile1",
            'rowid':"row"+str(rowcnt+1),
            'colid':"col1",
            'tileclass':"panel panel-default",
            'rowclass':"row",
            'colclass':"col-sm-12"
            }
            );
        crosstable.make_tableparameters(
            tableparameters = {
            "tabletype":tabletype,
            'tableid':'table1',
            "tablefilters":tablefilters,
            "tableclass":"table  table-condensed table-hover",
    		'tableformtileid':'tile1',
            "tablekeymap":tablekeymap,
            "tableheaders":tableheaders,}
            );
        self.add_parameters(crosstable.get_parameters());
        self.update_tile2datamap("tabletile1",tabletile2datamap);

        if add_table_data:
            self.add_data(
                data_table,
                data_table_keys,
                data_table_nestkeys
                );
            datacnt += 1;

    def make_nChart2dDictList(self,
            data_svg,
            data_svg_keys,
            data_svg_nestkeys,
            data_svg_keymap,
            svgtype,
            svgkeymap,
            svgtile2datamap,
            svgfilters,
            svgtileheader,
            svgparameters_I={},):
        '''make the SVG objects from a dictList
        USE:
        large data set partitioned into individual data sets
        same svgtype
        same keys, nestkeys, and keymaps

        TODO: add support for scatter line plot
        '''

        rowcnt = 1;
        colcnt = 1;
        cnt = 0;
        for k,v in data_svg.items():
            svgtileid = "tilesvg"+str(cnt);
            svgid = 'svg'+str(cnt);
            iter=cnt+1; #start at 1
            if (cnt % 2 == 0):
                rowcnt = rowcnt+1;#even
                colcnt = 1;
            else:
                colcnt = colcnt+1;
            # svg
            svg = ddt_tile();
            svg.make_tileparameters(
                tileparameters={
                'tileheader':svgtileheader + ' ' + str(k),
                'tiletype':'svg',
                'tileid':svgtileid,
                'rowid':"row"+str(rowcnt),
                'colid':"col"+str(colcnt),
                'tileclass':"panel panel-default",
                'rowclass':"row",
                'colclass':"col-sm-6"
                });
            svg.make_svgparameters(
                svgparameters={
                "svgtype":svgtype,
                "svgkeymap":[data_svg_keymap],
                'svgid':'svg'+str(cnt),
                "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
                "svgwidth":500,
                "svgheight":350,
                "svgx1axislabel":data_svg_keymap['xdata'],
                "svgy1axislabel":data_svg_keymap['ydata']
                }
                    );
            svg.specificparameters.update(svgparameters_I);
            self.add_parameters(svg.get_parameters());
            self.update_tile2datamap(svgtileid,[iter]);
            self.add_data(
                v,
                data_svg_keys,
                data_svg_nestkeys
                );
            cnt+=1;

        return rowcnt;
    def make_nChart2dListDict(self,
            add_svg_data,
            data_svg,
            data_svg_keys,
            data_svg_nestkeys,
            data_svg_keymap,
            svgtype,
            svgkeymap,
            svgtile2datamap,
            svgfilters,
            svgtileheader,
            svgparameters_I={},):
        '''make the SVG objects from a listDict
        USE:
        multiple views of the same data set
        multiple svgtypes
        multiple keys, nestkeys, and keymaps
        '''
        rowcnt = 1;
        colcnt = 1;
        cnt = 0;
        for i in range(len(svgtile2datamap)):
            svgtileid = "tilesvg"+str(cnt);
            svgid = 'svg'+str(cnt);
            iter=cnt+1; #start at 1
            if (cnt % 2 == 0):
                rowcnt = rowcnt+1;#even
                colcnt = 1;
            else:
                colcnt = colcnt+1;
            # svg
            svg = ddt_tile();
            svg.make_tileparameters(
                tileparameters={
                'tileheader':svgtileheader,
                'tiletype':'svg',
                'tileid':svgtileid,
                'rowid':"row"+str(rowcnt),
                'colid':"col"+str(colcnt),
                'tileclass':"panel panel-default",
                'rowclass':"row",
                'colclass':"col-sm-6"
                });
            svg.make_svgparameters(
                svgparameters={
                "svgtype":svgtype[i],
                "svgkeymap":svgkeymap[i],
                'svgid':'svg'+str(cnt),
                "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
                "svgwidth":500,
                "svgheight":350,
                "svgx1axislabel":data_svg_keymap[i]['xdata'],
                "svgy1axislabel":data_svg_keymap[i]['ydata']
                }
                    );
            svgparameters.update(svgparameters_I);
            self.add_parameters(svg.get_parameters());
            self.update_tile2datamap(svgtileid,svgtile2datamap[i]);
            cnt+=1;
        return rowcnt;

        if add_svg_data:
            self.add_data(
                data_svg,
                data_svg_keys,
                data_svg_nestkeys
                );

    def make_chart2dListDict(self,
            add_svg_data,
            data_svg,
            data_svg_keys,
            data_svg_nestkeys,
            data_svg_keymap,
            svgtype,
            svgkeymap,
            svgtile2datamap,
            svgfilters,
            svgtileheader,
            svgparameters_I={},
            ):
        '''make a single SVG object from a listDict
        '''

        cnt = 0;
        svgtileid = "tilesvg"+str(cnt);
        svgid = 'svg'+str(cnt);
        rowcnt = 2;
        colcnt = 1;
        # make the svg object
        svg = ddt_tile();
        svg.make_tileparameters(
            tileparameters={
            'tileheader':svgtileheader,
            'tiletype':'svg',
            'tileid':svgtileid,
            'rowid':"row"+str(rowcnt),
            'colid':"col"+str(colcnt),
            'tileclass':"panel panel-default",
            'rowclass':"row",
            'colclass':"col-sm-6"
            });
        # make the svg parameters
        svgparameters={
            "svgtype":svgtype,
            "svgkeymap":svgkeymap,
            'svgid':svgid,
            "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
            "svgwidth":350,
            "svgheight":250,
            "svgx1axislabel":data_svg_keymap['xdata'],
            "svgy1axislabel":data_svg_keymap['ydata']
            };
        svgparameters.update(svgparameters_I);
        svg.make_svgparameters(
            svgparameters=svgparameters
                );
        self.add_parameters(svg.get_parameters());
        self.update_tile2datamap(svgtileid,svgtile2datamap);

        #add data 2
        if add_svg_data:
            self.add_data(
                data_svg,
                data_svg_keys,
                data_svg_nestkeys
                );
            cnt+=1;
        return rowcnt;

    def parse_data(self,data_filtermenu,data_svg,data_table):
        '''parse out the data'''

        add_svg_data = True;
        add_table_data = True;
        #parse out the data
        if data_filtermenu is None or not data_filtermenu:
            print('no filtermenu data provided.');
            #exit(-1);
            data_filtermenu = [];
            data_svg = data_filtermenu;
            add_svg_data = False;
            data_table = data_filtermenu;
            add_table_data = False;
        if data_svg is None or not data_svg:
            data_svg = data_filtermenu;
            add_svg_data = False;
        if data_table is None or not data_table:
            data_table = data_filtermenu;
            add_table_data = False;
        return data_filtermenu,data_svg,data_table,add_svg_data,add_table_data;

    def parse_keys(self,
        data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,
        data_svg_keys,data_svg_nestkeys,data_svg_keymap,
        data_table_keys,data_table_nestkeys,data_table_keymap,
        ):
        '''parse out the data keys'''

        if data_filtermenu_keys is None or not data_filtermenu_keys:
            print('no filtermenu keys provided.');
            exit(-1);
        if data_svg_keys is None or not data_svg_keys:
            data_svg_keys = data_filtermenu_keys;
            data_svg_nestkeys = data_filtermenu_nestkeys;
            data_svg_keymap = data_filtermenu_keymap;
        if data_table_keys is None or not data_table_keys:
            data_table_keys = data_filtermenu_keys;
            data_table_nestkeys = data_filtermenu_nestkeys;
            data_table_keymap = data_filtermenu_keymap;

        return data_filtermenu_keys,data_filtermenu_nestkeys,data_filtermenu_keymap,\
            data_svg_keys,data_svg_nestkeys,data_svg_keymap,\
            data_table_keys,data_table_nestkeys,data_table_keymap<|endoftext|>


<commit_before>"""
Redis plugin polls Redis for stats

"""
import logging

from newrelic_plugin_agent.plugins import base

LOGGER = logging.getLogger(__name__)

class Redis(base.SocketStatsPlugin):

    GUID = 'com.meetme.newrelic_redis_agent'

    DEFAULT_PORT = 6379

    def add_datapoints(self, stats):
        """Add all of the data points for a node

        :param dict stats: all of the nodes

        """
        self.add_gauge_value('Clients/Blocked', '',
                             stats.get('blocked_clients', 0))
        self.add_gauge_value('Clients/Connected', '',
                             stats.get('connected_clients', 0))
        self.add_gauge_value('Slaves/Connected', '',
                             stats.get('connected_slaves', 0))

        self.add_derive_value('Keys/Evicted', '',
                              stats.get('evicted_keys', 0))
        self.add_derive_value('Keys/Expired', '',
                              stats.get('expired_keys', 0))
        self.add_derive_value('Keys/Hit', '',
                              stats.get('keyspace_hits', 0))
        self.add_derive_value('Keys/Missed', '',
                              stats.get('keyspace_misses', 0))

        self.add_derive_value('Commands Processed', '',
                              stats.get('total_commands_processed', 0))
        self.add_derive_value('Connections', '',
                              stats.get('total_connections_received', 0))
        self.add_derive_value('Changes Since Last Save', '',
                              stats.get('changes_since_last_save', 0))

        self.add_gauge_value('Pubsub/Commands', '',
                             stats.get('pubsub_commands', 0))
        self.add_gauge_value('Pubsub/Patterns', '',
                             stats.get('pubsub_patterns', 0))

        self.add_derive_value('CPU/User/Self', 'sec',
                              stats.get('used_cpu_user', 0))
        self.add_derive_value('CPU/System/Self', 'sec',
                              stats.get('used_cpu_sys', 0))

        self.add_derive_value('CPU/User/Children', 'sec',
                              stats.get('used_cpu_user_childrens', 0))

        self.add_derive_value('CPU/System/Children', 'sec',
                              stats.get('used_cpu_sys_childrens', 0))

        self.add_gauge_value('Memory Use', 'MB',
                             stats.get('used_memory', 0) / 1048576,
                             max_val=stats.get('used_memory_peak',
                                                0) / 1048576)
        self.add_gauge_value('Memory Fragmentation', 'ratio',
                             stats.get('mem_fragmentation_ratio', 0))

        keys, expires = 0, 0
        for db in range(0, self.config.get('db_count', 16)):

            db_stats = stats.get('db%i' % db, dict())
            self.add_gauge_value('DB/%s/Expires' % db, '',
                                db_stats.get('expires', 0))
            self.add_gauge_value('DB/%s/Keys' % db, '',
                                 db_stats.get('keys', 0))
            keys += db_stats.get('keys', 0)
            expires += db_stats.get('expires', 0)

        self.add_gauge_value('Keys/Total', '', keys)
        self.add_gauge_value('Keys/Will Expire', '', expires)

    def connect(self):
        """Top level interface to create a socket and connect it to the
        memcached daemon.

        :rtype: socket

        """
        connection = super(Redis, self).connect()
        if self.config.get('password'):
            connection.send("*2\r\n$4\r\nAUTH\r\n$%i\r\n%s\r\n" %
                            (len(self.config['password']),
                             self.config['password']))
            buffer_value = connection.recv(self.SOCKET_RECV_MAX)
            if buffer_value == '+OK\r\n':
                return connection
            LOGGER.error('Authentication error: %s', buffer_value[4:].strip())
            return None
        return connection

    def fetch_data(self, connection):
        """Loop in and read in all the data until we have received it all.

        :param  socket connection: The connection
        :rtype: dict

        """
        connection.send("*0\r\ninfo\r\n")

        # Read in the first line $1437
        buffer_value = connection.recv(self.SOCKET_RECV_MAX)
        lines = buffer_value.split('\r\n')

        if lines[0][0] == '$':
            byte_size = int(lines[0][1:].strip())
        else:
            return None

        while len(buffer_value) < byte_size:
            buffer_value += connection.recv(self.SOCKET_RECV_MAX)

        lines = buffer_value.split('\r\n')
        values = dict()
        for line in lines:
            if ':' in line:
                key, value = line.strip().split(':')
                if key[:2] == 'db':
                    values[key] = dict()
                    subvalues = value.split(',')
                    for temp in subvalues:
                        subvalue = temp.split('=')
                        value = subvalue[-1]
                        try:
                            values[key][subvalue[0]] = int(value)
                        except ValueError:
                            try:
                                values[key][subvalue[0]] = float(value)
                            except ValueError:
                                values[key][subvalue[0]] = value
                    continue
                try:
                    values[key] = int(value)
                except ValueError:
                    try:
                        values[key] = float(value)
                    except ValueError:
                        values[key] = value
        return values
<commit_msg>redis hits ratio<commit_after>"""
Redis plugin polls Redis for stats

"""
import logging

from newrelic_plugin_agent.plugins import base

LOGGER = logging.getLogger(__name__)

class Redis(base.SocketStatsPlugin):

    GUID = 'com.meetme.newrelic_redis_agent'

    DEFAULT_PORT = 6379

    def add_datapoints(self, stats):
        """Add all of the data points for a node

        :param dict stats: all of the nodes

        """
        self.add_gauge_value('Clients/Blocked', '',
                             stats.get('blocked_clients', 0))
        self.add_gauge_value('Clients/Connected', '',
                             stats.get('connected_clients', 0))
        self.add_gauge_value('Slaves/Connected', '',
                             stats.get('connected_slaves', 0))

        # must happen before saving the new values
        # but only if we have the previous values
        if 'Keys/Hit' in self.derive_last_interval.keys() and 'Keys/Missed' in self.derive_last_interval.keys():
            prev_hits   = self.derive_last_interval['Keys/Hit']
            prev_misses = self.derive_last_interval['Keys/Missed']

            # hits and misses since the last measure
            hits   = stats.get('keyspace_hits', 0)   - prev_hits
            misses = stats.get('keyspace_misses', 0) - prev_misses

            # total queries since the last measure
            total = hits + misses

            if total > 0:
                self.add_gauge_value('Hit Ratio', '', 100 * hits / total)

        self.add_derive_value('Keys/Evicted', '',
                              stats.get('evicted_keys', 0))
        self.add_derive_value('Keys/Expired', '',
                              stats.get('expired_keys', 0))
        self.add_derive_value('Keys/Hit', '',
                              stats.get('keyspace_hits', 0))
        self.add_derive_value('Keys/Missed', '',
                              stats.get('keyspace_misses', 0))

        self.add_derive_value('Commands Processed', '',
                              stats.get('total_commands_processed', 0))
        self.add_derive_value('Connections', '',
                              stats.get('total_connections_received', 0))
        self.add_derive_value('Changes Since Last Save', '',
                              stats.get('changes_since_last_save', 0))

        self.add_gauge_value('Pubsub/Commands', '',
                             stats.get('pubsub_commands', 0))
        self.add_gauge_value('Pubsub/Patterns', '',
                             stats.get('pubsub_patterns', 0))

        self.add_derive_value('CPU/User/Self', 'sec',
                              stats.get('used_cpu_user', 0))
        self.add_derive_value('CPU/System/Self', 'sec',
                              stats.get('used_cpu_sys', 0))

        self.add_derive_value('CPU/User/Children', 'sec',
                              stats.get('used_cpu_user_childrens', 0))

        self.add_derive_value('CPU/System/Children', 'sec',
                              stats.get('used_cpu_sys_childrens', 0))

        self.add_gauge_value('Memory Use', 'MB',
                             stats.get('used_memory', 0) / 1048576,
                             max_val=stats.get('used_memory_peak',
                                                0) / 1048576)
        self.add_gauge_value('Memory Fragmentation', 'ratio',
                             stats.get('mem_fragmentation_ratio', 0))

        keys, expires = 0, 0
        for db in range(0, self.config.get('db_count', 16)):

            db_stats = stats.get('db%i' % db, dict())
            self.add_gauge_value('DB/%s/Expires' % db, '',
                                db_stats.get('expires', 0))
            self.add_gauge_value('DB/%s/Keys' % db, '',
                                 db_stats.get('keys', 0))
            keys += db_stats.get('keys', 0)
            expires += db_stats.get('expires', 0)

        self.add_gauge_value('Keys/Total', '', keys)
        self.add_gauge_value('Keys/Will Expire', '', expires)

    def connect(self):
        """Top level interface to create a socket and connect it to the
        memcached daemon.

        :rtype: socket

        """
        connection = super(Redis, self).connect()
        if self.config.get('password'):
            connection.send("*2\r\n$4\r\nAUTH\r\n$%i\r\n%s\r\n" %
                            (len(self.config['password']),
                             self.config['password']))
            buffer_value = connection.recv(self.SOCKET_RECV_MAX)
            if buffer_value == '+OK\r\n':
                return connection
            LOGGER.error('Authentication error: %s', buffer_value[4:].strip())
            return None
        return connection

    def fetch_data(self, connection):
        """Loop in and read in all the data until we have received it all.

        :param  socket connection: The connection
        :rtype: dict

        """
        connection.send("*0\r\ninfo\r\n")

        # Read in the first line $1437
        buffer_value = connection.recv(self.SOCKET_RECV_MAX)
        lines = buffer_value.split('\r\n')

        if lines[0][0] == '$':
            byte_size = int(lines[0][1:].strip())
        else:
            return None

        while len(buffer_value) < byte_size:
            buffer_value += connection.recv(self.SOCKET_RECV_MAX)

        lines = buffer_value.split('\r\n')
        values = dict()
        for line in lines:
            if ':' in line:
                key, value = line.strip().split(':')
                if key[:2] == 'db':
                    values[key] = dict()
                    subvalues = value.split(',')
                    for temp in subvalues:
                        subvalue = temp.split('=')
                        value = subvalue[-1]
                        try:
                            values[key][subvalue[0]] = int(value)
                        except ValueError:
                            try:
                                values[key][subvalue[0]] = float(value)
                            except ValueError:
                                values[key][subvalue[0]] = value
                    continue
                try:
                    values[key] = int(value)
                except ValueError:
                    try:
                        values[key] = float(value)
                    except ValueError:
                        values[key] = value
        return values
<|endoftext|>


<commit_before><commit_msg>the command line should retain focus while cycling through the images<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function, division

import click
from collections import Counter
import os
import pandas as pd

from ninja_utils.utils import find_between
from ninja_utils.utils import verify_make_dir

from ninja_dojo.taxonomy import NCBITree

from ninja_shogun.aligners.bowtie import bowtie2

def yield_alignments_from_sam_inf(inf):
    with open(inf) as fh:
        for i in fh:
            line = i.split('\t')
            # this function yields qname, rname
            try:
                yield line[0], line[2]
            except IndexError:
                print('Incorrect SAM input %s' % (inf))

def lca_gg(taxonomy_a, taxonomy_b):
    taxonomy_1 = taxonomy_a.split(';')
    taxonomy_2 = taxonomy_b.split(';')
    lca = []
    for i in zip(taxonomy_1, taxonomy_2):
        if i[0] == i[1]:
            lca.append(i[0])
        else:
            break
    if lca:
        return ';'.join(lca)

def collapse(lca_map, depth):
    if depth > 0:
        for name in lca_map:
            taxonomy = lca_map[name]
            if taxonomy:
                taxonomy = taxonomy.split(';')
                if len(taxonomy) < depth:
                    lca_map[name] = None
                elif len(taxonomy) > depth:
                    lca_map[name] = ';'.join(taxonomy[:depth])
        return lca_map
    else:
        return lca_map

@click.command()
@click.option('-i', '--input', type=click.Path(), default=os.getcwd())
@click.option('-o', '--output', type=click.Path(), default=os.getcwd())
@click.option('-b', '--bt2_indx')
@click.option('-x', '--extract_ncbi_tid', default='ncbi_tid|,|')
@click.option('-d', '--depth', type=click.INT, default=7, help='The depth of the search (7=species default, 0=No Collapse)')
@click.option('-p', '--threads', type=click.INT, default=1)
def shogun_bt2_lca(input, output, bt2_indx, extract_ncbi_tid, depth, threads):
    verify_make_dir(output)

    fna_files = [os.path.join(input, filename) for filename in os.listdir(input) if filename.endswith('.fna')]

    for fna_file in fna_files:
        sam_outf = os.path.join(output, '.'.join(str(os.path.basename(fna_file)).split('.')[:-1]) + '.sam')
        print(bowtie2(fna_file, sam_outf, bt2_indx, num_threads=threads))

    tree = NCBITree()
    begin, end = extract_ncbi_tid.split(',')

    counts = []
    sam_files = [os.path.join(output, filename) for filename in os.listdir(output) if filename.endswith('.sam')]

    for sam_file in sam_files:
        lca_map = {}
        for qname, rname in yield_alignments_from_sam_inf(sam_file):
            ncbi_tid = find_between(rname, begin, end)
            print(ncbi_tid)
            if qname in lca_map:
                new_taxon = tree.gg_lineage(ncbi_tid)
                current_rname = lca_map[qname]
                if current_rname and new_taxon:
                    if current_rname != new_taxon:
                        lca_map[qname] = lca_gg(current_rname, new_taxon)
            else:
                print(tree.gg_lineage(ncbi_tid))
                lca_map[qname] = tree.gg_lineage(ncbi_tid)

        lca_map = collapse(lca_map, depth)
        taxon_counts = Counter(filter(None, lca_map.values()))
        counts.append(taxon_counts)

    df = pd.DataFrame(counts, index=['#' + os.path.basename(sample)[:-3] for sample in sam_files])
    df.T.to_csv(os.path.join(output, 'taxon_counts.csv'))

if __name__ == '__main__':
    shogun_bt2_lca()
<commit_msg>Checking the tree<commit_after>#!/usr/bin/env python
from __future__ import print_function, division

import click
from collections import Counter
import os
import pandas as pd

from ninja_utils.utils import find_between
from ninja_utils.utils import verify_make_dir

from ninja_dojo.taxonomy import NCBITree

from ninja_shogun.aligners.bowtie import bowtie2

def yield_alignments_from_sam_inf(inf):
    with open(inf) as fh:
        for i in fh:
            line = i.split('\t')
            # this function yields qname, rname
            try:
                yield line[0], line[2]
            except IndexError:
                print('Incorrect SAM input %s' % (inf))

def lca_gg(taxonomy_a, taxonomy_b):
    taxonomy_1 = taxonomy_a.split(';')
    taxonomy_2 = taxonomy_b.split(';')
    lca = []
    for i in zip(taxonomy_1, taxonomy_2):
        if i[0] == i[1]:
            lca.append(i[0])
        else:
            break
    if lca:
        return ';'.join(lca)

def collapse(lca_map, depth):
    if depth > 0:
        for name in lca_map:
            taxonomy = lca_map[name]
            if taxonomy:
                taxonomy = taxonomy.split(';')
                if len(taxonomy) < depth:
                    lca_map[name] = None
                elif len(taxonomy) > depth:
                    lca_map[name] = ';'.join(taxonomy[:depth])
        return lca_map
    else:
        return lca_map

@click.command()
@click.option('-i', '--input', type=click.Path(), default=os.getcwd())
@click.option('-o', '--output', type=click.Path(), default=os.getcwd())
@click.option('-b', '--bt2_indx')
@click.option('-x', '--extract_ncbi_tid', default='ncbi_tid|,|')
@click.option('-d', '--depth', type=click.INT, default=7, help='The depth of the search (7=species default, 0=No Collapse)')
@click.option('-p', '--threads', type=click.INT, default=1)
def shogun_bt2_lca(input, output, bt2_indx, extract_ncbi_tid, depth, threads):
    verify_make_dir(output)

    fna_files = [os.path.join(input, filename) for filename in os.listdir(input) if filename.endswith('.fna')]

    for fna_file in fna_files:
        sam_outf = os.path.join(output, '.'.join(str(os.path.basename(fna_file)).split('.')[:-1]) + '.sam')
        print(bowtie2(fna_file, sam_outf, bt2_indx, num_threads=threads))

    tree = NCBITree()
    begin, end = extract_ncbi_tid.split(',')

    counts = []
    sam_files = [os.path.join(output, filename) for filename in os.listdir(output) if filename.endswith('.sam')]

    for sam_file in sam_files:
        lca_map = {}
        for qname, rname in yield_alignments_from_sam_inf(sam_file):
            ncbi_tid = find_between(rname, begin, end)
            print(ncbi_tid)
            if qname in lca_map:
                new_taxon = tree.gg_lineage(ncbi_tid)
                current_rname = lca_map[qname]
                if current_rname and new_taxon:
                    if current_rname != new_taxon:
                        lca_map[qname] = lca_gg(current_rname, new_taxon)
            else:
                print(tree.gg_lineage(741158))
                lca_map[qname] = tree.gg_lineage(ncbi_tid)

        lca_map = collapse(lca_map, depth)
        taxon_counts = Counter(filter(None, lca_map.values()))
        counts.append(taxon_counts)

    df = pd.DataFrame(counts, index=['#' + os.path.basename(sample)[:-3] for sample in sam_files])
    df.T.to_csv(os.path.join(output, 'taxon_counts.csv'))

if __name__ == '__main__':
    shogun_bt2_lca()
<|endoftext|>