<commit_before># Copyright 2015 Hewlett-Packard Development Company, L.P.
# Copyright 2016 Rackspace Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import time

from neutron_lbaas.services.loadbalancer import constants as lb_const
from tempest import config
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
from tempest import test

from f5lbaasdriver.test.tempest.tests.api import base

CONF = config.CONF

class MemberStatusTestJSON(base.BaseTestCase):
    """Test that pool member's operating status is set correctly."""

    @classmethod
    def resource_setup(cls):
        """Setup client fixtures for test suite."""
        super(MemberStatusTestJSON, cls).resource_setup()
        if not test.is_extension_enabled('lbaas', 'network'):
            msg = "lbaas extension not enabled."
            raise cls.skipException(msg)
        network_name = data_utils.rand_name('network')
        cls.network = cls.create_network(network_name)
        cls.subnet = cls.create_subnet(cls.network)

        cls.create_lb_kwargs = {'tenant_id': cls.subnet['tenant_id'],
                                'vip_subnet_id': cls.subnet['id']}
        cls.load_balancer = \
            cls._create_active_load_balancer(**cls.create_lb_kwargs)
        cls.load_balancer_id = cls.load_balancer['id']

        # create listener
        listener_kwargs = {'loadbalancer_id': cls.load_balancer_id,
                           'protocol': 'HTTP',
                           'protocol_port': 80}
        cls.listener = cls._create_listener(**listener_kwargs)

        # get an RPC client for calling into driver
        cls.client = cls.plugin_rpc.get_client()
        cls.context = cls.plugin_rpc.get_context()

    @classmethod
    def resource_cleanup(cls):
        super(MemberStatusTestJSON, cls).resource_cleanup()

    @test.attr(type='smoke')
    def test_no_monitor(self):
        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        # create member
        allocation_pool = self.subnet['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet['id']}
        member = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member.get('id'))

        # Because pool does not have monitor, expect NO_MONITOR status
        expected_status = lb_const.NO_MONITOR
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

    @test.attr(type='smoke')
    def test_online(self):
        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        #  add ping monitor
        ping_monitor_kwargs = {'pool_id': pool['id'],
                               'type': 'PING',
                               'delay': 1,
                               'timeout': 1,
                               'max_retries': 5}
        self._create_health_monitor(**ping_monitor_kwargs)

        # create member
        allocation_pool = self.subnet['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet['id']}
        member = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member.get('id'))

        # Because monitor is PING, expect monitor to verify member up
        expected_status = "ONLINE"
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

    @test.attr(type='smoke')
    def test_offline(self):
        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        #  add HTTP monitor
        ping_monitor_kwargs = {'pool_id': pool['id'],
                               'type': 'HTTP',
                               'delay': 1,
                               'timeout': 1,
                               'max_retries': 5}
        self._create_health_monitor(**ping_monitor_kwargs)

        # create member
        allocation_pool = self.subnet['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet['id']}
        member = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member.get('id'))

        # Because monitor is HTTP, never expect monitor to verify member up
        expected_status = lb_const.OFFLINE
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

        # verify degraded state for loadbalancer
        status_tree = self.load_balancers_client.\
            get_load_balancer_status_tree(self.load_balancer_id)
        lb_status = status_tree.get('loadbalancer').get('operating_status')
        assert lb_status == lb_const.DEGRADED

    @decorators.skip_because(bug="497")
    def test_disabled(self):
        """Test setting admin state down.

        Skipped because setting admin state is not currently supported. Remove
        decorator when issue has been fixed.
        """
        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        #  add ping monitor
        ping_monitor_kwargs = {'pool_id': pool['id'],
                               'type': 'PING',
                               'delay': 1,
                               'timeout': 1,
                               'max_retries': 5}
        self._create_health_monitor(**ping_monitor_kwargs)

        # create member
        allocation_pool = self.subnet['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet['id']}
        member = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member.get('id'))

        # verify member up
        expected_status = lb_const.ONLINE
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

        # Set admin down -- status should change to disabled
        member_opts = {"admin_state_up": False}
        member = self._update_member(pool.get('id'),
                                     member.get('id'),
                                     **member_opts)
        expected_status = lb_const.OFFLINE
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

    @test.attr(type='smoke')
    def _wait_for_member_status(self, member, expected_status):
        """Query member operating status until it changes to expected status.

        :param member: member to query status.
        :param expected_status: Status to wait for.
        :return: Final status, either from changing to expected_status, or
        result after timing out.
        """
        status = ""
        count = 0
        while status != expected_status and count < 60:
            count += 1
            time.sleep(2)
            svc = self.client.call(self.context,
                                   'get_service_by_loadbalancer_id',
                                   loadbalancer_id=self.load_balancer_id)
            m = self._get_member_from_service(svc, member['id'])
            status = m["operating_status"]

        return status

    @staticmethod
    def _get_member_from_service(svc, member_id):
        for m in svc['members']:
            if m['id'] == member_id:
                return m
        return None
<commit_msg>Change copyright statement<commit_after># Copyright 2017 F5 Networks Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import time

from neutron_lbaas.services.loadbalancer import constants as lb_const
from tempest import config
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
from tempest import test

from f5lbaasdriver.test.tempest.tests.api import base

CONF = config.CONF

class MemberStatusTestJSON(base.BaseTestCase):
    """Test that pool member's operating status is set correctly."""

    @classmethod
    def resource_setup(cls):
        """Setup client fixtures for test suite."""
        super(MemberStatusTestJSON, cls).resource_setup()
        if not test.is_extension_enabled('lbaas', 'network'):
            msg = "lbaas extension not enabled."
            raise cls.skipException(msg)
        network_name = data_utils.rand_name('network')
        cls.network = cls.create_network(network_name)
        cls.subnet = cls.create_subnet(cls.network)

        cls.create_lb_kwargs = {'tenant_id': cls.subnet['tenant_id'],
                                'vip_subnet_id': cls.subnet['id']}
        cls.load_balancer = \
            cls._create_active_load_balancer(**cls.create_lb_kwargs)
        cls.load_balancer_id = cls.load_balancer['id']

        # create listener
        listener_kwargs = {'loadbalancer_id': cls.load_balancer_id,
                           'protocol': 'HTTP',
                           'protocol_port': 80}
        cls.listener = cls._create_listener(**listener_kwargs)

        # get an RPC client for calling into driver
        cls.client = cls.plugin_rpc.get_client()
        cls.context = cls.plugin_rpc.get_context()

    @classmethod
    def resource_cleanup(cls):
        super(MemberStatusTestJSON, cls).resource_cleanup()

    @test.attr(type='smoke')
    def test_no_monitor(self):
        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        # create member
        allocation_pool = self.subnet['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet['id']}
        member = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member.get('id'))

        # Because pool does not have monitor, expect NO_MONITOR status
        expected_status = lb_const.NO_MONITOR
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

    @test.attr(type='smoke')
    def test_online(self):
        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        #  add ping monitor
        ping_monitor_kwargs = {'pool_id': pool['id'],
                               'type': 'PING',
                               'delay': 1,
                               'timeout': 1,
                               'max_retries': 5}
        self._create_health_monitor(**ping_monitor_kwargs)

        # create member
        allocation_pool = self.subnet['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet['id']}
        member = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member.get('id'))

        # Because monitor is PING, expect monitor to verify member up
        expected_status = "ONLINE"
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

    @test.attr(type='smoke')
    def test_offline(self):
        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        #  add HTTP monitor
        ping_monitor_kwargs = {'pool_id': pool['id'],
                               'type': 'HTTP',
                               'delay': 1,
                               'timeout': 1,
                               'max_retries': 5}
        self._create_health_monitor(**ping_monitor_kwargs)

        # create member
        allocation_pool = self.subnet['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet['id']}
        member = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member.get('id'))

        # Because monitor is HTTP, never expect monitor to verify member up
        expected_status = lb_const.OFFLINE
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

        # verify degraded state for loadbalancer
        status_tree = self.load_balancers_client.\
            get_load_balancer_status_tree(self.load_balancer_id)
        lb_status = status_tree.get('loadbalancer').get('operating_status')
        assert lb_status == lb_const.DEGRADED

    @decorators.skip_because(bug="497")
    def test_disabled(self):
        """Test setting admin state down.

        Skipped because setting admin state is not currently supported. Remove
        decorator when issue has been fixed.
        """
        # create pool
        pool_kwargs = {'listener_id': self.listener['id'],
                       'protocol': 'HTTP',
                       'lb_algorithm': 'ROUND_ROBIN'}
        pool = self._create_pool(**pool_kwargs)
        self.addCleanup(self._delete_pool, pool.get('id'))

        #  add ping monitor
        ping_monitor_kwargs = {'pool_id': pool['id'],
                               'type': 'PING',
                               'delay': 1,
                               'timeout': 1,
                               'max_retries': 5}
        self._create_health_monitor(**ping_monitor_kwargs)

        # create member
        allocation_pool = self.subnet['allocation_pools'][0]
        member_kwargs = {'pool_id': pool['id'],
                         'address': allocation_pool['start'],
                         'protocol_port': 8080,
                         'subnet_id': self.subnet['id']}
        member = self._create_member(**member_kwargs)
        self.addCleanup(self._delete_member, pool.get('id'), member.get('id'))

        # verify member up
        expected_status = lb_const.ONLINE
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

        # Set admin down -- status should change to disabled
        member_opts = {"admin_state_up": False}
        member = self._update_member(pool.get('id'),
                                     member.get('id'),
                                     **member_opts)
        expected_status = lb_const.OFFLINE
        assert self._wait_for_member_status(member,
                                            expected_status) == expected_status

    @test.attr(type='smoke')
    def _wait_for_member_status(self, member, expected_status):
        """Query member operating status until it changes to expected status.

        :param member: member to query status.
        :param expected_status: Status to wait for.
        :return: Final status, either from changing to expected_status, or
        result after timing out.
        """
        status = ""
        count = 0
        while status != expected_status and count < 60:
            count += 1
            time.sleep(2)
            svc = self.client.call(self.context,
                                   'get_service_by_loadbalancer_id',
                                   loadbalancer_id=self.load_balancer_id)
            m = self._get_member_from_service(svc, member['id'])
            status = m["operating_status"]

        return status

    @staticmethod
    def _get_member_from_service(svc, member_id):
        for m in svc['members']:
            if m['id'] == member_id:
                return m
        return None
<|endoftext|>


<commit_before>from __future__ import division

def index_solution(tokens):
  from collections import namedtuple
  IndexSolution = namedtuple('IndexSolution',
                             'penalty sdcell fracn latt cell')

  cell = tuple(map(float, tokens[5:11]))
  return IndexSolution(int(tokens[1]), float(tokens[2]), float(tokens[3]),
                       tokens[4], cell)

def parse_index_log(mosflm_output):
  n_solutions = 0
  solutions = [ ]

  for record in mosflm_output:
    if 'DIRECT SPACE VECTORS DID NOT RESULT INTO A ORIENTATION' in record:
      from Exceptions import AutoindexError
      raise AutoindexError, 'indexing failed'

  for j, record in enumerate(mosflm_output):
    if ' No PENALTY SDCELL FRACN LATT      a        b        c' in record:
      k = j
      while n_solutions < 44:
        try:
          lattice_character = int(mosflm_output[k].split()[0])
          if not 'unrefined' in mosflm_output[k]:
            solutions.append(index_solution(
                mosflm_output[k].split()))

          n_solutions += 1
          k += 1
        except ValueError, e:
          k += 1
          continue
        except IndexError, e:
          k += 1
          continue

  return solutions

def set_distance(detector, distance):
  from scitbx import matrix
  import math
  assert len(detector) == 1
  panel = detector[0]
  d_normal = matrix.col(panel.get_normal())
  d_origin = matrix.col(panel.get_origin())
  assert d_origin.dot(d_normal) == panel.get_distance()
  translation = d_normal * (distance - panel.get_distance())
  new_origin = d_origin + translation
  assert math.fabs(new_origin.dot(d_normal) - distance) < 0.001
  fast = panel.get_fast_axis()
  slow = panel.get_slow_axis()
  panel.set_frame(panel.get_fast_axis(), panel.get_slow_axis(), new_origin.elems)
  assert matrix.col(panel.get_fast_axis()).angle(
    matrix.col(fast), deg=True) < 1e-6
  assert matrix.col(panel.get_slow_axis()).angle(
    matrix.col(slow), deg=True) < 1e-6
  assert math.fabs(panel.get_distance() - distance) < 0.001

def crystal_model_from_mosflm_mat(mosflm_mat_lines, unit_cell, space_group):
  from scitbx import matrix
  from cctbx import uctbx
  if not isinstance(unit_cell, uctbx.unit_cell):
    unit_cell = uctbx.unit_cell(unit_cell)
  from dxtbx.model.crystal import crystal_model_from_mosflm_matrix
  mosflm_matrix = matrix.sqr([float(i) for line in mosflm_mat_lines[:3]
                              for i in line.split()][:9])
  crystal_model = crystal_model_from_mosflm_matrix(
    mosflm_matrix,
    unit_cell=unit_cell,
    space_group=space_group)
  return crystal_model
<commit_msg>add debugging information to assertions<commit_after>from __future__ import division

def index_solution(tokens):
  from collections import namedtuple
  IndexSolution = namedtuple('IndexSolution',
                             'penalty sdcell fracn latt cell')

  cell = tuple(map(float, tokens[5:11]))
  return IndexSolution(int(tokens[1]), float(tokens[2]), float(tokens[3]),
                       tokens[4], cell)

def parse_index_log(mosflm_output):
  n_solutions = 0
  solutions = [ ]

  for record in mosflm_output:
    if 'DIRECT SPACE VECTORS DID NOT RESULT INTO A ORIENTATION' in record:
      from Exceptions import AutoindexError
      raise AutoindexError, 'indexing failed'

  for j, record in enumerate(mosflm_output):
    if ' No PENALTY SDCELL FRACN LATT      a        b        c' in record:
      k = j
      while n_solutions < 44:
        try:
          lattice_character = int(mosflm_output[k].split()[0])
          if not 'unrefined' in mosflm_output[k]:
            solutions.append(index_solution(
                mosflm_output[k].split()))

          n_solutions += 1
          k += 1
        except ValueError, e:
          k += 1
          continue
        except IndexError, e:
          k += 1
          continue

  return solutions

def set_distance(detector, distance):
  from scitbx import matrix
  import math
  assert len(detector) == 1, len(detector)
  panel = detector[0]
  d_normal = matrix.col(panel.get_normal())
  d_origin = matrix.col(panel.get_origin())
  assert d_origin.dot(d_normal) == panel.get_distance(), (
    (d_origin.d_normal), panel.get_distance())
  translation = d_normal * (distance - panel.get_distance())
  new_origin = d_origin + translation
  delta = math.fabs(new_origin.dot(d_normal) - distance)
  assert delta < 0.001, delta
  fast = panel.get_fast_axis()
  slow = panel.get_slow_axis()
  panel.set_frame(panel.get_fast_axis(), panel.get_slow_axis(), new_origin.elems)
  d_fast = matrix.col(panel.get_fast_axis()).angle(matrix.col(fast), deg=True)
  assert d_fast < 1e-6, d_fast
  d_slow = matrix.col(panel.get_slow_axis()).angle(matrix.col(slow), deg=True)
  assert d_slow < 1e-6, d_slow
  d_distance = math.fabs(panel.get_distance() - distance)
  assert d_distance < 0.001, d_distance

def crystal_model_from_mosflm_mat(mosflm_mat_lines, unit_cell, space_group):
  from scitbx import matrix
  from cctbx import uctbx
  if not isinstance(unit_cell, uctbx.unit_cell):
    unit_cell = uctbx.unit_cell(unit_cell)
  from dxtbx.model.crystal import crystal_model_from_mosflm_matrix
  mosflm_matrix = matrix.sqr([float(i) for line in mosflm_mat_lines[:3]
                              for i in line.split()][:9])
  crystal_model = crystal_model_from_mosflm_matrix(
    mosflm_matrix,
    unit_cell=unit_cell,
    space_group=space_group)
  return crystal_model
<|endoftext|>


<commit_before>"""BridgeGroupInterfaces class."""

from fmcapi.api_objects.apiclasstemplate import APIClassTemplate
from fmcapi.api_objects.object_services.securityzones import SecurityZones
from .devicerecords import DeviceRecords
from .physicalinterfaces import PhysicalInterfaces
import logging

class BridgeGroupInterfaces(APIClassTemplate):
    """The Bridge Group Interface Object in the FMC."""

    VALID_JSON_DATA = [
        "id",
        "name",
        "type",
        "mode",
        "enabled",
        "MTU",
        "managementOnly",
        "ipAddress",
        "selectedInterfaces",
        "bridgeGroupId",
        "macLearn",
        "ifname",
        "securityZone",
        "arpConfig",
        "ipv4",
        "ipv6",
        "macTable",
        "enableAntiSpoofing",
        "fragmentReassembly",
        "enableDNSLookup",
        "activeMACAddress",
        "standbyMACAddress",
    ]
    VALID_FOR_KWARGS = VALID_JSON_DATA + ["device_name"]
    VALID_CHARACTERS_FOR_NAME = """[.\w\d_\-\/\. ]"""
    PREFIX_URL = "/devices/devicerecords"
    URL_SUFFIX = None
    REQUIRED_FOR_POST = ["bridgeGroupId"]
    REQUIRED_FOR_PUT = ["id", "device_id"]
    VALID_FOR_IPV4 = ["static", "dhcp", "pppoe"]
    VALID_FOR_MODE = ["INLINE", "PASSIVE", "TAP", "ERSPAN", "NONE"]
    VALID_FOR_MTU = range(64, 9000)

    def __init__(self, fmc, **kwargs):
        """
        Initialize BridgeGroupInterfaces object.

        Set self.type to "BridgeGroupInterface" and parse the kwargs.

        :param fmc (object): FMC object
        :param **kwargs: Any other values passed during instantiation.
        :return: None
        """
        super().__init__(fmc, **kwargs)
        logging.debug("In __init__() for BridgeGroupInterfaces class.")
        self.parse_kwargs(**kwargs)
        self.type = "BridgeGroupInterface"

    def parse_kwargs(self, **kwargs):
        """
        Parse the kwargs and set self variables to match.

        :return: None
        """
        super().parse_kwargs(**kwargs)
        logging.debug("In parse_kwargs() for BridgeGroupInterfaces class.")
        if "device_name" in kwargs:
            self.device(device_name=kwargs["device_name"])
        if "ipv4" in kwargs:
            if list(kwargs["ipv4"].keys())[0] in self.VALID_FOR_IPV4:
                self.ipv4 = kwargs["ipv4"]
            else:
                logging.warning(f"Method {kwargs['ipv4']} is not a valid ipv4 type.")
        if "mode" in kwargs:
            if kwargs["mode"] in self.VALID_FOR_MODE:
                self.mode = kwargs["mode"]
            else:
                logging.warning(f"Mode {kwargs['mode']} is not a valid mode.")
        if "MTU" in kwargs:
            if kwargs["MTU"] in self.VALID_FOR_MTU:
                self.MTU = kwargs["MTU"]
            else:
                logging.warning(f"MTU {kwargs['MTU']} should be in the range 64-9000.")
                self.MTU = 1500

    def device(self, device_name):
        """
        Associate device to this group.

        :param device_name: (str) Name of device.
        :return: None
        """
        logging.debug("In device() for BridgeGroupInterfaces class.")
        device1 = DeviceRecords(fmc=self.fmc)
        device1.get(name=device_name)
        if "id" in device1.__dict__:
            self.device_id = device1.id
            self.URL = f"{self.fmc.configuration_url}{self.PREFIX_URL}/{self.device_id}/bridgegroupinterfaces"
            self.device_added_to_url = True
        else:
            logging.warning(
                f"Device {device_name} not found.  Cannot set up device for BridgeGroupInterfaces."
            )

    def sz(self, name):
        """
        Assign Security Zone to this group.

        :param name: (str) Name of Security Zone.
        :return: None
        """
        logging.debug("In sz() for BridgeGroupInterfaces class.")
        sz = SecurityZones(fmc=self.fmc)
        sz.get(name=name)
        if "id" in sz.__dict__:
            new_zone = {"name": sz.name, "id": sz.id, "type": sz.type}
            self.securityZone = new_zone
        else:
            logging.warning(
                f'Security Zone, "{name}", not found.  Cannot add to BridgeGroupInterfaces.'
            )

    def static(self, ipv4addr, ipv4mask):
        """
        Assign static IP to this bridge group.

        :param ipv4addr: (str) x.x.x.x
        :param ipv4mask: (str) bitmask
        :return: None
        """
        logging.debug("In static() for BridgeGroupInterfaces class.")
        self.ipv4 = {"static": {"address": ipv4addr, "netmask": ipv4mask}}

    def dhcp(self, enableDefault=True, routeMetric=1):
        """
        Configure this group with DHCP for addressing.

        :param enableDefault: (bool) Accept, or not, a default route via DHCP.
        :param routeMetric: (int) Set route metric.
        :return: None
        """
        logging.debug("In dhcp() for BridgeGroupInterfaces class.")
        self.ipv4 = {
            "dhcp": {
                "enableDefaultRouteDHCP": enableDefault,
                "dhcpRouteMetric": routeMetric,
            }
        }

    def p_interfaces(self, p_interfaces, device_name):
        """
        Define which physical interface on which device is a part of this bridge group.

        :param p_interfaces: (str) Name of physical interface.
        :param device_name: (str) Name of device with that interface.
        :return: None
        """
        logging.debug("In p_interface() for BridgeGroupInterfaces class.")
        list1 = []
        for p_intf in p_interfaces:
            intf1 = PhysicalInterfaces(fmc=self.fmc)
            intf1.get(name=p_intf, device_name=device_name)
            if "id" in intf1.__dict__:
                list1.append({"name": intf1.name, "id": intf1.id, "type": intf1.type})
            else:
                logging.warning(
                    f'PhysicalInterface, "{intf1.name}", not found.  Cannot add to BridgeGroupInterfaces.'
                )
        self.selectedInterfaces = list1
<commit_msg>Update bridgegroupinterfaces.py<commit_after>"""BridgeGroupInterfaces class."""

from fmcapi.api_objects.apiclasstemplate import APIClassTemplate
from fmcapi.api_objects.object_services.securityzones import SecurityZones
from .devicerecords import DeviceRecords
from .physicalinterfaces import PhysicalInterfaces
import logging

class BridgeGroupInterfaces(APIClassTemplate):
    """The Bridge Group Interface Object in the FMC."""

    VALID_JSON_DATA = [
        "id",
        "name",
        "type",
        "mode",
        "enabled",
        "MTU",
        "managementOnly",
        "ipAddress",
        "selectedInterfaces",
        "bridgeGroupId",
        "macLearn",
        "ifname",
        "securityZone",
        "arpConfig",
        "ipv4",
        "ipv6",
        "macTable",
        "enableAntiSpoofing",
        "fragmentReassembly",
        "enableDNSLookup",
        "activeMACAddress",
        "standbyMACAddress",
    ]
    VALID_FOR_KWARGS = VALID_JSON_DATA + ["device_name"]
    VALID_CHARACTERS_FOR_NAME = """[.\w\d_\-\/\. ]"""
    PREFIX_URL = "/devices/devicerecords"
    URL_SUFFIX = None
    REQUIRED_FOR_POST = ["bridgeGroupId"]
    REQUIRED_FOR_PUT = ["id", "device_id"]
    VALID_FOR_IPV4 = ["static", "dhcp", "pppoe"]
    VALID_FOR_MODE = ["INLINE", "PASSIVE", "TAP", "ERSPAN", "NONE"]
    VALID_FOR_MTU = range(64, 9085)

    def __init__(self, fmc, **kwargs):
        """
        Initialize BridgeGroupInterfaces object.

        Set self.type to "BridgeGroupInterface" and parse the kwargs.

        :param fmc (object): FMC object
        :param **kwargs: Any other values passed during instantiation.
        :return: None
        """
        super().__init__(fmc, **kwargs)
        logging.debug("In __init__() for BridgeGroupInterfaces class.")
        self.parse_kwargs(**kwargs)
        self.type = "BridgeGroupInterface"

    def parse_kwargs(self, **kwargs):
        """
        Parse the kwargs and set self variables to match.

        :return: None
        """
        super().parse_kwargs(**kwargs)
        logging.debug("In parse_kwargs() for BridgeGroupInterfaces class.")
        if "device_name" in kwargs:
            self.device(device_name=kwargs["device_name"])
        if "ipv4" in kwargs:
            if list(kwargs["ipv4"].keys())[0] in self.VALID_FOR_IPV4:
                self.ipv4 = kwargs["ipv4"]
            else:
                logging.warning(f"Method {kwargs['ipv4']} is not a valid ipv4 type.")
        if "mode" in kwargs:
            if kwargs["mode"] in self.VALID_FOR_MODE:
                self.mode = kwargs["mode"]
            else:
                logging.warning(f"Mode {kwargs['mode']} is not a valid mode.")
        if "MTU" in kwargs:
            if kwargs["MTU"] in self.VALID_FOR_MTU:
                self.MTU = kwargs["MTU"]
            else:
                logging.warning(f"MTU {kwargs['MTU']} should be in the range 64-9000.")
                self.MTU = 1500

    def device(self, device_name):
        """
        Associate device to this group.

        :param device_name: (str) Name of device.
        :return: None
        """
        logging.debug("In device() for BridgeGroupInterfaces class.")
        device1 = DeviceRecords(fmc=self.fmc)
        device1.get(name=device_name)
        if "id" in device1.__dict__:
            self.device_id = device1.id
            self.URL = f"{self.fmc.configuration_url}{self.PREFIX_URL}/{self.device_id}/bridgegroupinterfaces"
            self.device_added_to_url = True
        else:
            logging.warning(
                f"Device {device_name} not found.  Cannot set up device for BridgeGroupInterfaces."
            )

    def sz(self, name):
        """
        Assign Security Zone to this group.

        :param name: (str) Name of Security Zone.
        :return: None
        """
        logging.debug("In sz() for BridgeGroupInterfaces class.")
        sz = SecurityZones(fmc=self.fmc)
        sz.get(name=name)
        if "id" in sz.__dict__:
            new_zone = {"name": sz.name, "id": sz.id, "type": sz.type}
            self.securityZone = new_zone
        else:
            logging.warning(
                f'Security Zone, "{name}", not found.  Cannot add to BridgeGroupInterfaces.'
            )

    def static(self, ipv4addr, ipv4mask):
        """
        Assign static IP to this bridge group.

        :param ipv4addr: (str) x.x.x.x
        :param ipv4mask: (str) bitmask
        :return: None
        """
        logging.debug("In static() for BridgeGroupInterfaces class.")
        self.ipv4 = {"static": {"address": ipv4addr, "netmask": ipv4mask}}

    def dhcp(self, enableDefault=True, routeMetric=1):
        """
        Configure this group with DHCP for addressing.

        :param enableDefault: (bool) Accept, or not, a default route via DHCP.
        :param routeMetric: (int) Set route metric.
        :return: None
        """
        logging.debug("In dhcp() for BridgeGroupInterfaces class.")
        self.ipv4 = {
            "dhcp": {
                "enableDefaultRouteDHCP": enableDefault,
                "dhcpRouteMetric": routeMetric,
            }
        }

    def p_interfaces(self, p_interfaces, device_name):
        """
        Define which physical interface on which device is a part of this bridge group.

        :param p_interfaces: (str) Name of physical interface.
        :param device_name: (str) Name of device with that interface.
        :return: None
        """
        logging.debug("In p_interface() for BridgeGroupInterfaces class.")
        list1 = []
        for p_intf in p_interfaces:
            intf1 = PhysicalInterfaces(fmc=self.fmc)
            intf1.get(name=p_intf, device_name=device_name)
            if "id" in intf1.__dict__:
                list1.append({"name": intf1.name, "id": intf1.id, "type": intf1.type})
            else:
                logging.warning(
                    f'PhysicalInterface, "{intf1.name}", not found.  Cannot add to BridgeGroupInterfaces.'
                )
        self.selectedInterfaces = list1
<|endoftext|>


<commit_before>from django.conf.urls import patterns, url
from django.http import HttpResponse
from django.views.generic.base import View
from django.shortcuts import get_object_or_404
import json, csv
from django.forms.models import model_to_dict
from django.views.decorators.csrf import csrf_exempt

from thingstore.models import Thing

class APIView(View):
	filetype = "json"

	@csrf_exempt
	def dispatch(self, *args, **kwargs):
		return super(APIView, self).dispatch(*args, **kwargs)

	def getJSON(self, request, **kwargs):
		pass

	def getCSV(self, request, **kwargs):
		pass

	def get(self, request, **kwargs):
		if self.filetype == "json":
			data = self.getJSON(request, **kwargs)
			return HttpResponse(data, content_type="application/json")
		elif self.filetype == "csv":
			data = self.getCSV(request, **kwargs)
			return HttpResponse(data, content_type="text/plain")

	def putJSON(self, request, **kwargs):
		pass

	def putCSV(self, request, **kwargs):
		pass

	def put(self, request, **kwargs):
		if self.filetype == "json":
			data = self.putJson(request, **kwargs)
			return HttpResponse(data, content_type="application/json")
		elif self.filetype == "csv":
			data = self.putCSV(request, **kwargs)
			return HttpResponse(data, content_type="text/plain")

class ThingAPI(APIView):
	def getJSON(self, request, **kwargs):
		thing = get_object_or_404(Thing, pk=kwargs["thing_id"])

		data = model_to_dict(thing)

		data['metrics'] = {}
		metrics = thing.metrics.all()
		for metric in metrics:
			data['metrics'][metric.name] = model_to_dict(metric,fields=['name','unit'])
			data['metrics'][metric.name]['current_value'] = metric.current_value
			data['metrics'][metric.name]['current_value'] = metric.current_value
		return json.dumps(data)

	def getCSV(self, request, **kwargs):
		thing = get_object_or_404(Thing, pk=kwargs["thing_id"])
		metrics = thing.metrics.all()
		data = ""
		for metric in metrics:
			data = "%s%s,%s\n" % (data, metric.name, metric.current_value)
		return data

	def putCSV(self, request, **kwargs):
		body = request.body
		lines = body.split('\n')
		data = []
		for line in lines:
			line_data = line.split(',')
			if len(line_data) != 2:
				return "ERROR\n"
			data.append(line_data)
		print data
		return "troet\n"

def metric(request):

	pass

urls = patterns('',
	url(r'^thing/(?P<thing_id>\w+).json', ThingAPI.as_view(filetype="json")),
	url(r'^thing/(?P<thing_id>\w+).csv', ThingAPI.as_view(filetype="csv")),
)
<commit_msg>added some comments<commit_after>from django.conf.urls import patterns, url
from django.http import HttpResponse
from django.views.generic.base import View
from django.shortcuts import get_object_or_404
import json, csv
from django.forms.models import model_to_dict
from django.views.decorators.csrf import csrf_exempt

from thingstore.models import Thing

class APIView(View):
	filetype = "json"

	# Required because of django's csrf protection
	@csrf_exempt
	def dispatch(self, *args, **kwargs):
		return super(APIView, self).dispatch(*args, **kwargs)

	# abstract call when a json is requested
	def getJSON(self, request, **kwargs):
		pass

	# abstract call when a csv is requested
	def getCSV(self, request, **kwargs):
		pass

	# dispatches GET requests depending on requested file type
	def get(self, request, **kwargs):
		if self.filetype == "json":
			data = self.getJSON(request, **kwargs)
			return HttpResponse(data, content_type="application/json")
		elif self.filetype == "csv":
			data = self.getCSV(request, **kwargs)
			return HttpResponse(data, content_type="text/plain")

	# abstract call when a JSON is sent as a PUT request
	def putJSON(self, request, **kwargs):
		pass

	# abstract call when a CSV is sent as a PUT request
	def putCSV(self, request, **kwargs):
		pass

	# dispatches PUT requests
	def put(self, request, **kwargs):
		if self.filetype == "json":
			data = self.putJson(request, **kwargs)
			return HttpResponse(data, content_type="application/json")
		elif self.filetype == "csv":
			data = self.putCSV(request, **kwargs)
			return HttpResponse(data, content_type="text/plain")

# API resource of Things
class ThingAPI(APIView):

	# Return a JSON describing the thing, with current values of the metrics
	def getJSON(self, request, **kwargs):
		thing = get_object_or_404(Thing, pk=kwargs["thing_id"])

		data = model_to_dict(thing)

		data['metrics'] = {}
		metrics = thing.metrics.all()
		for metric in metrics:
			data['metrics'][metric.name] = model_to_dict(metric,fields=['name','unit'])
			data['metrics'][metric.name]['current_value'] = metric.current_value
			data['metrics'][metric.name]['current_value'] = metric.current_value
		return json.dumps(data)

	# Return a CSV with current values of all metrics of the thing
	def getCSV(self, request, **kwargs):
		thing = get_object_or_404(Thing, pk=kwargs["thing_id"])
		metrics = thing.metrics.all()
		data = ""
		for metric in metrics:
			data = "%s%s,%s\n" % (data, metric.name, metric.current_value)
		return data

	# Update a thing's metrics. Eventually, at least.
	def putCSV(self, request, **kwargs):
		body = request.body
		lines = body.split('\n')
		data = []
		for line in lines:
			line_data = line.split(',')
			if len(line_data) != 2:
				return "ERROR\n"
			data.append(line_data)
		print data
		return "troet\n"

def metric(request):

	pass

urls = patterns('',
	url(r'^thing/(?P<thing_id>\w+).json', ThingAPI.as_view(filetype="json")),
	url(r'^thing/(?P<thing_id>\w+).csv', ThingAPI.as_view(filetype="csv")),
)
<|endoftext|>


<commit_before>
"""
test_utils
----------

"""

import numpy as np

from ..utils.sampling_utils import create_times_randompoints,\
    create_times_randomregimes, create_times_randombursts
from ..utils.util_operations import join_regimes, format_as_regular_ts,\
    apply_gaussianconvolved_ts
from ..utils.sliding_utils import sliding_embeded_transf
from ..utils.fit_utils import general_multiscale_fit,\
    fit_loglogleastsquares

def test():
    create_times_randompoints(100)
    create_times_randomregimes(100, [.1, .2])
    ts0 = create_times_randombursts(np.array([0.]), [(100, 1.), (5, .01)])
    ts1 = create_times_randombursts(np.array([0.]), [(100, 1.)])

    #### Operations
    # Parameters
    tss = [ts0, ts1]
    regimes = [.4, 2.]
    intervals = (0, 100, .2)

    times, values = join_regimes(tss, regimes)
    gridtimes, gridvalues = format_as_regular_ts(times, values, intervals)
    apply_gaussianconvolved_ts(gridvalues, 5, .2)

    ### Sliding utils
    #################
    D, step = 2, 1
    tau = 3

    sliding_embeded_transf(values, tau, D,  step)
    try:
        tau = .4
        boolean = False
        sliding_embeded_transf(values, tau, D,  step)
        boolean = True
    except:
        if boolean:
            raise Exception()
    try:
        tau = len(values)
        boolean = False
        sliding_embeded_transf(values, tau, D,  step)
        boolean = True
    except:
        if boolean:
            raise Exception()

    ### Fit utils
    ##############
    x = np.random.random(1000).cumsum()
    y = x + np.random.randn(1000)
    general_multiscale_fit(x, y, 'loglogLSQ')
    fit_loglogleastsquares(x, y)
<commit_msg>Exploring tests: metric tests.<commit_after>
"""
test_utils
----------

"""

import numpy as np

from ..utils.sampling_utils import create_times_randompoints,\
    create_times_randomregimes, create_times_randombursts
from ..utils.util_operations import join_regimes, format_as_regular_ts,\
    apply_gaussianconvolved_ts
from ..utils.sliding_utils import sliding_embeded_transf
from ..utils.fit_utils import general_multiscale_fit,\
    fit_loglogleastsquares

def test():
    create_times_randompoints(100)
    create_times_randomregimes(100, [.1, .2])
    ts0 = create_times_randombursts(np.array([0.]), [(100, 1.), (5, .01)])
    ts1 = create_times_randombursts(np.array([0.]), [(100, 1.)])

    #### Operations
    # Parameters
    tss = [ts0, ts1]
    regimes = [.4, 2.]
    intervals = (0, 100, .2)

    times, values = join_regimes(tss, regimes)
    gridtimes, gridvalues = format_as_regular_ts(times, values, intervals)
    apply_gaussianconvolved_ts(gridvalues, 5, .2)

    ### Sliding utils
    #################
    D, step = 2, 1
    tau = 3

    sliding_embeded_transf(values, tau, D,  step)
    try:
        tau = .4
        boolean = False
        sliding_embeded_transf(values, tau, D,  step)
        boolean = True
    except:
        if boolean:
            raise Exception()
    try:
        tau = len(values)
        boolean = False
        sliding_embeded_transf(values, tau, D,  step)
        boolean = True
    except:
        if boolean:
            raise Exception()

    ### Fit utils
    ##############
#    x = np.random.random(1000).cumsum()
#    y = x + np.random.randn(1000)*0.01
#    general_multiscale_fit(x, y, 'loglogLSQ')
#    fit_loglogleastsquares(x, y)
<|endoftext|>


<commit_before>''' -- imports from installed packages -- '''
from django.http import StreamingHttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response, render
from django.template import RequestContext
from django.contrib.auth.models import User
from django.contrib.auth.decorators import user_passes_test

from gnowsys_ndf.settings import LANGUAGES
from gnowsys_ndf.ndf.models import node_collection, triple_collection
from gnowsys_ndf.ndf.views.methods import *

import json
import datetime

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClass(request, class_name='GSystemType'):
    '''
    fetching class's objects
    '''
    if request.method == "POST":
        search = request.POST.get("search","")
        classtype = request.POST.get("class","")
        nodes = node_collection.find({'name':{'$regex':search,'$options': 'i' },'_type':classtype}).sort('last_update', -1)
    else :
        nodes = node_collection.find({'_type':class_name}).sort('last_update', -1)

    objects_details = []
    for each in nodes:
        member = []
        member_of_list = []
        collection_list = []
        attribute_type_set = []
        relation_type_set = []
        for e in each.member_of:
            member_of_list.append(node_collection.one({'_id':e}).name+" ")

        for members in each.member_of:
            member.append(node_collection.one({ '_id': members}).name+" " )

        # for coll in each.collection_set:
        #     collection_list.append(node_collection.one({ '_id': coll}).name+" - "+str(coll))

        if class_name in ("GSystemType"):
            for at_set in each.attribute_type_set:
                attribute_type_set.append(at_set.name+" ")
            for rt_set in each.relation_type_set:
                relation_type_set.append(rt_set.name+" " )
            objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at,'member_of':",".join(member_of_list), "collection_list":",".join(collection_list), "attribute_type_set":",".join(attribute_type_set), "relation_type_set":",".join(relation_type_set)})
        else :
		objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at,'member_of':",".join(member_of_list), "collection_list":",".join(collection_list)})
    groups = []
    group = node_collection.find({'_type':"Group"})
    for each in group:
        groups.append({'id':each._id,"title":each.name})

    systemtypes = []
    systemtype = node_collection.find({'_type':"GSystemType"})
    for each in systemtype:
        systemtypes.append({'id':each._id,"title":each.name})

    meta_types = []
    meta_type = node_collection.find({'_type':"MetaType"})
    for each in meta_type:
        meta_types.append({'id':each._id,"title":each.name})

    groupid = ""
    group_obj= node_collection.find({'$and':[{"_type":u'Group'},{"name":u'home'}]})
    if group_obj:
	groupid = str(group_obj[0]._id)

    template = "ndf/adminDashboard.html"
    variable = RequestContext(request, {'class_name':class_name,"nodes":objects_details,"Groups":groups,"systemtypes":systemtypes,"url":"designer","groupid":groupid,'meta_types':meta_types,'group_id':groupid})
    return render_to_response(template, variable)

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClassCreate(request, class_name='GSystemType', node_id=None):
    '''
    delete class's objects
    '''
    global LANGUAGES
    new_instance_type = None
    LANGUAGES = '' if not LANGUAGES else LANGUAGES
    definitionlist = []
    contentlist = []
    dependencylist = []
    options = []
    translate=request.GET.get('translate','')
    if class_name == "AttributeType":
        definitionlist = ['name','altnames','language','subject_type','data_type','applicable_node_type','member_of','verbose_name','null','blank','help_text','max_digits','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        # definitionlist = ['name','altnames','language','subject_type','data_type','member_of','verbose_name','null','blank','help_text','max_digits','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "GSystemType":
        definitionlist = ['name','altnames','language','status','member_of','meta_type_set','attribute_type_set','relation_type_set','type_of']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "RelationType":
        definitionlist = ['name','inverse_name','altnames','language','subject_type','object_type','subject_cardinality','object_cardinality','subject_applicable_nodetype','object_applicable_nodetype','is_symmetric','is_reflexive','is_transitive','status','member_of']
        # definitionlist = ['name','inverse_name','altnames','language','subject_type','object_type','subject_cardinality','object_cardinality','is_symmetric','is_reflexive','is_transitive','status','member_of']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    else :
        definitionlist = []
        contentlist = []
        dependencylist = []
        options = []

    class_structure = eval(class_name).structure

    required_fields = eval(class_name).required_fields

    newdict = {}
    if node_id:
        new_instance_type = node_collection.one({'_type': unicode(class_name), '_id': ObjectId(node_id)})

    else:
        new_instance_type = eval("node_collection.collection"+"."+class_name)()

    if request.method=="POST":
        if translate:
            new_instance_type = eval("node_collection.collection"+"."+class_name)()
        # print new_instance_type
        for key,value in class_structure.items():
            if value == bool:
                if request.POST.get(key,""):
                    if request.POST.get(key,"") in ('1','2'):
                        if request.POST.get(key,"") == '1':
                            new_instance_type[key] = True
                        else :
                            new_instance_type[key] = False

            elif value == unicode:
                if request.POST.get(key,""):

                    if key == "content_org":
                        new_instance_type[key] = unicode(request.POST.get(key,""))
                        # Required to link temporary files with the current user who is modifying this document
                        usrname = request.user.username
                        filename = slugify(new_instance_type['name']) + "-" + usrname + "-"
                        new_instance_type['content'] = org2html(new_instance_type[key], file_prefix=filename)
                    else :
                        if translate:
                            if key in ("name","inverse_name"):
                                new_instance_type[key] = unicode(request.POST.get(key+"_trans",""))
                                language= request.POST.get('lan')
                                new_instance_type.language = get_language_tuple(language)

                            else:
                                new_instance_type[key] = unicode(request.POST.get(key,""))
                        else:
                            new_instance_type[key] = unicode(request.POST.get(key,""))

            elif value == list:
                if request.POST.get(key,""):
                    new_instance_type[key] = request.POST.get(key,"").split(",")

            elif type(value) == list:
                if request.POST.get(key,""):
                    if key in ("tags","applicable_node_type"):
                        new_instance_type[key] = request.POST.get(key,"").split(",")
                    elif key in ["meta_type_set","attribute_type_set","relation_type_set"]:
                        listoflist = []
                        for each in request.POST.get(key,"").split(","):
                            listoflist.append(node_collection.one({"_id":ObjectId(each)}))
                        new_instance_type[key] = listoflist
                    else :
                        listoflist = []

                        for each in request.POST.get(key,"").split(","):

                            listoflist.append(ObjectId(each))
                        new_instance_type[key] = listoflist

            elif type(value) == tuple:
                new_instance_type[key] = tuple(eval(request.POST.get(key,"")))

            elif value == datetime.datetime:
                if key == "last_update":
                    new_instance_type[key] = datetime.datetime.now()

            elif key == "status":
                if request.POST.get(key,""):
                    new_instance_type[key] = unicode(request.POST.get(key,""))

            # elif key == "created_by":
            #     new_instance_type[key] = request.user.id

            elif value == int:
                if request.POST.get(key,""):
                    new_instance_type[key] = int(request.POST.get(key,""))

            else:

                if request.POST.get(key,""):
                    new_instance_type[key] = request.POST.get(key,"")

        user_id = request.user.id
        if not new_instance_type.has_key('_id'):
            new_instance_type.created_by = user_id

        new_instance_type.modified_by = user_id

        if user_id not in new_instance_type.contributors:
            new_instance_type.contributors.append(user_id)
        parent_node=node_collection.one({'_id':ObjectId(node_id)})
        if translate and class_name == "RelationType":
            new_instance_type.subject_type = parent_node.subject_type
            new_instance_type.object_type = parent_node.object_type
        if translate and class_name == "AttributeType":
            new_instance_type.data_type = parent_node.data_type
            new_instance_type.subject_type = parent_node.subject_type

        new_instance_type.save()
        if translate:
            relation_type=node_collection.one({'$and':[{'name':'translation_of'},{'_type':'RelationType'}]})
            grelation=node_collection.collection.GRelation()
            grelation.relation_type=relation_type
            grelation.subject=new_instance_type['_id']
            grelation.right_subject=ObjectId(node_id)
            grelation.name=u""
            grelation.save()

        return HttpResponseRedirect("/admin/designer/"+class_name)

    # If GET request ---------------------------------------------------------------------------------------

    for key,value in class_structure.items():
        if value == bool:
            # newdict[key] = "bool"
            newdict[key] = ["bool", new_instance_type[key]]
        elif value == unicode:
            if key == "language":
                newdict[key] = ["list", new_instance_type[key]]
            else:
                # newdict[key] = "unicode"
                newdict[key] = ["unicode", new_instance_type[key]]

        elif value == list:
            # newdict[key] = "list"

            newdict[key] = ["list", new_instance_type[key]]
        elif type(value) == list:
            # newdict[key] = "list"
            newdict[key] = ["list", new_instance_type[key]]
        elif value == datetime.datetime:
            # newdict[key] = "datetime"
            newdict[key] = ["datetime", new_instance_type[key]]
        elif value == int:
            # newdict[key] = "int"
            newdict[key] = ["int", new_instance_type[key]]
        elif key == "status":
            # newdict[key] = "status"
            newdict[key] = ["status", new_instance_type[key]]
        else:
            # newdict[key] = value
            newdict[key] = [value, new_instance_type[key]]

    class_structure = newdict

    groupid = ""
    group_obj= node_collection.find({'$and':[{"_type":u'Group'},{"name":u'home'}]})
    if group_obj:
	groupid = str(group_obj[0]._id)

    template = "ndf/adminDashboardCreate.html"

    variable =  None
    class_structure_with_values = {}
    if node_id:

        for key, value in class_structure.items():
            class_structure_with_values[key] = [class_structure[key][0], new_instance_type[key]]

        variable = RequestContext(request, {'node': new_instance_type,
                                            'class_name': class_name, 'class_structure': class_structure_with_values, 'url': "designer",
                                            'definitionlist': definitionlist, 'contentlist': contentlist, 'dependencylist': dependencylist,
                                            'options': options, 'required_fields': required_fields,"translate":translate,"lan":LANGUAGES,
                                            'groupid': groupid,'group_id':groupid
                                        })

    else:

        variable = RequestContext(request, {'class_name':class_name, "url":"designer", "class_structure":class_structure, 'definitionlist':definitionlist, 'contentlist':contentlist, 'dependencylist':dependencylist, 'options':options, "required_fields":required_fields,"groupid":groupid,"translate":translate,"lan":LANGUAGES,'group_id':groupid })

    return render_to_response(template, variable)<commit_msg>Changes in the display of GSystemType Designer Dashboard few additions<commit_after>''' -- imports from installed packages -- '''
from django.http import StreamingHttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response, render
from django.template import RequestContext
from django.contrib.auth.models import User
from django.contrib.auth.decorators import user_passes_test

from gnowsys_ndf.settings import LANGUAGES
from gnowsys_ndf.ndf.models import node_collection, triple_collection
from gnowsys_ndf.ndf.views.methods import *

import json
import datetime

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClass(request, class_name='GSystemType'):
    '''
    fetching class's objects
    '''
    if request.method == "POST":
        search = request.POST.get("search","")
        classtype = request.POST.get("class","")
        nodes = node_collection.find({'name':{'$regex':search,'$options': 'i' },'_type':classtype}).sort('last_update', -1)
    else :
        nodes = node_collection.find({'_type':class_name}).sort('last_update', -1)

    objects_details = []
    for each in nodes:
        member = []
        member_of_list = []
        collection_list = []
        attribute_type_set = []
        relation_type_set = []
        for e in each.member_of:
            member_of_list.append(node_collection.one({'_id':e}).name+" ")
            # member_of_list.append(node_collection.one({'_id':e}).name+" - "+str(e))

        for members in each.member_of:
            member.append(node_collection.one({ '_id': members}).name+" ")
            # member.append(node_collection.one({ '_id': members}).name+" - "+str(members))

        # for coll in each.collection_set:
        #     collection_list.append(node_collection.one({ '_id': coll}).name+" - "+str(coll))

        if class_name in ("GSystemType"):
            for at_set in each.attribute_type_set:
                attribute_type_set.append(at_set.name+" ")
                # attribute_type_set.append(at_set.name+" - "+str(at_set._id))
            for rt_set in each.relation_type_set:
                relation_type_set.append(rt_set.name+" ")
                # relation_type_set.append(rt_set.name+" - "+str(rt_set._id))
            objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at,'member_of':",".join(member_of_list), "collection_list":",".join(collection_list), "attribute_type_set":",".join(attribute_type_set), "relation_type_set":",".join(relation_type_set)})
        else :
		objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at,'member_of':",".join(member_of_list), "collection_list":",".join(collection_list)})
    groups = []
    group = node_collection.find({'_type':"Group"})
    for each in group:
        groups.append({'id':each._id,"title":each.name})

    systemtypes = []
    systemtype = node_collection.find({'_type':"GSystemType"})
    for each in systemtype:
        systemtypes.append({'id':each._id,"title":each.name})

    meta_types = []
    meta_type = node_collection.find({'_type':"MetaType"})
    for each in meta_type:
        meta_types.append({'id':each._id,"title":each.name})

    groupid = ""
    group_obj= node_collection.find({'$and':[{"_type":u'Group'},{"name":u'home'}]})
    if group_obj:
	groupid = str(group_obj[0]._id)

    template = "ndf/adminDashboard.html"
    variable = RequestContext(request, {'class_name':class_name,"nodes":objects_details,"Groups":groups,"systemtypes":systemtypes,"url":"designer","groupid":groupid,'meta_types':meta_types,'group_id':groupid})
    return render_to_response(template, variable)

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClassCreate(request, class_name='GSystemType', node_id=None):
    '''
    delete class's objects
    '''
    global LANGUAGES
    new_instance_type = None
    LANGUAGES = '' if not LANGUAGES else LANGUAGES
    definitionlist = []
    contentlist = []
    dependencylist = []
    options = []
    translate=request.GET.get('translate','')
    if class_name == "AttributeType":
        definitionlist = ['name','altnames','language','subject_type','data_type','applicable_node_type','member_of','verbose_name','null','blank','help_text','max_digits','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        # definitionlist = ['name','altnames','language','subject_type','data_type','member_of','verbose_name','null','blank','help_text','max_digits','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "GSystemType":
        definitionlist = ['name','altnames','language','status','member_of','meta_type_set','attribute_type_set','relation_type_set','type_of']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "RelationType":
        definitionlist = ['name','inverse_name','altnames','language','subject_type','object_type','subject_cardinality','object_cardinality','subject_applicable_nodetype','object_applicable_nodetype','is_symmetric','is_reflexive','is_transitive','status','member_of']
        # definitionlist = ['name','inverse_name','altnames','language','subject_type','object_type','subject_cardinality','object_cardinality','is_symmetric','is_reflexive','is_transitive','status','member_of']
        contentlist = ['content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_at','start_publication','tags','url','last_update','login_required']
    else :
        definitionlist = []
        contentlist = []
        dependencylist = []
        options = []

    class_structure = eval(class_name).structure

    required_fields = eval(class_name).required_fields

    newdict = {}
    if node_id:
        new_instance_type = node_collection.one({'_type': unicode(class_name), '_id': ObjectId(node_id)})

    else:
        new_instance_type = eval("node_collection.collection"+"."+class_name)()

    if request.method=="POST":
        if translate:
            new_instance_type = eval("node_collection.collection"+"."+class_name)()
        # print new_instance_type
        for key,value in class_structure.items():
            if value == bool:
                if request.POST.get(key,""):
                    if request.POST.get(key,"") in ('1','2'):
                        if request.POST.get(key,"") == '1':
                            new_instance_type[key] = True
                        else :
                            new_instance_type[key] = False

            elif value == unicode:
                if request.POST.get(key,""):

                    if key == "content_org":
                        new_instance_type[key] = unicode(request.POST.get(key,""))
                        # Required to link temporary files with the current user who is modifying this document
                        usrname = request.user.username
                        filename = slugify(new_instance_type['name']) + "-" + usrname + "-"
                        new_instance_type['content'] = org2html(new_instance_type[key], file_prefix=filename)
                    else :
                        if translate:
                            if key in ("name","inverse_name"):
                                new_instance_type[key] = unicode(request.POST.get(key+"_trans",""))
                                language= request.POST.get('lan')
                                new_instance_type.language = get_language_tuple(language)

                            else:
                                new_instance_type[key] = unicode(request.POST.get(key,""))
                        else:
                            new_instance_type[key] = unicode(request.POST.get(key,""))

            elif value == list:
                if request.POST.get(key,""):
                    new_instance_type[key] = request.POST.get(key,"").split(",")

            elif type(value) == list:
                if request.POST.get(key,""):
                    if key in ("tags","applicable_node_type"):
                        new_instance_type[key] = request.POST.get(key,"").split(",")
                    elif key in ["meta_type_set","attribute_type_set","relation_type_set"]:
                        listoflist = []
                        for each in request.POST.get(key,"").split(","):
                            listoflist.append(node_collection.one({"_id":ObjectId(each)}))
                        new_instance_type[key] = listoflist
                    else :
                        listoflist = []

                        for each in request.POST.get(key,"").split(","):

                            listoflist.append(ObjectId(each))
                        new_instance_type[key] = listoflist

            elif type(value) == tuple:
                new_instance_type[key] = tuple(eval(request.POST.get(key,"")))

            elif value == datetime.datetime:
                if key == "last_update":
                    new_instance_type[key] = datetime.datetime.now()

            elif key == "status":
                if request.POST.get(key,""):
                    new_instance_type[key] = unicode(request.POST.get(key,""))

            # elif key == "created_by":
            #     new_instance_type[key] = request.user.id

            elif value == int:
                if request.POST.get(key,""):
                    new_instance_type[key] = int(request.POST.get(key,""))

            else:

                if request.POST.get(key,""):
                    new_instance_type[key] = request.POST.get(key,"")

        user_id = request.user.id
        if not new_instance_type.has_key('_id'):
            new_instance_type.created_by = user_id

        new_instance_type.modified_by = user_id

        if user_id not in new_instance_type.contributors:
            new_instance_type.contributors.append(user_id)
        parent_node=node_collection.one({'_id':ObjectId(node_id)})
        if translate and class_name == "RelationType":
            new_instance_type.subject_type = parent_node.subject_type
            new_instance_type.object_type = parent_node.object_type
        if translate and class_name == "AttributeType":
            new_instance_type.data_type = parent_node.data_type
            new_instance_type.subject_type = parent_node.subject_type

        new_instance_type.save()
        if translate:
            relation_type=node_collection.one({'$and':[{'name':'translation_of'},{'_type':'RelationType'}]})
            grelation=node_collection.collection.GRelation()
            grelation.relation_type=relation_type
            grelation.subject=new_instance_type['_id']
            grelation.right_subject=ObjectId(node_id)
            grelation.name=u""
            grelation.save()

        return HttpResponseRedirect("/admin/designer/"+class_name)

    # If GET request ---------------------------------------------------------------------------------------

    for key,value in class_structure.items():
        if value == bool:
            # newdict[key] = "bool"
            newdict[key] = ["bool", new_instance_type[key]]
        elif value == unicode:
            if key == "language":
                newdict[key] = ["list", new_instance_type[key]]
            else:
                # newdict[key] = "unicode"
                newdict[key] = ["unicode", new_instance_type[key]]

        elif value == list:
            # newdict[key] = "list"

            newdict[key] = ["list", new_instance_type[key]]
        elif type(value) == list:
            # newdict[key] = "list"
            newdict[key] = ["list", new_instance_type[key]]
        elif value == datetime.datetime:
            # newdict[key] = "datetime"
            newdict[key] = ["datetime", new_instance_type[key]]
        elif value == int:
            # newdict[key] = "int"
            newdict[key] = ["int", new_instance_type[key]]
        elif key == "status":
            # newdict[key] = "status"
            newdict[key] = ["status", new_instance_type[key]]
        else:
            # newdict[key] = value
            newdict[key] = [value, new_instance_type[key]]

    class_structure = newdict

    groupid = ""
    group_obj= node_collection.find({'$and':[{"_type":u'Group'},{"name":u'home'}]})
    if group_obj:
	groupid = str(group_obj[0]._id)

    template = "ndf/adminDashboardCreate.html"

    variable =  None
    class_structure_with_values = {}
    if node_id:

        for key, value in class_structure.items():
            class_structure_with_values[key] = [class_structure[key][0], new_instance_type[key]]

        variable = RequestContext(request, {'node': new_instance_type,
                                            'class_name': class_name, 'class_structure': class_structure_with_values, 'url': "designer",
                                            'definitionlist': definitionlist, 'contentlist': contentlist, 'dependencylist': dependencylist,
                                            'options': options, 'required_fields': required_fields,"translate":translate,"lan":LANGUAGES,
                                            'groupid': groupid,'group_id':groupid
                                        })

    else:

        variable = RequestContext(request, {'class_name':class_name, "url":"designer", "class_structure":class_structure, 'definitionlist':definitionlist, 'contentlist':contentlist, 'dependencylist':dependencylist, 'options':options, "required_fields":required_fields,"groupid":groupid,"translate":translate,"lan":LANGUAGES,'group_id':groupid })

    return render_to_response(template, variable)<|endoftext|>


<commit_before>"""Computes sounding statistics for each storm object.

"Sounding statistics" may also be referred to as "sounding indices" or "NSE
(near-storm environment) variables".

--- EXAMPLE USAGE ---

To compute stats at valid time only:

sounding_stats_for_storm_objects.py --spc_date_string="20110404"
--lead_times_seconds 0 --input_ruc_directory_name=${RUC_DIRECTORY_NAME}
--input_tracking_dir_name=${TRACKING_DIR_NAME}
--output_dir_name=${OUTPUT_DIR_NAME}

To compute stats at lead times of 7.5, 22.5, 37.5, 52.5, and 75 minutes:

sounding_stats_for_storm_objects.py --spc_date_string="20110404"
--lead_times_seconds 0 450 1350 2250 3150 4500
--input_ruc_directory_name=${RUC_DIRECTORY_NAME}
--input_tracking_dir_name=${TRACKING_DIR_NAME}
--output_dir_name=${OUTPUT_DIR_NAME}
"""

import argparse
import numpy
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import soundings
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.gg_utils import echo_top_tracking
from gewittergefahr.gg_utils import storm_tracking_utils as tracking_utils

SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n'

TIME_FORMAT_IN_OUTPUT_FILE_NAMES = '%Y%m%d%H%M%S'

WGRIB_EXE_NAME = '/condo/swatwork/ralager/wgrib/wgrib'
WGRIB2_EXE_NAME = '/condo/swatwork/ralager/grib2/wgrib2/wgrib2'

SPC_DATE_INPUT_ARG = 'spc_date_string'
START_TIME_INPUT_ARG = 'start_time_unix_sec'
END_TIME_INPUT_ARG = 'end_time_unix_sec'
LEAD_TIMES_INPUT_ARG = 'lead_times_seconds'
TRACKING_SCALE_INPUT_ARG = 'tracking_scale_metres2'
RUC_DIRECTORY_INPUT_ARG = 'input_ruc_directory_name'
TRACKING_DIR_INPUT_ARG = 'input_tracking_dir_name'
OUTPUT_DIR_INPUT_ARG = 'output_dir_name'

SPC_DATE_HELP_STRING = 'See help string for `{0:s}` and `{1:s}`.'.format(
    START_TIME_INPUT_ARG, END_TIME_INPUT_ARG)
TIME_HELP_STRING = (
    'Time in Unix seconds.  Sounding stats will be computed for all storm '
    'objects from `{0:s}`...`{1:s}` on SPC date `{2:s}`.  If `{0:s}` or `{1:s}`'
    ' is left empty, sounding stats will be computed for all storm objects on '
    'SPC date `{2:s}`.').format(
        START_TIME_INPUT_ARG, END_TIME_INPUT_ARG, SPC_DATE_INPUT_ARG)
LEAD_TIMES_HELP_STRING = (
    'Array of lead times.  For each storm object, sounding stats will be '
    'computed for each lead time.')
TRACKING_SCALE_HELP_STRING = (
    'Tracking scale (minimum storm area).  Will be used to find input data.')
RUC_DIRECTORY_HELP_STRING = (
    'Name of top-level directory with Rapid Update Cycle (RUC) data.  RUC data '
    'will be interpolated to storm objects to create soundings.')
TRACKING_DIR_HELP_STRING = (
    'Name of top-level directory with storm-tracking data.')
OUTPUT_DIR_HELP_STRING = (
    'Name of output directory.  A single Pickle file, with sounding stats for '
    'each storm object, will be written here.')

DEFAULT_LEAD_TIMES_SECONDS = [0]
DEFAULT_TRACKING_SCALE_METRES2 = int(numpy.round(
    echo_top_tracking.DUMMY_TRACKING_SCALE_METRES2))

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + SPC_DATE_INPUT_ARG, type=str, required=True,
    help=SPC_DATE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + START_TIME_INPUT_ARG, type=str, required=False, default=-1,
    help=TIME_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + END_TIME_INPUT_ARG, type=str, required=False, default=-1,
    help=TIME_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + LEAD_TIMES_INPUT_ARG, type=int, nargs='+', required=False,
    default=[0], help=LEAD_TIMES_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + TRACKING_SCALE_INPUT_ARG, type=int, required=False,
    default=DEFAULT_TRACKING_SCALE_METRES2, help=TRACKING_SCALE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + RUC_DIRECTORY_INPUT_ARG, type=str, required=True,
    help=RUC_DIRECTORY_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + TRACKING_DIR_INPUT_ARG, type=str, required=True,
    help=TRACKING_DIR_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_DIR_INPUT_ARG, type=str, required=True,
    help=OUTPUT_DIR_HELP_STRING)

def _get_pathless_output_file_name(
        spc_date_string, default_time_limits, start_time_unix_sec=None,
        end_time_unix_sec=None):
    """Creates pathless name for output file.

    Each output file should contain sounding stats for all storm objects in the
    given time period.

    :param spc_date_string: SPC date for storm objects (format "yyyymmdd").
    :param default_time_limits: Boolean flag.  If default_time_limits = True,
        the file will contain all storm objects in the SPC date.  If
        default_time_limits = False, the file will contain only storm objects in
        the time range `start_time_unix_sec`...`end_time_unix_sec`.
    :param start_time_unix_sec: See documentation for `default_time_limits`.
    :param end_time_unix_sec: See documentation for `default_time_limits`.
    :return: pathless_output_file_name: Pathless name for output file.
    """

    if default_time_limits:
        return 'sounding_statistics_{0:s}.p'.format(spc_date_string)

    return 'sounding_statistics_{0:s}_{1:s}-{2:s}.p'.format(
        spc_date_string,
        time_conversion.unix_sec_to_string(
            start_time_unix_sec, TIME_FORMAT_IN_OUTPUT_FILE_NAMES),
        time_conversion.unix_sec_to_string(
            end_time_unix_sec, TIME_FORMAT_IN_OUTPUT_FILE_NAMES))

def _compute_sounding_stats(
        spc_date_string, start_time_unix_sec, end_time_unix_sec,
        lead_times_seconds, tracking_scale_metres2, top_ruc_directory_name,
        top_tracking_dir_name, output_dir_name):
    """Computes sounding statistics for each storm object.

    :param spc_date_string: See documentation for `start_time_unix_sec`.
    :param start_time_unix_sec: Time in Unix seconds.  Sounding stats will be
        computed for all storm objects from `start_time_unix_sec`...
        `end_time_unix_sec` on SPC date `spc_date_string`.  If
        `start_time_unix_sec` or `end_time_unix_sec` is left empty, sounding
        stats will be computed for all storm objects on SPC date
        `spc_date_string`.
    :param end_time_unix_sec: See documentation for `start_time_unix_sec`.
    :param lead_times_seconds: Array of lead times.  For each storm object,
        sounding stats will be computed for each lead time.
    :param tracking_scale_metres2: Tracking scale (minimum storm area).  Will be
        used to find input data.
    :param top_ruc_directory_name: Name of top-level directory with Rapid Update
        Cycle (RUC) data.  RUC data will be interpolated to storm objects to
        create soundings.
    :param top_tracking_dir_name: Name of top-level directory with storm-
        tracking data.
    :param output_dir_name: Name of output directory.  A single Pickle file,
        with sounding stats for each storm object, will be written here.
    """

    lead_times_seconds = numpy.array(lead_times_seconds, dtype=int)

    if start_time_unix_sec == -1 or end_time_unix_sec == -1:
        start_time_unix_sec = 0
        end_time_unix_sec = int(1e10)
        default_time_limits = True
    else:
        default_time_limits = False

    tracking_file_names = tracking_io.find_processed_files_one_spc_date(
        spc_date_string=spc_date_string,
        data_source=tracking_utils.SEGMOTION_SOURCE_ID,
        top_processed_dir_name=top_tracking_dir_name,
        tracking_scale_metres2=tracking_scale_metres2)

    file_times_unix_sec = [
        tracking_io.processed_file_name_to_time(f) for f in tracking_file_names]
    time_in_range_flags = numpy.array(
        [start_time_unix_sec <= t <= end_time_unix_sec
         for t in file_times_unix_sec])
    time_in_range_indices = numpy.where(time_in_range_flags)[0]

    tracking_file_names = [
        tracking_file_names[i] for i in time_in_range_indices]

    storm_object_table = tracking_io.read_many_processed_files(
        tracking_file_names)
    print SEPARATOR_STRING

    sounding_stat_table = soundings.get_sounding_stats_for_storm_objects(
        storm_object_table=storm_object_table,
        lead_times_seconds=lead_times_seconds, all_ruc_grids=True,
        top_grib_directory_name=top_ruc_directory_name,
        wgrib_exe_name=WGRIB_EXE_NAME, wgrib2_exe_name=WGRIB2_EXE_NAME,
        raise_error_if_missing=False)
    print SEPARATOR_STRING

    num_lead_times = len(lead_times_seconds)
    sounding_stat_file_names = [''] * num_lead_times
    for j in range(num_lead_times):
        this_pathless_file_name = _get_pathless_output_file_name(
            spc_date_string=spc_date_string,
            default_time_limits=default_time_limits,
            start_time_unix_sec=start_time_unix_sec,
            end_time_unix_sec=end_time_unix_sec)

        sounding_stat_file_names[j] = '{0:s}/lead_time_{1:d}sec/{2:s}'.format(
            output_dir_name, lead_times_seconds[j], this_pathless_file_name)

    print 'Writing sounding stats to output files...'
    soundings.write_sounding_stats_for_storm_objects(
        sounding_stat_table, lead_times_seconds=lead_times_seconds,
        pickle_file_names=sounding_stat_file_names)

    print (
        'Sounding stats have been written to the following files:'
        '\n{0:s}').format(sounding_stat_file_names)

def add_input_arguments(argument_parser_object):
    """Adds input args for this script to `argparse.ArgumentParser` object.

    :param argument_parser_object: `argparse.ArgumentParser` object, which may
        or may not already contain input args.
    :return: argument_parser_object: Same as input object, but with new input
        args added.
    """

    argument_parser_object.add_argument(
        '--' + LEAD_TIMES_INPUT_ARG, type=int, nargs='+', required=False,
        default=[0], help=LEAD_TIMES_HELP_STRING)

    argument_parser_object.add_argument(
        '--' + TRACKING_SCALE_INPUT_ARG, type=int, required=False,
        default=DEFAULT_TRACKING_SCALE_METRES2, help=TRACKING_SCALE_HELP_STRING)

    argument_parser_object.add_argument(
        '--' + RUC_DIRECTORY_INPUT_ARG, type=str, required=True,
        help=RUC_DIRECTORY_HELP_STRING)

    argument_parser_object.add_argument(
        '--' + TRACKING_DIR_INPUT_ARG, type=str, required=True,
        help=TRACKING_DIR_HELP_STRING)

    argument_parser_object.add_argument(
        '--' + OUTPUT_DIR_INPUT_ARG, type=str, required=True,
        help=OUTPUT_DIR_HELP_STRING)

    return argument_parser_object

if __name__ == '__main__':
    INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args()

    SPC_DATE_STRING = getattr(INPUT_ARG_OBJECT, SPC_DATE_INPUT_ARG)
    START_TIME_UNIX_SEC = getattr(INPUT_ARG_OBJECT, START_TIME_INPUT_ARG)
    END_TIME_UNIX_SEC = getattr(INPUT_ARG_OBJECT, END_TIME_INPUT_ARG)
    LEAD_TIMES_SECONDS = getattr(INPUT_ARG_OBJECT, LEAD_TIMES_INPUT_ARG)
    TRACKING_SCALE_METRES2 = getattr(INPUT_ARG_OBJECT, TRACKING_SCALE_INPUT_ARG)
    TOP_RUC_DIRECTORY_NAME = getattr(INPUT_ARG_OBJECT, RUC_DIRECTORY_INPUT_ARG)
    TOP_TRACKING_DIR_NAME = getattr(INPUT_ARG_OBJECT, TRACKING_DIR_INPUT_ARG)
    OUTPUT_DIR_NAME = getattr(INPUT_ARG_OBJECT, OUTPUT_DIR_INPUT_ARG)

    _compute_sounding_stats(
        spc_date_string=SPC_DATE_STRING,
        start_time_unix_sec=START_TIME_UNIX_SEC,
        end_time_unix_sec=END_TIME_UNIX_SEC,
        lead_times_seconds=LEAD_TIMES_SECONDS,
        tracking_scale_metres2=TRACKING_SCALE_METRES2,
        top_ruc_directory_name=TOP_RUC_DIRECTORY_NAME,
        top_tracking_dir_name=TOP_TRACKING_DIR_NAME,
        output_dir_name=OUTPUT_DIR_NAME)
<commit_msg>Added debugging print statements to sounding_stats_for_storm_objects.py.<commit_after>"""Computes sounding statistics for each storm object.

"Sounding statistics" may also be referred to as "sounding indices" or "NSE
(near-storm environment) variables".

--- EXAMPLE USAGE ---

To compute stats at valid time only:

sounding_stats_for_storm_objects.py --spc_date_string="20110404"
--lead_times_seconds 0 --input_ruc_directory_name=${RUC_DIRECTORY_NAME}
--input_tracking_dir_name=${TRACKING_DIR_NAME}
--output_dir_name=${OUTPUT_DIR_NAME}

To compute stats at lead times of 7.5, 22.5, 37.5, 52.5, and 75 minutes:

sounding_stats_for_storm_objects.py --spc_date_string="20110404"
--lead_times_seconds 0 450 1350 2250 3150 4500
--input_ruc_directory_name=${RUC_DIRECTORY_NAME}
--input_tracking_dir_name=${TRACKING_DIR_NAME}
--output_dir_name=${OUTPUT_DIR_NAME}
"""

import argparse
import numpy
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import soundings
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.gg_utils import echo_top_tracking
from gewittergefahr.gg_utils import storm_tracking_utils as tracking_utils

SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n'

TIME_FORMAT_IN_OUTPUT_FILE_NAMES = '%Y%m%d%H%M%S'

WGRIB_EXE_NAME = '/condo/swatwork/ralager/wgrib/wgrib'
WGRIB2_EXE_NAME = '/condo/swatwork/ralager/grib2/wgrib2/wgrib2'

SPC_DATE_INPUT_ARG = 'spc_date_string'
START_TIME_INPUT_ARG = 'start_time_unix_sec'
END_TIME_INPUT_ARG = 'end_time_unix_sec'
LEAD_TIMES_INPUT_ARG = 'lead_times_seconds'
TRACKING_SCALE_INPUT_ARG = 'tracking_scale_metres2'
RUC_DIRECTORY_INPUT_ARG = 'input_ruc_directory_name'
TRACKING_DIR_INPUT_ARG = 'input_tracking_dir_name'
OUTPUT_DIR_INPUT_ARG = 'output_dir_name'

SPC_DATE_HELP_STRING = 'See help string for `{0:s}` and `{1:s}`.'.format(
    START_TIME_INPUT_ARG, END_TIME_INPUT_ARG)
TIME_HELP_STRING = (
    'Time in Unix seconds.  Sounding stats will be computed for all storm '
    'objects from `{0:s}`...`{1:s}` on SPC date `{2:s}`.  If `{0:s}` or `{1:s}`'
    ' is left empty, sounding stats will be computed for all storm objects on '
    'SPC date `{2:s}`.').format(
        START_TIME_INPUT_ARG, END_TIME_INPUT_ARG, SPC_DATE_INPUT_ARG)
LEAD_TIMES_HELP_STRING = (
    'Array of lead times.  For each storm object, sounding stats will be '
    'computed for each lead time.')
TRACKING_SCALE_HELP_STRING = (
    'Tracking scale (minimum storm area).  Will be used to find input data.')
RUC_DIRECTORY_HELP_STRING = (
    'Name of top-level directory with Rapid Update Cycle (RUC) data.  RUC data '
    'will be interpolated to storm objects to create soundings.')
TRACKING_DIR_HELP_STRING = (
    'Name of top-level directory with storm-tracking data.')
OUTPUT_DIR_HELP_STRING = (
    'Name of output directory.  A single Pickle file, with sounding stats for '
    'each storm object, will be written here.')

DEFAULT_LEAD_TIMES_SECONDS = [0]
DEFAULT_TRACKING_SCALE_METRES2 = int(numpy.round(
    echo_top_tracking.DUMMY_TRACKING_SCALE_METRES2))

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + SPC_DATE_INPUT_ARG, type=str, required=True,
    help=SPC_DATE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + START_TIME_INPUT_ARG, type=str, required=False, default=-1,
    help=TIME_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + END_TIME_INPUT_ARG, type=str, required=False, default=-1,
    help=TIME_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + LEAD_TIMES_INPUT_ARG, type=int, nargs='+', required=False,
    default=[0], help=LEAD_TIMES_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + TRACKING_SCALE_INPUT_ARG, type=int, required=False,
    default=DEFAULT_TRACKING_SCALE_METRES2, help=TRACKING_SCALE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + RUC_DIRECTORY_INPUT_ARG, type=str, required=True,
    help=RUC_DIRECTORY_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + TRACKING_DIR_INPUT_ARG, type=str, required=True,
    help=TRACKING_DIR_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_DIR_INPUT_ARG, type=str, required=True,
    help=OUTPUT_DIR_HELP_STRING)

def _get_pathless_output_file_name(
        spc_date_string, default_time_limits, start_time_unix_sec=None,
        end_time_unix_sec=None):
    """Creates pathless name for output file.

    Each output file should contain sounding stats for all storm objects in the
    given time period.

    :param spc_date_string: SPC date for storm objects (format "yyyymmdd").
    :param default_time_limits: Boolean flag.  If default_time_limits = True,
        the file will contain all storm objects in the SPC date.  If
        default_time_limits = False, the file will contain only storm objects in
        the time range `start_time_unix_sec`...`end_time_unix_sec`.
    :param start_time_unix_sec: See documentation for `default_time_limits`.
    :param end_time_unix_sec: See documentation for `default_time_limits`.
    :return: pathless_output_file_name: Pathless name for output file.
    """

    if default_time_limits:
        return 'sounding_statistics_{0:s}.p'.format(spc_date_string)

    return 'sounding_statistics_{0:s}_{1:s}-{2:s}.p'.format(
        spc_date_string,
        time_conversion.unix_sec_to_string(
            start_time_unix_sec, TIME_FORMAT_IN_OUTPUT_FILE_NAMES),
        time_conversion.unix_sec_to_string(
            end_time_unix_sec, TIME_FORMAT_IN_OUTPUT_FILE_NAMES))

def _compute_sounding_stats(
        spc_date_string, start_time_unix_sec, end_time_unix_sec,
        lead_times_seconds, tracking_scale_metres2, top_ruc_directory_name,
        top_tracking_dir_name, output_dir_name):
    """Computes sounding statistics for each storm object.

    :param spc_date_string: See documentation for `start_time_unix_sec`.
    :param start_time_unix_sec: Time in Unix seconds.  Sounding stats will be
        computed for all storm objects from `start_time_unix_sec`...
        `end_time_unix_sec` on SPC date `spc_date_string`.  If
        `start_time_unix_sec` or `end_time_unix_sec` is left empty, sounding
        stats will be computed for all storm objects on SPC date
        `spc_date_string`.
    :param end_time_unix_sec: See documentation for `start_time_unix_sec`.
    :param lead_times_seconds: Array of lead times.  For each storm object,
        sounding stats will be computed for each lead time.
    :param tracking_scale_metres2: Tracking scale (minimum storm area).  Will be
        used to find input data.
    :param top_ruc_directory_name: Name of top-level directory with Rapid Update
        Cycle (RUC) data.  RUC data will be interpolated to storm objects to
        create soundings.
    :param top_tracking_dir_name: Name of top-level directory with storm-
        tracking data.
    :param output_dir_name: Name of output directory.  A single Pickle file,
        with sounding stats for each storm object, will be written here.
    """

    lead_times_seconds = numpy.array(lead_times_seconds, dtype=int)

    if start_time_unix_sec == -1 or end_time_unix_sec == -1:
        start_time_unix_sec = 0
        end_time_unix_sec = int(1e10)
        default_time_limits = True
    else:
        default_time_limits = False

    tracking_file_names = tracking_io.find_processed_files_one_spc_date(
        spc_date_string=spc_date_string,
        data_source=tracking_utils.SEGMOTION_SOURCE_ID,
        top_processed_dir_name=top_tracking_dir_name,
        tracking_scale_metres2=tracking_scale_metres2)

    file_times_unix_sec = [
        tracking_io.processed_file_name_to_time(f) for f in tracking_file_names]
    time_in_range_flags = numpy.array(
        [start_time_unix_sec <= t <= end_time_unix_sec
         for t in file_times_unix_sec])
    print time_in_range_flags
    time_in_range_indices = numpy.where(time_in_range_flags)[0]
    print time_in_range_indices

    tracking_file_names = [
        tracking_file_names[i] for i in time_in_range_indices]

    storm_object_table = tracking_io.read_many_processed_files(
        tracking_file_names)
    print SEPARATOR_STRING

    sounding_stat_table = soundings.get_sounding_stats_for_storm_objects(
        storm_object_table=storm_object_table,
        lead_times_seconds=lead_times_seconds, all_ruc_grids=True,
        top_grib_directory_name=top_ruc_directory_name,
        wgrib_exe_name=WGRIB_EXE_NAME, wgrib2_exe_name=WGRIB2_EXE_NAME,
        raise_error_if_missing=False)
    print SEPARATOR_STRING

    num_lead_times = len(lead_times_seconds)
    sounding_stat_file_names = [''] * num_lead_times
    for j in range(num_lead_times):
        this_pathless_file_name = _get_pathless_output_file_name(
            spc_date_string=spc_date_string,
            default_time_limits=default_time_limits,
            start_time_unix_sec=start_time_unix_sec,
            end_time_unix_sec=end_time_unix_sec)

        sounding_stat_file_names[j] = '{0:s}/lead_time_{1:d}sec/{2:s}'.format(
            output_dir_name, lead_times_seconds[j], this_pathless_file_name)

    print 'Writing sounding stats to output files...'
    soundings.write_sounding_stats_for_storm_objects(
        sounding_stat_table, lead_times_seconds=lead_times_seconds,
        pickle_file_names=sounding_stat_file_names)

    print (
        'Sounding stats have been written to the following files:'
        '\n{0:s}').format(sounding_stat_file_names)

def add_input_arguments(argument_parser_object):
    """Adds input args for this script to `argparse.ArgumentParser` object.

    :param argument_parser_object: `argparse.ArgumentParser` object, which may
        or may not already contain input args.
    :return: argument_parser_object: Same as input object, but with new input
        args added.
    """

    argument_parser_object.add_argument(
        '--' + LEAD_TIMES_INPUT_ARG, type=int, nargs='+', required=False,
        default=[0], help=LEAD_TIMES_HELP_STRING)

    argument_parser_object.add_argument(
        '--' + TRACKING_SCALE_INPUT_ARG, type=int, required=False,
        default=DEFAULT_TRACKING_SCALE_METRES2, help=TRACKING_SCALE_HELP_STRING)

    argument_parser_object.add_argument(
        '--' + RUC_DIRECTORY_INPUT_ARG, type=str, required=True,
        help=RUC_DIRECTORY_HELP_STRING)

    argument_parser_object.add_argument(
        '--' + TRACKING_DIR_INPUT_ARG, type=str, required=True,
        help=TRACKING_DIR_HELP_STRING)

    argument_parser_object.add_argument(
        '--' + OUTPUT_DIR_INPUT_ARG, type=str, required=True,
        help=OUTPUT_DIR_HELP_STRING)

    return argument_parser_object

if __name__ == '__main__':
    INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args()

    SPC_DATE_STRING = getattr(INPUT_ARG_OBJECT, SPC_DATE_INPUT_ARG)
    START_TIME_UNIX_SEC = getattr(INPUT_ARG_OBJECT, START_TIME_INPUT_ARG)
    END_TIME_UNIX_SEC = getattr(INPUT_ARG_OBJECT, END_TIME_INPUT_ARG)
    LEAD_TIMES_SECONDS = getattr(INPUT_ARG_OBJECT, LEAD_TIMES_INPUT_ARG)
    TRACKING_SCALE_METRES2 = getattr(INPUT_ARG_OBJECT, TRACKING_SCALE_INPUT_ARG)
    TOP_RUC_DIRECTORY_NAME = getattr(INPUT_ARG_OBJECT, RUC_DIRECTORY_INPUT_ARG)
    TOP_TRACKING_DIR_NAME = getattr(INPUT_ARG_OBJECT, TRACKING_DIR_INPUT_ARG)
    OUTPUT_DIR_NAME = getattr(INPUT_ARG_OBJECT, OUTPUT_DIR_INPUT_ARG)

    _compute_sounding_stats(
        spc_date_string=SPC_DATE_STRING,
        start_time_unix_sec=START_TIME_UNIX_SEC,
        end_time_unix_sec=END_TIME_UNIX_SEC,
        lead_times_seconds=LEAD_TIMES_SECONDS,
        tracking_scale_metres2=TRACKING_SCALE_METRES2,
        top_ruc_directory_name=TOP_RUC_DIRECTORY_NAME,
        top_tracking_dir_name=TOP_TRACKING_DIR_NAME,
        output_dir_name=OUTPUT_DIR_NAME)
<|endoftext|>