<commit_before># Storage Management Providers
#
# Copyright (C) 2013-2014 Red Hat, Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation are
# those of the authors and should not be interpreted as representing official
# policies, either expressed or implied, of the FreeBSD Project.
#
# Authors: Jan Synacek <jsynacek@redhat.com>
#

"""
Mounting management functions.
"""

from lmi.scripts.common.errors import LmiFailed
from lmi.scripts.common import get_logger
LOG = get_logger(__name__)
from lmi.scripts.storage import common

_OPTS = ['AllowExecution',
         'AllowMandatoryLock',
         'AllowSUID',
         'AllowUserMount',
         'AllowWrite',
         'Auto',
         'Dump',
         'FileSystemCheckOrder',
         'InterpretDevices',
         'OtherOptions',
         'Silent',
         'SynchronousDirectoryUpdates',
         'SynchronousIO',
         'UpdateAccessTimes',
         'UpdateDirectoryAccessTimes',
         'UpdateFullAccessTimes',
         'UpdateRelativeAccessTimes']

def build_opts_str(mnt):
    """
    Build option strings from an LMI_MountedFileSystem instance.

    :type mnt: an LMIInstance of LMI_MountedFileSystem
    :rtype: tuple of option strings
    """
    opts = []
    otheropts = []
    setting = mnt.associators(ResultClass='LMI_MountedFileSystemSetting')[0]
    for k, v in setting.properties_dict().iteritems():
        if v is None: continue
        if k == 'OtherOptions':
            otheropts = v
        elif k in _OPTS:
            opts.append(k + ':' + unicode(v))
    return (', '.join(sorted(opts)), ', '.join(sorted(otheropts)))

def get_setting_from_opts(ns, options, other_options):
    """
    Create a setting instance from option strings.

    :type options: string of comma-separated options
    :type other_options: string of comma-separated options
    :rtype: an instance of LMI_MountedFileSystemSetting
    """
    cap = ns.LMI_MountedFileSystemCapabilities.first_instance()

    (ret, outparams, _err) = cap.CreateSetting()
    if ret != 0:
        raise LmiFailed('Could not create setting')

    setting = outparams['setting'].to_instance()

    if other_options is not None:
        setting.OtherOptions = other_options.split(',')

    if options is None:
        return setting

    for opt_pair in options.split(','):
        opts = map(lambda o: o.strip(), opt_pair.split(':'))
        # bail out if the option is not in 'Option:Value' format
        # or if the Option is not in supported options
        if len(opts) != 2 or opts[0] not in _OPTS:
            raise LmiFailed('Invalid option: %s' % opt_pair)
        # ignore OtherOptions, there is a separate cmdline option for it
        if opts[0] == 'OtherOptions':
            continue
        # insist on using a number with FileSystemCheckOrder
        if opts[0] == 'FileSystemCheckOrder':
            if not opts[1].isdigit():
                raise LmiFailed('Value of FileSystemCheckOrder must be a number')
            opts[1] = int(opts[1])
        else:
            # otherwise check for true/false possibilities
            opts[1] = opts[1].lower()

            if opts[1] not in ['t', 'true', 'f', 'false']:
                raise LmiFailed('Invalid option value: %s' % opts[1])

            # sanitize the boolean option values
            if opts[1] in ['t', 'true']:
                opts[1] = True
            else:
                opts[1] = False

        setattr(setting, opts[0], opts[1])

    return setting

def get_mounts(ns):
    """
    Return all instances of LMI_MountedFileSystem.

    :rtype: list of LMI_MountedFileSystem
    """
    return ns.LMI_MountedFileSystem.instances()

def mount_create(ns, device, mountpoint, fs_type=None, options=None, other_options=None):
    """
    Create a mounted filesystem.

    :type device: string
    :param device: device path
    :type mountpoint: string
    :param mountpoint: path where device should be mounted
    :type fs_type: string
    :param fs_type: filesystem type
    :type options: string
    :param options: comma-separated string of mount options
    :type other_options: string
    :param other_options: comma-separated string of filesystem specific mount options
    """
    fs_setting = ns.LMI_FileSystemSetting.first_instance({'InstanceID':'LMI:LMI_FileSystemSetting:'+device})
    if fs_setting is None:
        raise LmiFailed('Wrong device: %s' % device)
    filesystem = fs_setting.associators()[0]
    if fs_type is None:
        fs_type = filesystem.FileSystemType
    service = ns.LMI_MountConfigurationService.first_instance()

    setting = get_setting_from_opts(ns, options, other_options)
    setting.push()

    # TODO for now
    # Mode 32768 == only mount (don't create any persistent info)
    (ret, _outparams, _err) = service.SyncCreateMount(Goal=setting.path,
                                                      FileSystemType=fs_type,
                                                      Mode=32768,
                                                      FileSystem=filesystem.path,
                                                      MountPoint=mountpoint,
                                                      FileSystemSpec=device)
    msg = '%s on %s' % (device, mountpoint)
    if ret != 0:
        errname = service.SyncCreateMount.CreateMountValues.value_name(ret)
        raise LmiFailed('Cannot create mount: %s: %s.' % (
                msg, errname))

    LOG().info('Successfully created mount: %s', msg)

def mount_delete(ns, target):
    """
    Unmount filesystem.

    :type target: string
    :param target: device path or mountpoint
    """
    mnt = ns.LMI_MountedFileSystem.first_instance({'FileSystemSpec':target}) or \
          ns.LMI_MountedFileSystem.first_instance({'MountPointPath':target})
    if mnt is None:
        raise LmiFailed('Target is not mounted: %s.' % target)

    service = ns.LMI_MountConfigurationService.first_instance()
    # TODO for now
    # Mode 32769 == only unmount (don't remove any persistent info)
    (ret, _outparams, _err) = service.SyncDeleteMount(Mount=mnt, Mode=32769)
    if ret != 0:
        raise LmiFailed('Cannot delete mount: %s.' % target)

    LOG().info('Successfully deleted mount: %s', target)
<commit_msg>Added heuristics to translate device names to full device paths.<commit_after># Storage Management Providers
#
# Copyright (C) 2013-2014 Red Hat, Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation are
# those of the authors and should not be interpreted as representing official
# policies, either expressed or implied, of the FreeBSD Project.
#
# Authors: Jan Synacek <jsynacek@redhat.com>
#

"""
Mounting management functions.
"""

from lmi.scripts.common.errors import LmiFailed
from lmi.scripts.common import get_logger
LOG = get_logger(__name__)
from lmi.scripts.storage import common

_OPTS = ['AllowExecution',
         'AllowMandatoryLock',
         'AllowSUID',
         'AllowUserMount',
         'AllowWrite',
         'Auto',
         'Dump',
         'FileSystemCheckOrder',
         'InterpretDevices',
         'OtherOptions',
         'Silent',
         'SynchronousDirectoryUpdates',
         'SynchronousIO',
         'UpdateAccessTimes',
         'UpdateDirectoryAccessTimes',
         'UpdateFullAccessTimes',
         'UpdateRelativeAccessTimes']

def build_opts_str(mnt):
    """
    Build option strings from an LMI_MountedFileSystem instance.

    :type mnt: an LMIInstance of LMI_MountedFileSystem
    :rtype: tuple of option strings
    """
    opts = []
    otheropts = []
    setting = mnt.associators(ResultClass='LMI_MountedFileSystemSetting')[0]
    for k, v in setting.properties_dict().iteritems():
        if v is None: continue
        if k == 'OtherOptions':
            otheropts = v
        elif k in _OPTS:
            opts.append(k + ':' + unicode(v))
    return (', '.join(sorted(opts)), ', '.join(sorted(otheropts)))

def get_setting_from_opts(ns, options, other_options):
    """
    Create a setting instance from option strings.

    :type options: string of comma-separated options
    :type other_options: string of comma-separated options
    :rtype: an instance of LMI_MountedFileSystemSetting
    """
    cap = ns.LMI_MountedFileSystemCapabilities.first_instance()

    (ret, outparams, _err) = cap.CreateSetting()
    if ret != 0:
        raise LmiFailed('Could not create setting')

    setting = outparams['setting'].to_instance()

    if other_options is not None:
        setting.OtherOptions = other_options.split(',')

    if options is None:
        return setting

    for opt_pair in options.split(','):
        opts = map(lambda o: o.strip(), opt_pair.split(':'))
        # bail out if the option is not in 'Option:Value' format
        # or if the Option is not in supported options
        if len(opts) != 2 or opts[0] not in _OPTS:
            raise LmiFailed('Invalid option: %s' % opt_pair)
        # ignore OtherOptions, there is a separate cmdline option for it
        if opts[0] == 'OtherOptions':
            continue
        # insist on using a number with FileSystemCheckOrder
        if opts[0] == 'FileSystemCheckOrder':
            if not opts[1].isdigit():
                raise LmiFailed('Value of FileSystemCheckOrder must be a number')
            opts[1] = int(opts[1])
        else:
            # otherwise check for true/false possibilities
            opts[1] = opts[1].lower()

            if opts[1] not in ['t', 'true', 'f', 'false']:
                raise LmiFailed('Invalid option value: %s' % opts[1])

            # sanitize the boolean option values
            if opts[1] in ['t', 'true']:
                opts[1] = True
            else:
                opts[1] = False

        setattr(setting, opts[0], opts[1])

    return setting

def get_mounts(ns):
    """
    Return all instances of LMI_MountedFileSystem.

    :rtype: list of LMI_MountedFileSystem
    """
    return ns.LMI_MountedFileSystem.instances()

def mount_create(ns, device, mountpoint, fs_type=None, options=None, other_options=None):
    """
    Create a mounted filesystem.

    :type device: string or LMIInstance/CIM_StorageExtent
    :param device: Device to mount or a mount specifier (like '//server/share' for CIFS).
    :type mountpoint: string
    :param mountpoint: path where device should be mounted
    :type fs_type: string
    :param fs_type: filesystem type
    :type options: string
    :param options: comma-separated string of mount options
    :type other_options: string
    :param other_options: comma-separated string of filesystem specific mount options
    """
    # Try to convert device name to full device path
    try:
        device_inst = common.str2device(ns, device)
        if device_inst:
            device = device_inst.Name
    except LmiFailed:
        # we did not find CIM_StorageExtent for the device, it must be non
        # device filesystem specification
        pass

    fs_setting = ns.LMI_FileSystemSetting.first_instance({'InstanceID':'LMI:LMI_FileSystemSetting:' + device})
    if fs_setting is None:
        raise LmiFailed('Wrong device: %s' % device)
    filesystem = fs_setting.associators()[0]
    if fs_type is None:
        fs_type = filesystem.FileSystemType
    service = ns.LMI_MountConfigurationService.first_instance()

    setting = get_setting_from_opts(ns, options, other_options)
    setting.push()

    # TODO for now
    # Mode 32768 == only mount (don't create any persistent info)
    (ret, _outparams, _err) = service.SyncCreateMount(Goal=setting.path,
                                                      FileSystemType=fs_type,
                                                      Mode=32768,
                                                      FileSystem=filesystem.path,
                                                      MountPoint=mountpoint,
                                                      FileSystemSpec=device)
    msg = '%s on %s' % (device, mountpoint)
    if ret != 0:
        errname = service.SyncCreateMount.CreateMountValues.value_name(ret)
        raise LmiFailed('Cannot create mount: %s: %s.' % (
                msg, errname))

    LOG().info('Successfully created mount: %s', msg)

def mount_delete(ns, target):
    """
    Unmount filesystem.

    :type target: string
    :param target: device path or mountpoint
    """
    # Try to convert device name to full device path
    try:
        device_inst = common.str2device(ns, target)
        if device_inst:
            target = device_inst.Name
    except LmiFailed:
        # we did not find CIM_StorageExtent for the device, it must be non
        # device filesystem specification
        pass

    mnt = ns.LMI_MountedFileSystem.first_instance({'FileSystemSpec':target}) or \
          ns.LMI_MountedFileSystem.first_instance({'MountPointPath':target})
    if mnt is None:
        raise LmiFailed('Target is not mounted: %s.' % target)

    service = ns.LMI_MountConfigurationService.first_instance()
    # TODO for now
    # Mode 32769 == only unmount (don't remove any persistent info)
    (ret, _outparams, _err) = service.SyncDeleteMount(Mount=mnt, Mode=32769)
    if ret != 0:
        raise LmiFailed('Cannot delete mount: %s.' % target)

    LOG().info('Successfully deleted mount: %s', target)
<|endoftext|>


<commit_before># Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import absolute_import
import re
import sys
import unittest
import six

from py_utils import exc_util

class FakeConnectionError(Exception):
  pass

class FakeDisconnectionError(Exception):
  pass

class FakeProcessingError(Exception):
  pass

class FakeCleanupError(Exception):
  pass

class FaultyClient(object):
  def __init__(self, *args):
    self.failures = set(args)
    self.called = set()

  def Connect(self):
    self.called.add('Connect')
    if FakeConnectionError in self.failures:
      raise FakeConnectionError('Oops!')

  def Process(self):
    self.called.add('Process')
    if FakeProcessingError in self.failures:
      raise FakeProcessingError('Oops!')

  @exc_util.BestEffort
  def Disconnect(self):
    self.called.add('Disconnect')
    if FakeDisconnectionError in self.failures:
      raise FakeDisconnectionError('Oops!')

  @exc_util.BestEffort
  def Cleanup(self):
    self.called.add('Cleanup')
    if FakeCleanupError in self.failures:
      raise FakeCleanupError('Oops!')

class ReraiseTests(unittest.TestCase):
  def assertCountEqualPy23(self, expected, actual):
    if six.PY2:
      self.assertItemsEqual(expected, actual)
    else:
      self.assertCountEqual(expected, actual) # pylint: disable=no-member

  def assertLogMatches(self, pattern):
    self.assertRegexpMatches(
        sys.stderr.getvalue(), pattern)  # pylint: disable=no-member

  def assertLogNotMatches(self, pattern):
    self.assertNotRegexpMatches(
        sys.stderr.getvalue(), pattern)  # pylint: disable=no-member

  def testTryRaisesExceptRaises(self):
    client = FaultyClient(FakeConnectionError, FakeDisconnectionError)

    # The connection error reaches the top level, while the disconnection
    # error is logged.
    with self.assertRaises(FakeConnectionError):
      try:
        client.Connect()
      except:
        client.Disconnect()
        raise

    self.assertLogMatches(re.compile(
        r'While handling a FakeConnectionError, .* was also raised:\n'
        r'Traceback \(most recent call last\):\n'
        r'.*\n'
        r'FakeDisconnectionError: Oops!\n', re.DOTALL))
    self.assertCountEqualPy23(client.called, ['Connect', 'Disconnect'])

  def testTryRaisesExceptDoesnt(self):
    client = FaultyClient(FakeConnectionError)

    # The connection error reaches the top level, disconnecting did not raise
    # an exception (so nothing is logged).
    with self.assertRaises(FakeConnectionError):
      try:
        client.Connect()
      except:
        client.Disconnect()
        raise

    self.assertLogNotMatches('FakeDisconnectionError')
    self.assertCountEqualPy23(client.called, ['Connect', 'Disconnect'])

  def testTryPassesNoException(self):
    client = FaultyClient(FakeDisconnectionError)

    # If there is no connection error, the except clause is not called (even if
    # it would have raised an exception).
    try:
      client.Connect()
    except:
      client.Disconnect()
      raise

    self.assertLogNotMatches('FakeConnectionError')
    self.assertLogNotMatches('FakeDisconnectionError')
    self.assertCountEqualPy23(client.called, ['Connect'])

  def testTryRaisesFinallyRaises(self):
    worker = FaultyClient(FakeProcessingError, FakeCleanupError)

    # The processing error reaches the top level, the cleanup error is logged.
    with self.assertRaises(FakeProcessingError):
      try:
        worker.Process()
      except:
        raise  # Needed for Cleanup to know if an exception is handled.
      finally:
        worker.Cleanup()

    self.assertLogMatches(re.compile(
        r'While handling a FakeProcessingError, .* was also raised:\n'
        r'Traceback \(most recent call last\):\n'
        r'.*\n'
        r'FakeCleanupError: Oops!\n', re.DOTALL))
    self.assertCountEqualPy23(worker.called, ['Process', 'Cleanup'])

  def testTryRaisesFinallyDoesnt(self):
    worker = FaultyClient(FakeProcessingError)

    # The processing error reaches the top level, the cleanup code runs fine.
    with self.assertRaises(FakeProcessingError):
      try:
        worker.Process()
      except:
        raise  # Needed for Cleanup to know if an exception is handled.
      finally:
        worker.Cleanup()

    self.assertLogNotMatches('FakeProcessingError')
    self.assertLogNotMatches('FakeCleanupError')
    self.assertCountEqualPy23(worker.called, ['Process', 'Cleanup'])

  def testTryPassesFinallyRaises(self):
    worker = FaultyClient(FakeCleanupError)

    # The processing code runs fine, the cleanup code raises an exception
    # which reaches the top level.
    with self.assertRaises(FakeCleanupError):
      try:
        worker.Process()
      except:
        raise  # Needed for Cleanup to know if an exception is handled.
      finally:
        worker.Cleanup()

    self.assertLogNotMatches('FakeProcessingError')
    self.assertLogNotMatches('FakeCleanupError')
    self.assertCountEqualPy23(worker.called, ['Process', 'Cleanup'])

  def testTryRaisesExceptRaisesFinallyRaises(self):
    worker = FaultyClient(
        FakeProcessingError, FakeDisconnectionError, FakeCleanupError)

    # Chaining try-except-finally works fine. Only the processing error reaches
    # the top level; the other two are logged.
    with self.assertRaises(FakeProcessingError):
      try:
        worker.Process()
      except:
        worker.Disconnect()
        raise
      finally:
        worker.Cleanup()

    self.assertLogMatches('FakeDisconnectionError')
    self.assertLogMatches('FakeCleanupError')
    self.assertCountEqualPy23(worker.called,
                              ['Process', 'Disconnect', 'Cleanup'])
<commit_msg>Python3 Migration - loosen requirements for @exec_util.BestEffort exceptions.<commit_after># Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import absolute_import
import re
import sys
import unittest
import six

from py_utils import exc_util

class FakeConnectionError(Exception):
  pass

class FakeDisconnectionError(Exception):
  pass

class FakeProcessingError(Exception):
  pass

class FakeCleanupError(Exception):
  pass

class FaultyClient(object):
  def __init__(self, *args):
    self.failures = set(args)
    self.called = set()

  def Connect(self):
    self.called.add('Connect')
    if FakeConnectionError in self.failures:
      raise FakeConnectionError('Oops!')

  def Process(self):
    self.called.add('Process')
    if FakeProcessingError in self.failures:
      raise FakeProcessingError('Oops!')

  @exc_util.BestEffort
  def Disconnect(self):
    self.called.add('Disconnect')
    if FakeDisconnectionError in self.failures:
      raise FakeDisconnectionError('Oops!')

  @exc_util.BestEffort
  def Cleanup(self):
    self.called.add('Cleanup')
    if FakeCleanupError in self.failures:
      raise FakeCleanupError('Oops!')

class ReraiseTests(unittest.TestCase):
  def assertCountEqualPy23(self, expected, actual):
    if six.PY2:
      self.assertItemsEqual(expected, actual)
    else:
      self.assertCountEqual(expected, actual) # pylint: disable=no-member

  def assertLogMatches(self, pattern):
    self.assertRegexpMatches(
        sys.stderr.getvalue(), pattern)  # pylint: disable=no-member

  def assertLogNotMatches(self, pattern):
    self.assertNotRegexpMatches(
        sys.stderr.getvalue(), pattern)  # pylint: disable=no-member

  def testTryRaisesExceptRaises(self):
    client = FaultyClient(FakeConnectionError, FakeDisconnectionError)

    # The connection error reaches the top level, while the disconnection
    # error is logged.
    with self.assertRaises(FakeConnectionError):
      try:
        client.Connect()
      except:
        client.Disconnect()
        raise

    self.assertLogMatches(re.compile(
        r'While handling a FakeConnectionError, .* was also raised:\n'
        r'.*'
        r'FakeDisconnectionError: Oops!\n', re.DOTALL))
    self.assertCountEqualPy23(client.called, ['Connect', 'Disconnect'])

  def testTryRaisesExceptDoesnt(self):
    client = FaultyClient(FakeConnectionError)

    # The connection error reaches the top level, disconnecting did not raise
    # an exception (so nothing is logged).
    with self.assertRaises(FakeConnectionError):
      try:
        client.Connect()
      except:
        client.Disconnect()
        raise

    self.assertLogNotMatches('FakeDisconnectionError')
    self.assertCountEqualPy23(client.called, ['Connect', 'Disconnect'])

  def testTryPassesNoException(self):
    client = FaultyClient(FakeDisconnectionError)

    # If there is no connection error, the except clause is not called (even if
    # it would have raised an exception).
    try:
      client.Connect()
    except:
      client.Disconnect()
      raise

    self.assertLogNotMatches('FakeConnectionError')
    self.assertLogNotMatches('FakeDisconnectionError')
    self.assertCountEqualPy23(client.called, ['Connect'])

  def testTryRaisesFinallyRaises(self):
    worker = FaultyClient(FakeProcessingError, FakeCleanupError)

    # The processing error reaches the top level, the cleanup error is logged.
    with self.assertRaises(FakeProcessingError):
      try:
        worker.Process()
      except:
        raise  # Needed for Cleanup to know if an exception is handled.
      finally:
        worker.Cleanup()

    self.assertLogMatches(re.compile(
        r'While handling a FakeProcessingError, .* was also raised:\n'
        r'.*'
        r'FakeCleanupError: Oops!\n', re.DOTALL))
    self.assertCountEqualPy23(worker.called, ['Process', 'Cleanup'])

  def testTryRaisesFinallyDoesnt(self):
    worker = FaultyClient(FakeProcessingError)

    # The processing error reaches the top level, the cleanup code runs fine.
    with self.assertRaises(FakeProcessingError):
      try:
        worker.Process()
      except:
        raise  # Needed for Cleanup to know if an exception is handled.
      finally:
        worker.Cleanup()

    self.assertLogNotMatches('FakeProcessingError')
    self.assertLogNotMatches('FakeCleanupError')
    self.assertCountEqualPy23(worker.called, ['Process', 'Cleanup'])

  def testTryPassesFinallyRaises(self):
    worker = FaultyClient(FakeCleanupError)

    # The processing code runs fine, the cleanup code raises an exception
    # which reaches the top level.
    with self.assertRaises(FakeCleanupError):
      try:
        worker.Process()
      except:
        raise  # Needed for Cleanup to know if an exception is handled.
      finally:
        worker.Cleanup()

    self.assertLogNotMatches('FakeProcessingError')
    self.assertLogNotMatches('FakeCleanupError')
    self.assertCountEqualPy23(worker.called, ['Process', 'Cleanup'])

  def testTryRaisesExceptRaisesFinallyRaises(self):
    worker = FaultyClient(
        FakeProcessingError, FakeDisconnectionError, FakeCleanupError)

    # Chaining try-except-finally works fine. Only the processing error reaches
    # the top level; the other two are logged.
    with self.assertRaises(FakeProcessingError):
      try:
        worker.Process()
      except:
        worker.Disconnect()
        raise
      finally:
        worker.Cleanup()

    self.assertLogMatches('FakeDisconnectionError')
    self.assertLogMatches('FakeCleanupError')
    self.assertCountEqualPy23(worker.called,
                              ['Process', 'Disconnect', 'Cleanup'])
<|endoftext|>


<commit_before># Copyright 2013,2014 Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Dunya
#
# Dunya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation (FSF), either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see http://www.gnu.org/licenses/
import json
import cStringIO
from compmusic.extractors.audioimages import AudioImages
from docserver import util
import struct
import tempfile
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from scipy.ndimage.filters import gaussian_filter
from seyiranalyzer import audioseyiranalyzer

class MakamAudioImage(AudioImages):
    _version = "0.3"
    _sourcetype = "mp3"
    _slug = "makamaudioimages"
    _depends = "dunyapitchmakam"

    _output = {
            "waveform8": {"extension": "png", "mimetype": "image/png", "parts": True},
            "spectrum8": {"extension": "png", "mimetype": "image/png", "parts": True},
            "smallfull": {"extension": "png", "mimetype": "image/png"},
        }

    _zoom_levels =  [8]
    _fft_size = 4096
    _scale_exp = 2
    _pallete = 2

    def run(self, musicbrainzid, fname):

        max_pitch = util.docserver_get_filename(musicbrainzid, "tomatodunya", "pitchmax", version="0.1")
        pitch = json.load(open(max_pitch))

        self._f_min = pitch['min']
        self._f_max = pitch['max']
        ret = super(MakamAudioImage, self).run(musicbrainzid, fname)

        pitchfile = util.docserver_get_filename(musicbrainzid, "jointanalysis", "pitch", version="0.1")
        pitch = np.array(json.load(open(pitchfile, 'r'))['pitch'])

        audioSeyirAnalyzer = audioseyiranalyzer.AudioSeyirAnalyzer()

        # compute number of frames from some simple rules set by the user
        duration = pitch[-1][0]
        min_num_frames = 40
        max_frame_dur = 30
        frame_dur = duration/min_num_frames if duration/min_num_frames<=max_frame_dur else max_frame_dur
        frame_dur = int(5 * round(float(frame_dur)/5))  # round to 5 seconds
        if not frame_dur:
            frame_dur = 5

        seyir_features = audioSeyirAnalyzer.analyze(pitch, frame_dur = frame_dur, hop_ratio = 0.5)

        fimage = tempfile.NamedTemporaryFile(mode='w+', suffix=".png")
        plot(seyir_features, fimage.name)
        fimage.flush()
        fileContent = None
        with open(fimage.name, mode='rb') as file:
            file_content = file.read()
        if not file_content:
            raise Exception("No image generated")
        ret['smallfull'] = file_content

        return ret

def plot(seyir_features, file_location, plot_average_pitch=True, plot_stable_pitches=True,
             plot_distribution=False):

    fig = plt.figure(frameon=False)
    if plot_distribution:
        time_starts = [sf['time_interval'][0] for sf in seyir_features]
        min_time = min(np.diff(time_starts))
        for sf in seyir_features:
            # plot the distributions through time
            yy = sf['pitch_distribution'].bins
            tt = sf['time_interval'][0] + sf['pitch_distribution'].vals * min_time*2
            plt.plot(tt, yy)

    for sf in seyir_features:
        if len(sf['stable_pitches']):
            t_st = sf['time_interval'][0]
            max_peak = max([sp['value'] for sp in sf['stable_pitches']])
            for sp in sf['stable_pitches']:
                clr = 'r' if sp['value'] == max_peak else 'b'
                # map the values from 0-1 to 1-6
                marker_thickness = sp['value']*5+1
                plt.plot(t_st, sp['frequency'], 'o', color = clr, ms=marker_thickness)

    if plot_average_pitch:
        tt = [sf['time_interval'][0] for sf in seyir_features]
        pp = [sf['average_pitch'] for sf in seyir_features]

        plt.plot(tt, pp, color='k', linewidth=3)

    fig.set_size_inches(31.25, 2.6)
    fig.set_tight_layout(True)
    plt.axis('off')
    plt.margins(0,0)

    plt.savefig(file_location, bbox_inches='tight', pad_inches=0)
<commit_msg>image generator extractor load pitch from audioanalysis<commit_after># Copyright 2013,2014 Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Dunya
#
# Dunya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation (FSF), either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see http://www.gnu.org/licenses/
import json
import cStringIO
from compmusic.extractors.audioimages import AudioImages
from docserver import util
import struct
import tempfile
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from scipy.ndimage.filters import gaussian_filter
from seyiranalyzer import audioseyiranalyzer

class MakamAudioImage(AudioImages):
    _version = "0.3"
    _sourcetype = "mp3"
    _slug = "makamaudioimages"
    _depends = "dunyapitchmakam"

    _output = {
            "waveform8": {"extension": "png", "mimetype": "image/png", "parts": True},
            "spectrum8": {"extension": "png", "mimetype": "image/png", "parts": True},
            "smallfull": {"extension": "png", "mimetype": "image/png"},
        }

    _zoom_levels =  [8]
    _fft_size = 4096
    _scale_exp = 2
    _pallete = 2

    def run(self, musicbrainzid, fname):

        max_pitch = util.docserver_get_filename(musicbrainzid, "tomatodunya", "pitchmax", version="0.1")
        pitch = json.load(open(max_pitch))

        self._f_min = pitch['min']
        self._f_max = pitch['max']
        ret = super(MakamAudioImage, self).run(musicbrainzid, fname)

        pitchfile = util.docserver_get_filename(musicbrainzid, "jointanalysis", "pitch", version="0.1")
        loaded_pitch = json.load(open(pitchfile, 'r'))['pitch']
        # If pitch extraction from jointanalysis failed then load it from audioanlysis
        if not loaded_pitch:
          pitchfile = util.docserver_get_filename(musicbrainzid, "audioanalysis", "pitch", version="0.1")
          loaded_pitch = json.load(open(pitchfile, 'r'))['pitch']

        pitch = np.array(loaded_pitch)

        audioSeyirAnalyzer = audioseyiranalyzer.AudioSeyirAnalyzer()

        # compute number of frames from some simple rules set by the user
        duration = pitch[-1][0]
        min_num_frames = 40
        max_frame_dur = 30
        frame_dur = duration/min_num_frames if duration/min_num_frames<=max_frame_dur else max_frame_dur
        frame_dur = int(5 * round(float(frame_dur)/5))  # round to 5 seconds
        if not frame_dur:
            frame_dur = 5

        seyir_features = audioSeyirAnalyzer.analyze(pitch, frame_dur = frame_dur, hop_ratio = 0.5)

        fimage = tempfile.NamedTemporaryFile(mode='w+', suffix=".png")
        plot(seyir_features, fimage.name)
        fimage.flush()
        fileContent = None
        with open(fimage.name, mode='rb') as file:
            file_content = file.read()
        if not file_content:
            raise Exception("No image generated")
        ret['smallfull'] = file_content

        return ret

def plot(seyir_features, file_location, plot_average_pitch=True, plot_stable_pitches=True,
             plot_distribution=False):

    fig = plt.figure(frameon=False)
    if plot_distribution:
        time_starts = [sf['time_interval'][0] for sf in seyir_features]
        min_time = min(np.diff(time_starts))
        for sf in seyir_features:
            # plot the distributions through time
            yy = sf['pitch_distribution'].bins
            tt = sf['time_interval'][0] + sf['pitch_distribution'].vals * min_time*2
            plt.plot(tt, yy)

    for sf in seyir_features:
        if len(sf['stable_pitches']):
            t_st = sf['time_interval'][0]
            max_peak = max([sp['value'] for sp in sf['stable_pitches']])
            for sp in sf['stable_pitches']:
                clr = 'r' if sp['value'] == max_peak else 'b'
                # map the values from 0-1 to 1-6
                marker_thickness = sp['value']*5+1
                plt.plot(t_st, sp['frequency'], 'o', color = clr, ms=marker_thickness)

    if plot_average_pitch:
        tt = [sf['time_interval'][0] for sf in seyir_features]
        pp = [sf['average_pitch'] for sf in seyir_features]

        plt.plot(tt, pp, color='k', linewidth=3)

    fig.set_size_inches(31.25, 2.6)
    fig.set_tight_layout(True)
    plt.axis('off')
    plt.margins(0,0)

    plt.savefig(file_location, bbox_inches='tight', pad_inches=0)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import subprocess
import os
import sys
import json

REPO_PATH = 'git-repo'

def git_clone(url):
    r = subprocess.run(['git', 'clone', url, REPO_PATH])

    if (r.returncode == 0):
        return True
    else:
        print("[COUT] Git clone error: Invalid argument to exit", file=sys.stderr)
        print("[COUT] CO_RESULT = false")
        return False

def flake8(file_name):
    r = subprocess.run(['flake8', file_name], stderr=subprocess.PIPE,
                       stdout=subprocess.PIPE)

    passed = True
    if (r.returncode != 0):
        passed = False

    out = str(r.stdout, 'utf-8').strip().split('\n')
    for o in out:
        o = parse_flake8_result(o)
        if o:
            print('[COUT] CO_JSON_CONTENT {}'.format(json.dumps(o)))

    return passed

def parse_flake8_result(line):
    line = line.split(':')
    if (len(line) < 4):
        return False
    return {'file': trim_repo_path(line[0]), 'line': line[1], 'col': line[2], 'msg': line[3]}

def trim_repo_path(n):
    return n[len(REPO_PATH) + 1:]

def parse_argument():
    data = os.environ.get('CO_DATA', None)
    if not data:
        return {}

    validate = ['git-url']
    ret = {}
    for s in data.split(' '):
        s = s.strip()
        if not s:
            continue
        arg = s.split('=')
        if len(arg) < 2:
            print('[COUT] Unknown Parameter: [{}]'.format(s))
            continue

        if arg[0] not in validate:
            print('[COUT] Unknown Parameter: [{}]'.format(s))
            continue

        ret[arg[0]] = arg[1]

    return ret

# git_clone('https://github.com/Lupino/python-aio-periodic.git')
# flake8('git-repo/aio_periodic/utils.py')
# print(parse_argument())

def main():
    argv = parse_argument()
    git_url = argv.get('git-url')
    if not git_url:
        print("[COUT] The git-url value is null", file=sys.stderr)
        print("[COUT] CO_RESULT = false")
        return

    if not git_clone(git_url):
        return

    all_true = True

    for root, dirs, files in os.walk(REPO_PATH):
        for file_name in files:
            if file_name.endswith('.py'):
                o = flake8(os.path.join(root, file_name))
                all_true = all_true and o

    print("[COUT] CO_RESULT = false")

main()
<commit_msg>Fixed. [COUT] CO_RESULT<commit_after>#!/usr/bin/env python3

import subprocess
import os
import sys
import json

REPO_PATH = 'git-repo'

def git_clone(url):
    r = subprocess.run(['git', 'clone', url, REPO_PATH])

    if (r.returncode == 0):
        return True
    else:
        print("[COUT] Git clone error: Invalid argument to exit", file=sys.stderr)
        print("[COUT] CO_RESULT = false")
        return False

def flake8(file_name):
    r = subprocess.run(['flake8', file_name], stderr=subprocess.PIPE,
                       stdout=subprocess.PIPE)

    passed = True
    if (r.returncode != 0):
        passed = False

    out = str(r.stdout, 'utf-8').strip().split('\n')
    for o in out:
        o = parse_flake8_result(o)
        if o:
            print('[COUT] CO_JSON_CONTENT {}'.format(json.dumps(o)))

    return passed

def parse_flake8_result(line):
    line = line.split(':')
    if (len(line) < 4):
        return False
    return {'file': trim_repo_path(line[0]), 'line': line[1], 'col': line[2], 'msg': line[3]}

def trim_repo_path(n):
    return n[len(REPO_PATH) + 1:]

def parse_argument():
    data = os.environ.get('CO_DATA', None)
    if not data:
        return {}

    validate = ['git-url']
    ret = {}
    for s in data.split(' '):
        s = s.strip()
        if not s:
            continue
        arg = s.split('=')
        if len(arg) < 2:
            print('[COUT] Unknown Parameter: [{}]'.format(s))
            continue

        if arg[0] not in validate:
            print('[COUT] Unknown Parameter: [{}]'.format(s))
            continue

        ret[arg[0]] = arg[1]

    return ret

# git_clone('https://github.com/Lupino/python-aio-periodic.git')
# flake8('git-repo/aio_periodic/utils.py')
# print(parse_argument())

def main():
    argv = parse_argument()
    git_url = argv.get('git-url')
    if not git_url:
        print("[COUT] The git-url value is null", file=sys.stderr)
        print("[COUT] CO_RESULT = false")
        return

    if not git_clone(git_url):
        return

    all_true = True

    for root, dirs, files in os.walk(REPO_PATH):
        for file_name in files:
            if file_name.endswith('.py'):
                o = flake8(os.path.join(root, file_name))
                all_true = all_true and o

    if all_true:
        print("[COUT] CO_RESULT = true")
    else:
        print("[COUT] CO_RESULT = false")

main()
<|endoftext|>


<commit_before>from conference_scheduler import parameters
import numpy as np

def test_tag_array(events):
    tag_array = parameters.tag_array(events)
    assert np.array_equal(tag_array, np.array([[1, 0], [1, 1], [0, 1]]))

def test_session_array(sessions):
    session_array = parameters.session_array(sessions)
    assert np.array_equal(session_array, np.array([[1, 1, 1, 0, 0, 0, 0],
                                                   [0, 0, 0, 1, 1, 0, 0],
                                                   [0, 0, 0, 0, 0, 1, 0],
                                                   [0, 0, 0, 0, 0, 0, 1]]))

def test_variables(shape):
    X = parameters.variables(shape)
    assert len(X) == 21

def test_schedule_all_events(shape, X):
    constraints = [c for c in parameters._schedule_all_events(shape, X)]
    assert len(constraints) == 3

def test_schedule_all_events_fails_np(shape):
    # Third talk is not scheduled
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 1, 0, 0],
                  [0, 0, 0, 0, 0, 0, 0]])
    constraint = all(parameters._schedule_all_events(shape, X))
    assert constraint is False

def test_schedule_all_events_pass_np(shape):
    # All talks are scheduled
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 1, 0, 0],
                  [0, 1, 0, 0, 0, 0, 0]])
    constraint = all(parameters._schedule_all_events(shape, X))
    assert constraint is True

def test_max_one_event_per_slot(shape, X):
    constraints = [c for c in parameters._max_one_event_per_slot(shape, X)]
    assert len(constraints) == 7

def test_max_one_events_per_slot_fail_np(shape):
    # Two talks are scheduled in the first slot
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [1, 0, 0, 0, 0, 0, 0],
                  [0, 1, 0, 0, 0, 0, 0]])
    constraint = all(parameters._max_one_event_per_slot(shape, X))
    assert constraint is False

def test_max_one_events_per_slot_pass_np(shape):
    # All slots have at most one talk
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 1, 0, 0, 0],
                  [0, 1, 0, 0, 0, 0, 0]])
    constraint = all(parameters._max_one_event_per_slot(shape, X))
    assert constraint is True

def test_slots_in_session(session_array):
    assert np.array_equal(parameters._slots_in_session(0, session_array),
                          np.array([0, 1, 2]))
    assert np.array_equal(parameters._slots_in_session(3, session_array),
                          np.array([6]))

def test_events_with_diff_tags(tag_array):
    assert np.array_equal(parameters._events_with_diff_tag(0, tag_array),
                          np.array([2]))
    assert np.array_equal(parameters._events_with_diff_tag(1, tag_array),
                          np.array([]))
    assert np.array_equal(parameters._events_with_diff_tag(2, tag_array),
                          np.array([0]))

def test_events_in_session_share_a_tag(session_array, tag_array, X):
    constraints = [c for c in
            parameters._events_in_session_share_a_tag(session_array, tag_array, X)]
    assert len(constraints) == 16

def test_events_in_session_share_a_tag_fails_np(session_array, tag_array):
    # An array where two talks are in same session but share no tag
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 0, 0, 1],
                  [0, 1, 0, 0, 0, 0, 0]])
    constraint = all(parameters._events_in_session_share_a_tag(session_array,
                                                              tag_array, X))
    assert constraint is False

def test_events_in_session_share_a_tag_passes_np(session_array, tag_array):
    # An array where no two talks are in same session if they do not share tags
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 0, 0, 1],
                  [0, 0, 0, 1, 0, 0, 0]])
    test = all(parameters._events_in_session_share_a_tag(session_array,
                                                        tag_array, X))
    assert test is True

def test_constraints(shape, session_array, tag_array, X):
    constraints = [c for c in parameters.constraints(shape, session_array,
                                                     tag_array, X)]
    assert len(constraints) == 26
<commit_msg>[#1] Fix PEP8 errors<commit_after>from conference_scheduler import parameters
import numpy as np

def test_tag_array(events):
    tag_array = parameters.tag_array(events)
    assert np.array_equal(tag_array, np.array([[1, 0], [1, 1], [0, 1]]))

def test_session_array(sessions):
    session_array = parameters.session_array(sessions)
    assert np.array_equal(session_array, np.array([[1, 1, 1, 0, 0, 0, 0],
                                                   [0, 0, 0, 1, 1, 0, 0],
                                                   [0, 0, 0, 0, 0, 1, 0],
                                                   [0, 0, 0, 0, 0, 0, 1]]))

def test_variables(shape):
    X = parameters.variables(shape)
    assert len(X) == 21

def test_schedule_all_events(shape, X):
    constraints = [c for c in parameters._schedule_all_events(shape, X)]
    assert len(constraints) == 3

def test_schedule_all_events_fails_np(shape):
    # Third talk is not scheduled
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 1, 0, 0],
                  [0, 0, 0, 0, 0, 0, 0]])
    constraint = all(parameters._schedule_all_events(shape, X))
    assert constraint is False

def test_schedule_all_events_pass_np(shape):
    # All talks are scheduled
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 1, 0, 0],
                  [0, 1, 0, 0, 0, 0, 0]])
    constraint = all(parameters._schedule_all_events(shape, X))
    assert constraint is True

def test_max_one_event_per_slot(shape, X):
    constraints = [c for c in parameters._max_one_event_per_slot(shape, X)]
    assert len(constraints) == 7

def test_max_one_events_per_slot_fail_np(shape):
    # Two talks are scheduled in the first slot
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [1, 0, 0, 0, 0, 0, 0],
                  [0, 1, 0, 0, 0, 0, 0]])
    constraint = all(parameters._max_one_event_per_slot(shape, X))
    assert constraint is False

def test_max_one_events_per_slot_pass_np(shape):
    # All slots have at most one talk
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 1, 0, 0, 0],
                  [0, 1, 0, 0, 0, 0, 0]])
    constraint = all(parameters._max_one_event_per_slot(shape, X))
    assert constraint is True

def test_slots_in_session(session_array):
    assert np.array_equal(parameters._slots_in_session(0, session_array),
                          np.array([0, 1, 2]))
    assert np.array_equal(parameters._slots_in_session(3, session_array),
                          np.array([6]))

def test_events_with_diff_tags(tag_array):
    assert np.array_equal(parameters._events_with_diff_tag(0, tag_array),
                          np.array([2]))
    assert np.array_equal(parameters._events_with_diff_tag(1, tag_array),
                          np.array([]))
    assert np.array_equal(parameters._events_with_diff_tag(2, tag_array),
                          np.array([0]))

def test_events_in_session_share_a_tag(session_array, tag_array, X):
    constraints = [c for c in
            parameters._events_in_session_share_a_tag(session_array,
                                                      tag_array, X)]
    assert len(constraints) == 16

def test_events_in_session_share_a_tag_fails_np(session_array, tag_array):
    # An array where two talks are in same session but share no tag
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 0, 0, 1],
                  [0, 1, 0, 0, 0, 0, 0]])
    constraint = all(parameters._events_in_session_share_a_tag(session_array,
                                                               tag_array, X))
    assert constraint is False

def test_events_in_session_share_a_tag_passes_np(session_array, tag_array):
    # An array where no two talks are in same session if they do not share tags
    X = np.array([[1, 0, 0, 0, 0, 0, 0],
                  [0, 0, 0, 0, 0, 0, 1],
                  [0, 0, 0, 1, 0, 0, 0]])
    test = all(parameters._events_in_session_share_a_tag(session_array,
                                                         tag_array, X))
    assert test is True

def test_constraints(shape, session_array, tag_array, X):
    constraints = [c for c in parameters.constraints(shape, session_array,
                                                     tag_array, X)]
    assert len(constraints) == 26
<|endoftext|>


<commit_before>import json

from django.utils.translation import ugettext_lazy as _
from django.db.models.fields.files import FileField, FieldFile

from rest_framework import serializers

from bulbs.images.conf import settings
from bulbs.images.storages import BettyCropperStorage

class RemoteImageSerializer(serializers.WritableField):

    def to_native(self, obj):
        return {
            'id': obj.id,
            'caption': obj.caption,
            'alt': obj.alt
        }

    def from_native(self, data):
        return json.dumps(data)

class RemoteImageFieldFile(FieldFile):
    def crop_url(self, width, ratio="original", format="jpg"):
        image_dir = ""
        for char in self.id:
            image_dir += char
            if len(image_dir) % 4 == 0:
                image_dir += "/"
        if image_dir.endswith("/"):
            image_dir = image_dir[:-1]
        return "%s/%s/%s/%s.%s" % (settings.BETTY_CROPPER['PUBLIC_URL'], image_dir, ratio, width, format)

    @property
    def id(self):
        try:
            data = json.loads(self.name)
            if isinstance(data, dict):
                return data.get('id')
        except (ValueError, TypeError):
            pass
        return self.name

    @id.setter
    def id(self, value):
        try:
            data = json.loads(self.name)
            if not isinstance(data, dict):
                data = {}
        except (ValueError, TypeError):
            data = {}
        data['id'] = value
        self.name = json.dumps(data)

    @property
    def caption(self):
        try:
            data = json.loads(self.name)
            if isinstance(data, dict):
                return data.get('caption')
        except (ValueError, TypeError):
            pass
        return None

    @caption.setter
    def caption(self, value):
        try:
            data = json.loads(self.name)
            if not isinstance(data, dict):
                data = {'id': self.name}
        except (ValueError, TypeError):
            data = {'id': self.name}
        data['caption'] = value
        self.name = json.dumps(data)

    @property
    def alt(self):
        try:
            data = json.loads(self.name)
            if isinstance(data, dict):
                return data.get('alt')
        except (ValueError, TypeError):
            pass
        return None

    @alt.setter
    def alt(self, value):
        try:
            data = json.loads(self.name)
            if not isinstance(data, dict):
                data = {'id': self.name}
        except (ValueError, TypeError):
            data = {'id': self.name}
        data['alt'] = value
        self.name = json.dumps(data)

class RemoteImageField(FileField):
    description = _("RemoteImage")
    attr_class = RemoteImageFieldFile

    def __init__(self, verbose_name=None, name=None, upload_to='', storage=None, **kwargs):
        for arg in ('primary_key', 'unique'):
            if arg in kwargs:
                raise TypeError("'%s' is not a valid argument for %s." % (arg, self.__class__))

        self.storage = BettyCropperStorage()
        self.upload_to = settings.BETTY_CROPPER['ADMIN_URL']

        kwargs['max_length'] = kwargs.get('max_length', 100)
        super(FileField, self).__init__(verbose_name, name, **kwargs)

    def get_prep_lookup(self, lookup_type, value):
        if hasattr(value, 'name'):
            value = value.name
        return super(FileField, self).get_prep_lookup(lookup_type, value)
<commit_msg>Fuck me<commit_after>import json

from django.utils.translation import ugettext_lazy as _
from django.db.models.fields.files import FileField, FieldFile

from rest_framework import serializers

from bulbs.images.conf import settings
from bulbs.images.storages import BettyCropperStorage

class RemoteImageSerializer(serializers.WritableField):

    def to_native(self, obj):
        return {
            'id': obj.id,
            'caption': obj.caption,
            'alt': obj.alt
        }

    def from_native(self, data):
        return json.dumps(data)

class RemoteImageFieldFile(FieldFile):
    def crop_url(self, width, ratio="original", format="jpg"):
        image_dir = ""
        for char in self.id:
            image_dir += char
            if len(image_dir) % 4 == 0:
                image_dir += "/"
        if image_dir.endswith("/"):
            image_dir = image_dir[:-1]
        return "%s/%s/%s/%s.%s" % (settings.BETTY_CROPPER['PUBLIC_URL'], image_dir, ratio, width, format)

    @property
    def id(self):
        try:
            data = json.loads(self.name)
            if isinstance(data, dict):
                return data.get('id')
        except (ValueError, TypeError):
            pass
        return self.name

    @id.setter
    def id(self, value):
        try:
            data = json.loads(self.name)
            if not isinstance(data, dict):
                data = {}
        except (ValueError, TypeError):
            data = {}
        data['id'] = value
        self.name = json.dumps(data)

    @property
    def caption(self):
        try:
            data = json.loads(self.name)
            if isinstance(data, dict):
                return data.get('caption')
        except (ValueError, TypeError):
            pass
        return None

    @caption.setter
    def caption(self, value):
        try:
            data = json.loads(self.name)
            if not isinstance(data, dict):
                data = {'id': self.name}
        except (ValueError, TypeError):
            data = {'id': self.name}
        data['caption'] = value
        self.name = json.dumps(data)

    @property
    def alt(self):
        try:
            data = json.loads(self.name)
            if isinstance(data, dict):
                return data.get('alt')
        except (ValueError, TypeError):
            pass
        return None

    @alt.setter
    def alt(self, value):
        try:
            data = json.loads(self.name)
            if not isinstance(data, dict):
                data = {'id': self.name}
        except (ValueError, TypeError):
            data = {'id': self.name}
        data['alt'] = value
        self.name = json.dumps(data)

class RemoteImageField(FileField):
    description = _("RemoteImage")
    attr_class = RemoteImageFieldFile

    def __init__(self, verbose_name=None, name=None, upload_to='', storage=None, **kwargs):
        for arg in ('primary_key', 'unique'):
            if arg in kwargs:
                raise TypeError("'%s' is not a valid argument for %s." % (arg, self.__class__))

        self.storage = BettyCropperStorage()
        self.upload_to = settings.BETTY_CROPPER['ADMIN_URL']

        kwargs['max_length'] = kwargs.get('max_length', 100)
        super(RemoteImageField, self).__init__(verbose_name, name, **kwargs)

    def get_prep_lookup(self, lookup_type, value):
        if hasattr(value, 'name'):
            value = value.name
        return super(RemoteImageField, self).get_prep_lookup(lookup_type, value)
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
"""
.. module:: burpui.api
    :platform: Unix
    :synopsis: Burp-UI api module.

.. moduleauthor:: Ziirish <hi+burpui@ziirish.me>

"""
import os
import sys
import logging

from flask import Blueprint, Response, request, current_app, session
from flask_restplus import Api as ApiPlus
from flask_login import current_user
from importlib import import_module
from functools import wraps

from .custom.namespace import Namespace
from ..server import BUIServer  # noqa
from ..exceptions import BUIserverException
from ..config import config
from ..ext.cache import cache

bui = current_app  # type: BUIServer
EXEMPT_METHODS = set(['OPTIONS'])

def cache_key():
    return '{}-{}-{}-{}'.format(current_user.name, request.path, request.values, session.get('language', ''))

def api_login_required(func):
    """Custom login decorator that is able to parse Basic credentials as well as
    Cookies set with the traditional login.
    """
    @wraps(func)
    def decorated_view(*args, **kwargs):
        """decorator"""
        if request.method in EXEMPT_METHODS:
            return func(*args, **kwargs)
        # 'func' is a Flask.view.MethodView so we have access to some special
        # params
        cls = func.view_class
        login_required = getattr(cls, 'login_required', True)
        if (bui.auth != 'none' and
                login_required and
                not bui.config.get('LOGIN_DISABLED', False)):
            if not current_user.is_authenticated:
                if request.headers.get('X-From-UI', False):
                    return Response('Access denied', 403)
                return Response(
                    'Could not verify your access level for that URL.\n'
                    'You have to login with proper credentials', 401,
                    {'WWW-Authenticate': 'Basic realm="Login Required"'})
        return func(*args, **kwargs)
    return decorated_view

class Api(ApiPlus):
    """Wrapper class around :class:`flask_restplus.Api`"""
    logger = logging.getLogger('burp-ui')
    # TODO: should use global object instead of reference
    cache = cache
    loaded = False
    release = None
    __doc__ = None
    __url__ = None
    CELERY_REQUIRED = ['async']

    def load_all(self):
        if config['WITH_LIMIT']:
            from ..ext.limit import limiter
            self.decorators.append(limiter.limit(config['BUI_RATIO']))
        """hack to automatically import api modules"""
        if not self.loaded:
            sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
            self.loaded = True
            for f in os.listdir(__path__[0]):
                name, ext = os.path.splitext(f)
                if (os.path.isfile(os.path.join(__path__[0], f)) and
                        ext == '.py' and
                        name not in ['__init__', '.', '..']):
                    mod = '.' + name
                    if name not in self.CELERY_REQUIRED or config['WITH_CELERY']:
                        self.logger.debug('Loading API module: {}'.format(mod))
                        try:
                            import_module(mod, __name__)
                        except:
                            self.logger.critical('Unable to load {}'.format(mod))
                    else:
                        self.logger.warning('Skipping API module: {}'.format(mod))

    def acl_admin_required(self, message='Access denied', code=403):
        def decorator(func):
            @wraps(func)
            def decorated(resource, *args, **kwargs):
                if not resource.is_admin:
                    resource.abort(code, message)
                return func(resource, *args, **kwargs)
            return decorated
        return decorator

    def acl_own_or_admin(self, key='name', message='Access denied', code=403):
        def decorator(func):
            @wraps(func)
            def decorated(resource, *args, **kwargs):
                if key not in kwargs:
                    resource.abort(500, "key '{}' not found".format(key))
                if kwargs[key] != resource.username and not resource.is_admin:
                    resource.abort(code, message)
                return func(resource, *args, **kwargs)
            return decorated
        return decorator

    def disabled_on_demo(self):
        def decorator(func):
            @wraps(func)
            def decorated(resource, *args, **kwargs):
                if config['BUI_DEMO']:
                    resource.abort(405, 'Sorry, this feature is not available on the demo')
                return func(resource, *args, **kwargs)
            return decorated
        return decorator

    def namespace(self, *args, **kwargs):
        """A namespace factory

        :returns Namespace: a new namespace instance
        """
        ns = Namespace(*args, **kwargs)
        self.add_namespace(ns)
        return ns

apibp = Blueprint('api', __name__, url_prefix='/api')
api = Api(apibp, title='Burp-UI API', description='Burp-UI API to interact with burp', doc='/doc', decorators=[api_login_required])

@api.errorhandler(BUIserverException)
def handle_bui_server_exception(error):
    """Forward a BUIserverException to the final user

    :param error: Custom exception
    :type error: :class:`burpui.exceptions.BUIserverException`
    """
    return {'message': error.description}, error.code
<commit_msg>give failure details<commit_after># -*- coding: utf8 -*-
"""
.. module:: burpui.api
    :platform: Unix
    :synopsis: Burp-UI api module.

.. moduleauthor:: Ziirish <hi+burpui@ziirish.me>

"""
import os
import sys
import logging

from flask import Blueprint, Response, request, current_app, session
from flask_restplus import Api as ApiPlus
from flask_login import current_user
from importlib import import_module
from functools import wraps

from .custom.namespace import Namespace
from ..server import BUIServer  # noqa
from ..exceptions import BUIserverException
from ..config import config
from ..ext.cache import cache

bui = current_app  # type: BUIServer
EXEMPT_METHODS = set(['OPTIONS'])

def cache_key():
    return '{}-{}-{}-{}'.format(current_user.name, request.path, request.values, session.get('language', ''))

def api_login_required(func):
    """Custom login decorator that is able to parse Basic credentials as well as
    Cookies set with the traditional login.
    """
    @wraps(func)
    def decorated_view(*args, **kwargs):
        """decorator"""
        if request.method in EXEMPT_METHODS:
            return func(*args, **kwargs)
        # 'func' is a Flask.view.MethodView so we have access to some special
        # params
        cls = func.view_class
        login_required = getattr(cls, 'login_required', True)
        if (bui.auth != 'none' and
                login_required and
                not bui.config.get('LOGIN_DISABLED', False)):
            if not current_user.is_authenticated:
                if request.headers.get('X-From-UI', False):
                    return Response('Access denied', 403)
                return Response(
                    'Could not verify your access level for that URL.\n'
                    'You have to login with proper credentials', 401,
                    {'WWW-Authenticate': 'Basic realm="Login Required"'})
        return func(*args, **kwargs)
    return decorated_view

class Api(ApiPlus):
    """Wrapper class around :class:`flask_restplus.Api`"""
    logger = logging.getLogger('burp-ui')
    # TODO: should use global object instead of reference
    cache = cache
    loaded = False
    release = None
    __doc__ = None
    __url__ = None
    CELERY_REQUIRED = ['async']

    def load_all(self):
        if config['WITH_LIMIT']:
            from ..ext.limit import limiter
            self.decorators.append(limiter.limit(config['BUI_RATIO']))
        """hack to automatically import api modules"""
        if not self.loaded:
            sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
            self.loaded = True
            for f in os.listdir(__path__[0]):
                name, ext = os.path.splitext(f)
                if (os.path.isfile(os.path.join(__path__[0], f)) and
                        ext == '.py' and
                        name not in ['__init__', '.', '..']):
                    mod = '.' + name
                    if name not in self.CELERY_REQUIRED or config['WITH_CELERY']:
                        self.logger.debug('Loading API module: {}'.format(mod))
                        try:
                            import_module(mod, __name__)
                        except:
                            import traceback
                            self.logger.critical('Unable to load {}:\n{}'.format(mod, traceback.format_exc()))
                    else:
                        self.logger.warning('Skipping API module: {}'.format(mod))

    def acl_admin_required(self, message='Access denied', code=403):
        def decorator(func):
            @wraps(func)
            def decorated(resource, *args, **kwargs):
                if not resource.is_admin:
                    resource.abort(code, message)
                return func(resource, *args, **kwargs)
            return decorated
        return decorator

    def acl_own_or_admin(self, key='name', message='Access denied', code=403):
        def decorator(func):
            @wraps(func)
            def decorated(resource, *args, **kwargs):
                if key not in kwargs:
                    resource.abort(500, "key '{}' not found".format(key))
                if kwargs[key] != resource.username and not resource.is_admin:
                    resource.abort(code, message)
                return func(resource, *args, **kwargs)
            return decorated
        return decorator

    def disabled_on_demo(self):
        def decorator(func):
            @wraps(func)
            def decorated(resource, *args, **kwargs):
                if config['BUI_DEMO']:
                    resource.abort(405, 'Sorry, this feature is not available on the demo')
                return func(resource, *args, **kwargs)
            return decorated
        return decorator

    def namespace(self, *args, **kwargs):
        """A namespace factory

        :returns Namespace: a new namespace instance
        """
        ns = Namespace(*args, **kwargs)
        self.add_namespace(ns)
        return ns

apibp = Blueprint('api', __name__, url_prefix='/api')
api = Api(apibp, title='Burp-UI API', description='Burp-UI API to interact with burp', doc='/doc', decorators=[api_login_required])

@api.errorhandler(BUIserverException)
def handle_bui_server_exception(error):
    """Forward a BUIserverException to the final user

    :param error: Custom exception
    :type error: :class:`burpui.exceptions.BUIserverException`
    """
    return {'message': error.description}, error.code
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-

import matplotlib.pyplot as plt

def pretty_title(ax = None, message = '', description = ''):
	"""Adds a pretty title to the axes with a message and a description.

	Args:
	    ax: The Axes object you want to put the title on, defaults to current.
	    message: A headline with the business interpretation of the plot (e.g. "Amazon revenues more than doubled in the last 4 years")
	    description: A short description of what's plotted (e.g. "Amazon revenues from 2012 to 2016")

	Returns:
	    The title object (a Text artist)

	"""
	if not message and not description:
		return None
	if not ax:
		ax = plt.gca()
	ax.set_title("\n{}\n — {} — \n".format(message, description), fontsize=18)
<commit_msg>Decreased subtitle font size<commit_after># -*- encoding: utf-8 -*-

import matplotlib.pyplot as plt

def pretty_title(ax = None, message = '', description = ''):
	"""Adds a pretty title to the axes with a message and a description.

	Args:
	    ax: The Axes object you want to put the title on, defaults to current.
	    message: A headline with the business interpretation of the plot (e.g. "Amazon revenues more than doubled in the last 4 years")
	    description: A short description of what's plotted (e.g. "Amazon revenues from 2012 to 2016")

	Returns:
	    The title object (a Text artist)

	"""
	if not message and not description:
		return None
	if not ax:
		ax = plt.gca()
	# TODO: find a correct way to do this (esp. the Axes margin part)
	ax.set_title("\n{}\n\n".format(message), fontsize=18)
	ax.text(0.5, 1, "— {} —\n\n\n".format(description), fontsize=14, transform=ax.transAxes, ha='center', va='center')
<|endoftext|>


<commit_before>"""
This submodule provides utility functions to act on `sunpy.map.GenericMap` instances.
"""
from itertools import chain, product

import numpy as np

from astropy import units as u
from astropy.coordinates import SkyCoord

from sunpy.coordinates import Helioprojective

__all__ = ['all_pixel_indices_from_map', 'all_coordinates_from_map',
           'map_edges', 'contains_full_disk',
           'is_all_off_disk', 'is_all_on_disk', 'contains_limb',
           'coordinate_is_on_disk', 'on_disk_bounding_coordinates']

def all_pixel_indices_from_map(smap):
    """
    Returns pixel pair indices of every pixel in a map.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        A SunPy map.

    Returns
    -------
    out : `~numpy.array`
        A numpy array with the all the pixel indices built from the
        dimensions of the map.
    """
    return np.meshgrid(*[np.arange(v.value) for v in smap.dimensions]) * u.pix

def all_coordinates_from_map(smap):
    """
    Returns the co-ordinates of every pixel in a map.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        A SunPy map.

    Returns
    -------
    out : `~astropy.coordinates.SkyCoord`
        An array of sky coordinates in the coordinate system "coordinate_system".
    """
    x, y = all_pixel_indices_from_map(smap)
    return smap.pixel_to_world(x, y)

def map_edges(smap):
    """
    Returns the pixel locations of the edges of a rectangular input map.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    maps_edges : `~dict`
        Returns the pixels of edge of the map
    """
    # Calculate all the edge pixels
    nx, ny = smap.dimensions.x.value, smap.dimensions.y.value
    bottom = list(product([0.0], np.arange(nx))) * u.pix
    top = list(product([ny - 1], np.arange(nx))) * u.pix
    lhs = list(product(np.arange(ny), [0])) * u.pix
    rhs = list(product(np.arange(ny), [nx - 1])) * u.pix
    return {"top": top, "bottom": bottom, "lhs": lhs, "rhs": rhs}

def contains_full_disk(smap):
    """
    Checks if a map contains the full disk of the Sun.

    A map contains the full disk of the Sun if the following two
    conditions are met: (1) all the pixels at the edge of the map are
    more than 1 solar radius from the center of the Sun and, (2) the
    map is not all off disk.  If both these conditions are met, the
    function returns True. Otherwise, the function returns False.
    Note that the function assumes that the input map is rectangular.
    Note also that in the case of coronagraph images the disk itself
    need not be observed.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    contains_full_disk : `~bool`
        Returns False if any of the edge pixels are less than one solar radius
        away from the center of the Sun.
    """
    # Calculate all the edge pixels
    edges = map_edges(smap)
    edge_pixels = list(chain.from_iterable([edges["lhs"], edges["rhs"], edges["top"], edges["bottom"]]))
    x = [p[0] for p in edge_pixels] * u.pix
    y = [p[1] for p in edge_pixels] * u.pix

    # Calculate the edge of the world
    edge_of_world = smap.pixel_to_world(x, y).transform_to(Helioprojective)

    # Calculate the distance of the edge of the world in solar radii
    distance = u.R_sun * np.sqrt(edge_of_world.Tx ** 2 + edge_of_world.Ty ** 2) / smap.rsun_obs

    # Test if all the edge pixels are more than one solar radius distant
    # and that the whole map is not all off disk.
    return np.all(distance > 1*u.R_sun) and ~is_all_off_disk(smap)

def is_all_off_disk(smap):
    """
    Checks if the entire map is off the solar disk.

    The check is performed by calculating the distance of every
    pixel from the center of the Sun.  If they are all off-disk,
    then the function returns True. Otherwise, the function
    returns False.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    is_all_off_disk : `~bool`
        Returns True if all map pixels are strictly more than one solar radius
        away from the center of the Sun.
    """
    return np.all(~coordinate_is_on_disk(all_coordinates_from_map(smap), smap.rsun_obs))

def is_all_on_disk(smap):
    """
    Checks if the entire map is on the solar disk.

    The check is performed by calculating the distance of every pixel
    from the center of the Sun.  If they are all on-disk, then the
    function returns True. Otherwise, the function returns False.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    is_all_on_disk : `~bool`
        Returns True if all map pixels are strictly less than one solar radius
        away from the center of the Sun.
    """
    return np.all(coordinate_is_on_disk(all_coordinates_from_map(smap), smap.rsun_obs))

def contains_limb(smap):
    """
    Checks if a map contains a portion of the solar limb.

    The check is performed by calculating the distance of every pixel from
    the center of the Sun.  If at least one pixel is on disk and at least one
    pixel is off disk, the function returns True.  Otherwise, the function
    returns False. Note that this function will also true if the entire solar
    limb is within the field of view of the map.  Note also that in the case
    of coronagraph images the limb itself need not be observed.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    contains_limb : `~bool`
        Returns True If at least one pixel is on disk and at least one pixel
        is off disk.
    """
    on_disk = coordinate_is_on_disk(all_coordinates_from_map(smap), smap.rsun_obs)
    return np.logical_and(np.any(on_disk), np.any(~on_disk))

@u.quantity_input
def coordinate_is_on_disk(coordinate, scale: u.arcsecond):
    """
    Checks if the helioprojective Cartesian coordinate is on disk.

    The check is performed by comparing the coordinate's distance
    from the center of the Sun to the solar radius.

    Parameters
    ----------
    coordinate : `~astropy.coordinates.SkyCoord`, `~sunpy.coordinates.frames.Helioprojective`
        The input coordinate.  The frame property of the input
        coordinate must be helioprojective.

    scale : `~astropy.units.Quantity`
        The pixel scale size in units of arcseconds.

    Returns
    -------
    is_on_disk : `~bool`
        Returns True if the coordinate is on disk, False otherwise.
    """
    # Calculate the radii of every pixel in helioprojective Cartesian
    # co-ordinate distance units.
    if not isinstance(coordinate.frame, Helioprojective):
        raise ValueError("Input coordinate must be in the Helioprojective Cartesian frame.")
    return u.R_sun * (np.sqrt(coordinate.Tx ** 2 + coordinate.Ty ** 2) / scale) < 1 * u.R_sun

def on_disk_bounding_coordinates(smap):
    """
    Returns the the bottom left and top-right coordinates of a minimal
    rectangular region that contains all the on-disk pixels in the input map.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    (bl, tr) : `~astropy.coordinates.SkyCoord`
        Returns the bottom left and top right coordinates that bound the
        spatial location of all the on-disk pixels.
    """
    # Check that the input map is not all off disk.
    if is_all_off_disk(smap):
        raise ValueError("The entire map is off disk.")

    # Get all the coordinates from the input map
    coordinates = all_coordinates_from_map(smap)

    # Find which coordinates are on the disk
    on_disk = coordinate_is_on_disk(coordinates, smap.rsun_obs)
    on_disk_coordinates = coordinates[on_disk]

    # The bottom left and top right coordinates that contain
    # the on disk coordinates.
    tx = on_disk_coordinates.Tx.value
    ty = on_disk_coordinates.Ty.value
    return SkyCoord([np.nanmin(tx), np.nanmax(tx)] * u.arcsec,
                    [np.nanmin(ty), np.nanmax(ty)] * u.arcsec,
                    frame=Helioprojective, observer=smap.observer_coordinate)
<commit_msg>updated docs<commit_after>"""
This submodule provides utility functions to act on `sunpy.map.GenericMap` instances.
"""
from itertools import chain, product

import numpy as np

from astropy import units as u
from astropy.coordinates import SkyCoord

from sunpy.coordinates import Helioprojective

__all__ = ['all_pixel_indices_from_map', 'all_coordinates_from_map',
           'map_edges', 'contains_full_disk',
           'is_all_off_disk', 'is_all_on_disk', 'contains_limb',
           'coordinate_is_on_disk', 'on_disk_bounding_coordinates']

def all_pixel_indices_from_map(smap):
    """
    Returns pixel pair indices of every pixel in a map.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        A SunPy map.

    Returns
    -------
    out : `~numpy.array`
        A numpy array with the all the pixel indices built from the
        dimensions of the map.
    """
    return np.meshgrid(*[np.arange(v.value) for v in smap.dimensions]) * u.pix

def all_coordinates_from_map(smap):
    """
    Returns the co-ordinates of every pixel in a map.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        A SunPy map.

    Returns
    -------
    out : `~astropy.coordinates.SkyCoord`
        An array of sky coordinates in the coordinate system "coordinate_system".
    """
    x, y = all_pixel_indices_from_map(smap)
    return smap.pixel_to_world(x, y)

def map_edges(smap):
    """
    Returns the pixel locations of the edges of a rectangular input map.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    maps_edges : `~dict`
        Returns the pixels of edge of the map
    """
    # Calculate all the edge pixels
    nx, ny = smap.dimensions.x.value, smap.dimensions.y.value
    bottom = list(product([0.0], np.arange(nx))) * u.pix
    top = list(product([ny - 1], np.arange(nx))) * u.pix
    lhs = list(product(np.arange(ny), [0])) * u.pix
    rhs = list(product(np.arange(ny), [nx - 1])) * u.pix
    return {"top": top, "bottom": bottom, "lhs": lhs, "rhs": rhs}

def contains_full_disk(smap):
    """
    Checks if a map contains the full disk of the Sun.

    A map contains the full disk of the Sun if the following two
    conditions are met: (1) all the pixels at the edge of the map are
    more than 1 solar radius from the center of the Sun and, (2) the
    map is not all off disk.  If both these conditions are met, the
    function returns True. Otherwise, the function returns False.
    Note that the function assumes that the input map is rectangular.
    Note also that in the case of coronagraph images the disk itself
    need not be observed.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    contains_full_disk : `~bool`
        Returns False if any of the edge pixels are less than one solar radius
        away from the center of the Sun.
    """
    # Calculate all the edge pixels
    edges = map_edges(smap)
    edge_pixels = list(chain.from_iterable([edges["lhs"], edges["rhs"], edges["top"], edges["bottom"]]))
    x = [p[0] for p in edge_pixels] * u.pix
    y = [p[1] for p in edge_pixels] * u.pix

    # Calculate the edge of the world
    edge_of_world = smap.pixel_to_world(x, y).transform_to(Helioprojective)

    # Calculate the distance of the edge of the world in solar radii
    distance = u.R_sun * np.sqrt(edge_of_world.Tx ** 2 + edge_of_world.Ty ** 2) / smap.rsun_obs

    # Test if all the edge pixels are more than one solar radius distant
    # and that the whole map is not all off disk.
    return np.all(distance > 1*u.R_sun) and ~is_all_off_disk(smap)

def is_all_off_disk(smap):
    """
    Checks if the entire map is off the solar disk.

    The check is performed by calculating the distance of every
    pixel from the center of the Sun.  If they are all off-disk,
    then the function returns True. Otherwise, the function
    returns False.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    is_all_off_disk : `~bool`
        Returns True if all map pixels are strictly more than one solar radius
        away from the center of the Sun.
    """
    return np.all(~coordinate_is_on_disk(all_coordinates_from_map(smap), smap.rsun_obs))

def is_all_on_disk(smap):
    """
    Checks if the entire map is on the solar disk.

    The check is performed by calculating the distance of every pixel
    from the center of the Sun.  If they are all on-disk, then the
    function returns True. Otherwise, the function returns False.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    is_all_on_disk : `~bool`
        Returns True if all map pixels are strictly less than one solar radius
        away from the center of the Sun.
    """
    return np.all(coordinate_is_on_disk(all_coordinates_from_map(smap), smap.rsun_obs))

def contains_limb(smap):
    """
    Checks if a map contains a portion of the solar limb.

    The check is performed by calculating the distance of every pixel from
    the center of the Sun.  If at least one pixel is on disk and at least one
    pixel is off disk, the function returns True.  Otherwise, the function
    returns False. Note that this function will also true if the entire solar
    limb is within the field of view of the map.  Note also that in the case
    of coronagraph images the limb itself need not be observed.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    contains_limb : `~bool`
        Returns True If at least one pixel is on disk and at least one pixel
        is off disk.
    """
    on_disk = coordinate_is_on_disk(all_coordinates_from_map(smap), smap.rsun_obs)
    return np.logical_and(np.any(on_disk), np.any(~on_disk))

@u.quantity_input
def coordinate_is_on_disk(coordinate, scale: u.arcsecond):
    """
    Checks if the helioprojective Cartesian coordinate is on disk.

    The check is performed by comparing the coordinate's distance
    from the center of the Sun to the solar radius.

    Parameters
    ----------
    coordinate : `~astropy.coordinates.SkyCoord`, `~sunpy.coordinates.frames.Helioprojective`
        The input coordinate.  The frame property of the input
        coordinate must be helioprojective.

    scale : `~astropy.units.Quantity`
        The pixel scale size in units of arcseconds.

    Returns
    -------
    is_on_disk : `~bool`
        Returns True if the coordinate is on disk, False otherwise.
    """
    # Calculate the radii of every pixel in helioprojective Cartesian
    # co-ordinate distance units.
    if not isinstance(coordinate.frame, Helioprojective):
        raise ValueError("Input coordinate must be in the Helioprojective Cartesian frame.")
    return u.R_sun * (np.sqrt(coordinate.Tx ** 2 + coordinate.Ty ** 2) / scale) < 1 * u.R_sun

def on_disk_bounding_coordinates(smap):
    """
    Returns the the bottom left and top-right coordinates of the smallest
    rectangular region that contains all the on-disk pixels in the input map.

    Parameters
    ----------
    smap : `~sunpy.map.GenericMap`
        The input map.

    Returns
    -------
    bl, tr : `~astropy.coordinates.SkyCoord`
        Returns a SkyCoord of length 2 such that the first entry is the bottom left
        coordinate and the second entry is the top right coordinate of the smallest
        rectangular region that contains all the on-disk pixels in the input map.
    """
    # Check that the input map is not all off disk.
    if is_all_off_disk(smap):
        raise ValueError("The entire map is off disk.")

    # Get all the coordinates from the input map
    coordinates = all_coordinates_from_map(smap)

    # Find which coordinates are on the disk
    on_disk = coordinate_is_on_disk(coordinates, smap.rsun_obs)
    on_disk_coordinates = coordinates[on_disk]

    # The bottom left and top right coordinates that contain
    # the on disk coordinates.
    tx = on_disk_coordinates.Tx.value
    ty = on_disk_coordinates.Ty.value
    return SkyCoord([np.nanmin(tx), np.nanmax(tx)] * u.arcsec,
                    [np.nanmin(ty), np.nanmax(ty)] * u.arcsec,
                    frame=Helioprojective, observer=smap.observer_coordinate)
<|endoftext|>


<commit_before>#! /usr/bin/env python3
"""

A supervisor-like XML-RPC server that offers an interface to retrieve resource
usage and limits of processes, and access to application log files (in addition
to the stdout and stderr access that supervisord already provides).

To use this, copy this file to each server you want to manage, and run it as
root (perhaps from supervisor itself).  It listens on 0.0.0.0:9002, so that the
suponoff web app can connect to it.  It requires the python module `psutil' to
be installed.

"""

import time
import os
import os.path
import logging.handlers
import shlex
import psutil
from datetime import date
from xmlrpc.server import SimpleXMLRPCServer

LOG = logging.getLogger()

def get_application_logfile(argv):
    for argi in range(len(argv)):
        if argi < (len(argv) - 1) and argv[argi] in ["--logfile", "--log-file"]:
            #print(argv[argi + 1])
            return argv[argi + 1]
        if argv[argi:argi+2] == ['bash', '-c']:
            argv1 = shlex.split(argv[argi + 2])
            return get_application_logfile(argv1)

class MonHelperRPCInterface:  # pylint: disable=R0923
    def __init__(self):
        pass

    def getProcessResourceUsage(self, pid_list):
        try:
            results = {}
            for pid in pid_list:
                if pid == 0:
                    continue
                result = {}
                results[str(pid)] = result
                try:
                    proc = psutil.Process(pid)
                except psutil.NoSuchProcess as e:
                    LOG.info("Process %s: %s", pid, e.msg)
                    continue
                except:
                    LOG.exception("Process %s:", pid)
                    continue
                result["fileno"] = proc.num_fds()
                try:
                    proc.rlimit
                except AttributeError:
                    max_fileno = -1
                    max_vmsize = -1
                else:
                    max_fileno = proc.rlimit(psutil.RLIMIT_NOFILE)[0]
                    max_vmsize = proc.rlimit(psutil.RLIMIT_AS)[0]
                if max_fileno != -1:
                    result["max_fileno"] = max_fileno

                result["numconnections"] = len(proc.connections('all'))
                result["numfiles"] = len(proc.open_files())

                if max_vmsize != -1:
                    result["max_vmsize"] = str(max_vmsize)
                result["vmsize"] = str(proc.memory_info()[1])
                result["numchildren"] = len(proc.children())
                result["numthreads"] = proc.num_threads()
                result["cpu"] = ",".join(str(x) for x in
                                         [time.time()] +
                                         list(proc.cpu_times()))
                result["diskio"] = ",".join(str(x) for x in
                                            [time.time()] +
                                            list(proc.io_counters()))
            return results
        except:
            LOG.exception("Error")
            return {}

    def tailApplicationLog(self, pid, offset, length):
        try:
            cmdline = psutil.Process(pid).cmdline()
            logfile = get_application_logfile(cmdline)

            if logfile is None:
                return ['', 0, False]

            if not os.path.exists(logfile):
                # logbook rotated logfiles have a date even for today
                base, ext = os.path.splitext(logfile)
                logfile2 = "{}-{}{}".format(base,
                                            date.today().strftime("%Y-%m-%d"),
                                            ext)
                if os.path.exists(logfile2):
                    logfile = logfile2

            if not os.path.exists(logfile):
                # rotatelogs rotated logfiles...
                logfile = "{}.{}".format(logfile,
                                         date.today().strftime("%Y-%m-%d"))

            if logfile is None or not os.path.exists(logfile):
                return ['', 0, False]

            return tailFile(logfile, int(offset), int(length))
        except:
            LOG.exception("Error")
            return ['', 0, False]

# copied from supervisor.options
def tailFile(filename, offset, length):
    """
    Read length bytes from the file named by filename starting at
    offset, automatically increasing offset and setting overflow
    flag if log size has grown beyond (offset + length).  If length
    bytes are not available, as many bytes as are available are returned.
    """

    overflow = False
    try:
        f = open(filename, 'rb')
        f.seek(0, 2)
        sz = f.tell()

        if sz > (offset + length):
            overflow = True
            offset = sz - 1

        if (offset + length) > sz:
            if offset > (sz - 1):
                length = 0
            offset = sz - length

        if offset < 0:
            offset = 0
        if length < 0:
            length = 0

        if length == 0:
            data = ''
        else:
            f.seek(offset)
            data = f.read(length)

        offset = sz
        return [data, offset, overflow]

    except (OSError, IOError):
        return ['', offset, False]

# copied from supervisor.options
def split_namespec(namespec):
    names = namespec.split(':', 1)
    if len(names) == 2:
        # group and and process name differ
        group_name, process_name = names
        if not process_name or process_name == '*':
            process_name = None
    else:
        # group name is same as process name
        group_name, process_name = namespec, namespec
    return group_name, process_name

def main():
    logging.basicConfig(format="%(asctime)s %(levelname)s "
                               "%(filename)s:%(lineno)d %(message)s")

    LOG.setLevel(logging.INFO)

    LOG.info("psutil version %s at %s", psutil.__version__, psutil.__file__)

    server = SimpleXMLRPCServer(("0.0.0.0", 9002))
    server.register_introspection_functions()
    server.register_instance(MonHelperRPCInterface())
    server.serve_forever()

if __name__ == '__main__':
    main()
<commit_msg>Monhelper: added the options "--address" and "--port"<commit_after>#! /usr/bin/env python3
"""

A supervisor-like XML-RPC server that offers an interface to retrieve resource
usage and limits of processes, and access to application log files (in addition
to the stdout and stderr access that supervisord already provides).

To use this, copy this file to each server you want to manage and run it,
perhaps from supervisor itself (it may need to be run as root).
By default, it listens on 0.0.0.0:9002, so that the suponoff web app can
connect to it. This may pose a security threat: use option "-a 127.0.0.0.1"
when the suponoff app is on the same machine or when the traffic can be
channelled through a secure connection (e.g., SSH or https).
It requires the python module `psutil' to be installed.

"""

import time
import os
import os.path
import logging.handlers
import shlex
import psutil
from datetime import date
from xmlrpc.server import SimpleXMLRPCServer

LOG = logging.getLogger()

def get_application_logfile(argv):
    for argi in range(len(argv)):
        if argi < (len(argv) - 1) and argv[argi] in ["--logfile", "--log-file"]:
            #print(argv[argi + 1])
            return argv[argi + 1]
        if argv[argi:argi+2] == ['bash', '-c']:
            argv1 = shlex.split(argv[argi + 2])
            return get_application_logfile(argv1)

class MonHelperRPCInterface:  # pylint: disable=R0923
    def __init__(self):
        pass

    def getProcessResourceUsage(self, pid_list):
        try:
            results = {}
            for pid in pid_list:
                if pid == 0:
                    continue
                result = {}
                results[str(pid)] = result
                try:
                    proc = psutil.Process(pid)
                except psutil.NoSuchProcess as e:
                    LOG.info("Process %s: %s", pid, e.msg)
                    continue
                except:
                    LOG.exception("Process %s:", pid)
                    continue
                result["fileno"] = proc.num_fds()
                try:
                    proc.rlimit
                except AttributeError:
                    max_fileno = -1
                    max_vmsize = -1
                else:
                    max_fileno = proc.rlimit(psutil.RLIMIT_NOFILE)[0]
                    max_vmsize = proc.rlimit(psutil.RLIMIT_AS)[0]
                if max_fileno != -1:
                    result["max_fileno"] = max_fileno

                result["numconnections"] = len(proc.connections('all'))
                result["numfiles"] = len(proc.open_files())

                if max_vmsize != -1:
                    result["max_vmsize"] = str(max_vmsize)
                result["vmsize"] = str(proc.memory_info()[1])
                result["numchildren"] = len(proc.children())
                result["numthreads"] = proc.num_threads()
                result["cpu"] = ",".join(str(x) for x in
                                         [time.time()] +
                                         list(proc.cpu_times()))
                result["diskio"] = ",".join(str(x) for x in
                                            [time.time()] +
                                            list(proc.io_counters()))
            return results
        except:
            LOG.exception("Error")
            return {}

    def tailApplicationLog(self, pid, offset, length):
        try:
            cmdline = psutil.Process(pid).cmdline()
            logfile = get_application_logfile(cmdline)

            if logfile is None:
                return ['', 0, False]

            if not os.path.exists(logfile):
                # logbook rotated logfiles have a date even for today
                base, ext = os.path.splitext(logfile)
                logfile2 = "{}-{}{}".format(base,
                                            date.today().strftime("%Y-%m-%d"),
                                            ext)
                if os.path.exists(logfile2):
                    logfile = logfile2

            if not os.path.exists(logfile):
                # rotatelogs rotated logfiles...
                logfile = "{}.{}".format(logfile,
                                         date.today().strftime("%Y-%m-%d"))

            if logfile is None or not os.path.exists(logfile):
                return ['', 0, False]

            return tailFile(logfile, int(offset), int(length))
        except:
            LOG.exception("Error")
            return ['', 0, False]

# copied from supervisor.options
def tailFile(filename, offset, length):
    """
    Read length bytes from the file named by filename starting at
    offset, automatically increasing offset and setting overflow
    flag if log size has grown beyond (offset + length).  If length
    bytes are not available, as many bytes as are available are returned.
    """

    overflow = False
    try:
        f = open(filename, 'rb')
        f.seek(0, 2)
        sz = f.tell()

        if sz > (offset + length):
            overflow = True
            offset = sz - 1

        if (offset + length) > sz:
            if offset > (sz - 1):
                length = 0
            offset = sz - length

        if offset < 0:
            offset = 0
        if length < 0:
            length = 0

        if length == 0:
            data = ''
        else:
            f.seek(offset)
            data = f.read(length)

        offset = sz
        return [data, offset, overflow]

    except (OSError, IOError):
        return ['', offset, False]

# copied from supervisor.options
def split_namespec(namespec):
    names = namespec.split(':', 1)
    if len(names) == 2:
        # group and and process name differ
        group_name, process_name = names
        if not process_name or process_name == '*':
            process_name = None
    else:
        # group name is same as process name
        group_name, process_name = namespec, namespec
    return group_name, process_name

def main(args=None):
    import argparse

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--address', '-a', default='0.0.0.0', help='the '
                        'address on which the server is listening (e.g., '
                        '"127.0.0.1", default "0.0.0.0").')
    parser.add_argument('--port', '-p', type=int, default=9002, help='the port'
                        ' on which the server is listening (default "9002").')
    args = parser.parse_args(args)

    logging.basicConfig(format="%(asctime)s %(levelname)s "
                               "%(filename)s:%(lineno)d %(message)s")

    LOG.setLevel(logging.INFO)

    LOG.info("psutil version %s at %s", psutil.__version__, psutil.__file__)

    server = SimpleXMLRPCServer((args.address, args.port))
    LOG.info("serving on %s:%d", args.address, args.port)
    server.register_introspection_functions()
    server.register_instance(MonHelperRPCInterface())
    server.serve_forever()

if __name__ == '__main__':
    main()
<|endoftext|>