<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import json
import logging
import os
import re
import shutil
import tempfile

from telemetry import page as page_module
from telemetry.util import cloud_storage

def AssertValidCloudStorageBucket(bucket):
  is_valid = bucket in (None,
                        cloud_storage.PUBLIC_BUCKET,
                        cloud_storage.PARTNER_BUCKET,
                        cloud_storage.INTERNAL_BUCKET)
  if not is_valid:
    raise ValueError("Cloud storage privacy bucket %s is invalid" % bucket)

class ArchiveError(Exception):
  pass

class WprArchiveInfo(object):
  def __init__(self, file_path, data, bucket):
    AssertValidCloudStorageBucket(bucket)
    self._file_path = file_path
    self._base_dir = os.path.dirname(file_path)
    self._data = data
    self._bucket = bucket

    # Ensure directory exists.
    if not os.path.exists(self._base_dir):
      os.makedirs(self._base_dir)

    # Map from the relative path (as it appears in the metadata file) of the
    # .wpr file to a list of user story names it supports.
    self._wpr_file_to_user_story_names = data['archives']

    # Map from the user_story name to a relative path (as it appears
    # in the metadata file) of the .wpr file.
    self._user_story_name_to_wpr_file = dict()
    # Find out the wpr file names for each user_story.
    for wpr_file in data['archives']:
      user_story_names = data['archives'][wpr_file]
      for user_story_name in user_story_names:
        self._user_story_name_to_wpr_file[user_story_name] = wpr_file
    self.temp_target_wpr_file_path = None

  @classmethod
  def FromFile(cls, file_path, bucket):
    if os.path.exists(file_path):
      with open(file_path, 'r') as f:
        data = json.load(f)
        return cls(file_path, data, bucket)
    return cls(file_path, {'archives': {}}, bucket)

  def DownloadArchivesIfNeeded(self):
    """Downloads archives iff the Archive has a bucket parameter and the user
    has permission to access the bucket.

    Raises cloud storage Permissions or Credentials error when there is no
    local copy of the archive and the user doesn't have permission to access
    the archive's bucket.

    Warns when a bucket is not specified or when the user doesn't have
    permission to access the archive's bucket but a local copy of the archive
    exists.
    """
    # Download all .wpr files.
    if not self._bucket:
      logging.warning('User story set in %s has no bucket specified, and '
                      'cannot be downloaded from cloud_storage.', )
    for archive_path in self._data['archives']:
      archive_path = self._WprFileNameToPath(archive_path)
      try:
        cloud_storage.GetIfChanged(archive_path, self._bucket)
      except (cloud_storage.CredentialsError, cloud_storage.PermissionError):
        if os.path.exists(archive_path):
          # If the archive exists, assume the user recorded their own and
          # simply warn.
          logging.warning('Need credentials to update WPR archive: %s',
                          archive_path)
        else:
          logging.error("You either aren't authenticated or don't have "
                        "permission to use the archives for this page set."
                        "\nYou may need to run gsutil config.")
          raise

  def WprFilePathForUserStory(self, story):
    if self.temp_target_wpr_file_path:
      return self.temp_target_wpr_file_path
    wpr_file = self._user_story_name_to_wpr_file.get(story.display_name, None)
    if wpr_file is None and isinstance(story, page_module.Page):
      # Some old pages always use the URL to identify a page rather than the
      # display_name, so try to look for that.
      wpr_file = self._user_story_name_to_wpr_file.get(story.url, None)
    if wpr_file:
      return self._WprFileNameToPath(wpr_file)
    return None

  def AddNewTemporaryRecording(self, temp_wpr_file_path=None):
    if temp_wpr_file_path is None:
      temp_wpr_file_handle, temp_wpr_file_path = tempfile.mkstemp()
      os.close(temp_wpr_file_handle)
    self.temp_target_wpr_file_path = temp_wpr_file_path

  def AddRecordedUserStories(self, user_stories, upload_to_cloud_storage=False):
    if not user_stories:
      os.remove(self.temp_target_wpr_file_path)
      return

    (target_wpr_file, target_wpr_file_path) = self._NextWprFileName()
    for user_story in user_stories:
      self._SetWprFileForUserStory(user_story.display_name, target_wpr_file)
    shutil.move(self.temp_target_wpr_file_path, target_wpr_file_path)

    # Update the hash file.
    with open(target_wpr_file_path + '.sha1', 'wb') as f:
      f.write(cloud_storage.CalculateHash(target_wpr_file_path))
      f.flush()

    self._WriteToFile()
    self._DeleteAbandonedWprFiles()

    # Upload to cloud storage
    if upload_to_cloud_storage:
      if not self._bucket:
        logging.warning('UserStorySet must have bucket specified to upload '
                        'user stories to cloud storage.')
        return
      try:
        cloud_storage.Insert(self._bucket, target_wpr_file,
                             target_wpr_file_path)
      except cloud_storage.CloudStorageError, e:
        logging.warning('Failed to upload wpr file %s to cloud storage. '
                        'Error:%s' % target_wpr_file_path, e)

  def _DeleteAbandonedWprFiles(self):
    # Update the metadata so that the abandoned wpr files don't have
    # empty user story name arrays.
    abandoned_wpr_files = self._AbandonedWprFiles()
    for wpr_file in abandoned_wpr_files:
      del self._wpr_file_to_user_story_names[wpr_file]
      # Don't fail if we're unable to delete some of the files.
      wpr_file_path = self._WprFileNameToPath(wpr_file)
      try:
        os.remove(wpr_file_path)
      except Exception:
        logging.warning('Failed to delete file: %s' % wpr_file_path)

  def _AbandonedWprFiles(self):
    abandoned_wpr_files = []
    for wpr_file, user_story_names in (
        self._wpr_file_to_user_story_names.iteritems()):
      if not user_story_names:
        abandoned_wpr_files.append(wpr_file)
    return abandoned_wpr_files

  def _WriteToFile(self):
    """Writes the metadata into the file passed as constructor parameter."""
    metadata = dict()
    metadata['description'] = (
        'Describes the Web Page Replay archives for a user story set. '
        'Don\'t edit by hand! Use record_wpr for updating.')
    metadata['archives'] = self._wpr_file_to_user_story_names.copy()
    # Don't write data for abandoned archives.
    abandoned_wpr_files = self._AbandonedWprFiles()
    for wpr_file in abandoned_wpr_files:
      del metadata['archives'][wpr_file]

    with open(self._file_path, 'w') as f:
      json.dump(metadata, f, indent=4)
      f.flush()

  def _WprFileNameToPath(self, wpr_file):
    return os.path.abspath(os.path.join(self._base_dir, wpr_file))

  def _NextWprFileName(self):
    """Creates a new file name for a wpr archive file."""
    # The names are of the format "some_thing_number.wpr". Read the numbers.
    highest_number = -1
    base = None
    for wpr_file in self._wpr_file_to_user_story_names:
      match = re.match(r'(?P<BASE>.*)_(?P<NUMBER>[0-9]+)\.wpr', wpr_file)
      if not match:
        raise Exception('Illegal wpr file name ' + wpr_file)
      highest_number = max(int(match.groupdict()['NUMBER']), highest_number)
      if base and match.groupdict()['BASE'] != base:
        raise Exception('Illegal wpr file name ' + wpr_file +
                        ', doesn\'t begin with ' + base)
      base = match.groupdict()['BASE']
    if not base:
      # If we're creating a completely new info file, use the base name of the
      # user story set file.
      base = os.path.splitext(os.path.basename(self._file_path))[0]
    new_filename = '%s_%03d.wpr' % (base, highest_number + 1)
    return new_filename, self._WprFileNameToPath(new_filename)

  def _SetWprFileForUserStory(self, user_story_name, wpr_file):
    """For modifying the metadata when we're going to record a new archive."""
    old_wpr_file = self._user_story_name_to_wpr_file.get(user_story_name, None)
    if old_wpr_file:
      self._wpr_file_to_user_story_names[old_wpr_file].remove(user_story_name)
    self._user_story_name_to_wpr_file[user_story_name] = wpr_file
    if wpr_file not in self._wpr_file_to_user_story_names:
      self._wpr_file_to_user_story_names[wpr_file] = []
    self._wpr_file_to_user_story_names[wpr_file].append(user_story_name)
<commit_msg>Adding assert for better debugging.<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import json
import logging
import os
import re
import shutil
import tempfile

from telemetry import page as page_module
from telemetry.util import cloud_storage

def AssertValidCloudStorageBucket(bucket):
  is_valid = bucket in (None,
                        cloud_storage.PUBLIC_BUCKET,
                        cloud_storage.PARTNER_BUCKET,
                        cloud_storage.INTERNAL_BUCKET)
  if not is_valid:
    raise ValueError("Cloud storage privacy bucket %s is invalid" % bucket)

class ArchiveError(Exception):
  pass

class WprArchiveInfo(object):
  def __init__(self, file_path, data, bucket):
    AssertValidCloudStorageBucket(bucket)
    self._file_path = file_path
    self._base_dir = os.path.dirname(file_path)
    self._data = data
    self._bucket = bucket

    # Ensure directory exists.
    if not os.path.exists(self._base_dir):
      os.makedirs(self._base_dir)

    # Map from the relative path (as it appears in the metadata file) of the
    # .wpr file to a list of user story names it supports.
    self._wpr_file_to_user_story_names = data['archives']

    # Map from the user_story name to a relative path (as it appears
    # in the metadata file) of the .wpr file.
    self._user_story_name_to_wpr_file = dict()
    # Find out the wpr file names for each user_story.
    for wpr_file in data['archives']:
      user_story_names = data['archives'][wpr_file]
      for user_story_name in user_story_names:
        self._user_story_name_to_wpr_file[user_story_name] = wpr_file
    self.temp_target_wpr_file_path = None

  @classmethod
  def FromFile(cls, file_path, bucket):
    if os.path.exists(file_path):
      with open(file_path, 'r') as f:
        data = json.load(f)
        return cls(file_path, data, bucket)
    return cls(file_path, {'archives': {}}, bucket)

  def DownloadArchivesIfNeeded(self):
    """Downloads archives iff the Archive has a bucket parameter and the user
    has permission to access the bucket.

    Raises cloud storage Permissions or Credentials error when there is no
    local copy of the archive and the user doesn't have permission to access
    the archive's bucket.

    Warns when a bucket is not specified or when the user doesn't have
    permission to access the archive's bucket but a local copy of the archive
    exists.
    """
    # Download all .wpr files.
    if not self._bucket:
      logging.warning('User story set in %s has no bucket specified, and '
                      'cannot be downloaded from cloud_storage.', )
      return
    assert 'archives' in self._data, 'Invalid data format in %s. \'archives\'' \
                                     ' field is needed' % self._file_path
    for archive_path in self._data['archives']:
      archive_path = self._WprFileNameToPath(archive_path)
      try:
        cloud_storage.GetIfChanged(archive_path, self._bucket)
      except (cloud_storage.CredentialsError, cloud_storage.PermissionError):
        if os.path.exists(archive_path):
          # If the archive exists, assume the user recorded their own and
          # simply warn.
          logging.warning('Need credentials to update WPR archive: %s',
                          archive_path)
        else:
          logging.error("You either aren't authenticated or don't have "
                        "permission to use the archives for this page set."
                        "\nYou may need to run gsutil config.")
          raise

  def WprFilePathForUserStory(self, story):
    if self.temp_target_wpr_file_path:
      return self.temp_target_wpr_file_path
    wpr_file = self._user_story_name_to_wpr_file.get(story.display_name, None)
    if wpr_file is None and isinstance(story, page_module.Page):
      # Some old pages always use the URL to identify a page rather than the
      # display_name, so try to look for that.
      wpr_file = self._user_story_name_to_wpr_file.get(story.url, None)
    if wpr_file:
      return self._WprFileNameToPath(wpr_file)
    return None

  def AddNewTemporaryRecording(self, temp_wpr_file_path=None):
    if temp_wpr_file_path is None:
      temp_wpr_file_handle, temp_wpr_file_path = tempfile.mkstemp()
      os.close(temp_wpr_file_handle)
    self.temp_target_wpr_file_path = temp_wpr_file_path

  def AddRecordedUserStories(self, user_stories, upload_to_cloud_storage=False):
    if not user_stories:
      os.remove(self.temp_target_wpr_file_path)
      return

    (target_wpr_file, target_wpr_file_path) = self._NextWprFileName()
    for user_story in user_stories:
      self._SetWprFileForUserStory(user_story.display_name, target_wpr_file)
    shutil.move(self.temp_target_wpr_file_path, target_wpr_file_path)

    # Update the hash file.
    with open(target_wpr_file_path + '.sha1', 'wb') as f:
      f.write(cloud_storage.CalculateHash(target_wpr_file_path))
      f.flush()

    self._WriteToFile()
    self._DeleteAbandonedWprFiles()

    # Upload to cloud storage
    if upload_to_cloud_storage:
      if not self._bucket:
        logging.warning('UserStorySet must have bucket specified to upload '
                        'user stories to cloud storage.')
        return
      try:
        cloud_storage.Insert(self._bucket, target_wpr_file,
                             target_wpr_file_path)
      except cloud_storage.CloudStorageError, e:
        logging.warning('Failed to upload wpr file %s to cloud storage. '
                        'Error:%s' % target_wpr_file_path, e)

  def _DeleteAbandonedWprFiles(self):
    # Update the metadata so that the abandoned wpr files don't have
    # empty user story name arrays.
    abandoned_wpr_files = self._AbandonedWprFiles()
    for wpr_file in abandoned_wpr_files:
      del self._wpr_file_to_user_story_names[wpr_file]
      # Don't fail if we're unable to delete some of the files.
      wpr_file_path = self._WprFileNameToPath(wpr_file)
      try:
        os.remove(wpr_file_path)
      except Exception:
        logging.warning('Failed to delete file: %s' % wpr_file_path)

  def _AbandonedWprFiles(self):
    abandoned_wpr_files = []
    for wpr_file, user_story_names in (
        self._wpr_file_to_user_story_names.iteritems()):
      if not user_story_names:
        abandoned_wpr_files.append(wpr_file)
    return abandoned_wpr_files

  def _WriteToFile(self):
    """Writes the metadata into the file passed as constructor parameter."""
    metadata = dict()
    metadata['description'] = (
        'Describes the Web Page Replay archives for a user story set. '
        'Don\'t edit by hand! Use record_wpr for updating.')
    metadata['archives'] = self._wpr_file_to_user_story_names.copy()
    # Don't write data for abandoned archives.
    abandoned_wpr_files = self._AbandonedWprFiles()
    for wpr_file in abandoned_wpr_files:
      del metadata['archives'][wpr_file]

    with open(self._file_path, 'w') as f:
      json.dump(metadata, f, indent=4)
      f.flush()

  def _WprFileNameToPath(self, wpr_file):
    return os.path.abspath(os.path.join(self._base_dir, wpr_file))

  def _NextWprFileName(self):
    """Creates a new file name for a wpr archive file."""
    # The names are of the format "some_thing_number.wpr". Read the numbers.
    highest_number = -1
    base = None
    for wpr_file in self._wpr_file_to_user_story_names:
      match = re.match(r'(?P<BASE>.*)_(?P<NUMBER>[0-9]+)\.wpr', wpr_file)
      if not match:
        raise Exception('Illegal wpr file name ' + wpr_file)
      highest_number = max(int(match.groupdict()['NUMBER']), highest_number)
      if base and match.groupdict()['BASE'] != base:
        raise Exception('Illegal wpr file name ' + wpr_file +
                        ', doesn\'t begin with ' + base)
      base = match.groupdict()['BASE']
    if not base:
      # If we're creating a completely new info file, use the base name of the
      # user story set file.
      base = os.path.splitext(os.path.basename(self._file_path))[0]
    new_filename = '%s_%03d.wpr' % (base, highest_number + 1)
    return new_filename, self._WprFileNameToPath(new_filename)

  def _SetWprFileForUserStory(self, user_story_name, wpr_file):
    """For modifying the metadata when we're going to record a new archive."""
    old_wpr_file = self._user_story_name_to_wpr_file.get(user_story_name, None)
    if old_wpr_file:
      self._wpr_file_to_user_story_names[old_wpr_file].remove(user_story_name)
    self._user_story_name_to_wpr_file[user_story_name] = wpr_file
    if wpr_file not in self._wpr_file_to_user_story_names:
      self._wpr_file_to_user_story_names[wpr_file] = []
    self._wpr_file_to_user_story_names[wpr_file].append(user_story_name)
<|endoftext|>


<commit_before>import logging
import os
import cStringIO
from django.conf import settings
from ho import pisa
from lino.core.web import extend_context
from lino.utils.appy_pod import PrintTableAction
from lino.core.tables import AbstractTable
from lino.utils.media import TmpMediaFile
from lino.utils.xmlgen import html as xghtml
from django.utils.translation import ugettext_lazy as _

class PrintTableActionPisa(PrintTableAction):
    template_name = "table.pisa.html"

    def run_from_ui(self, ar, **kw):
        # Prepare tmp file
        mf = TmpMediaFile(ar, self.target_file_format)
        settings.SITE.makedirs_if_missing(os.path.dirname(mf.name))

        # Render
        self.render(ar, mf.name)

        ar.response.update(success=True)
        ar.response.update(open_url=mf.url)

    def as_html_table(self, ar):
        t = xghtml.Table()
        ar.dump2html(t, ar.data_iterator)

        return xghtml.E.tostring(t.as_element())

    def render(self, ar, output_file):
        context = {
            'action': self,
            'ar': ar,
            'as_html_table': self.as_html_table,
        }

        extend_context(context)

        template = settings.SITE.jinja_env.get_template(self.template_name)
        html = template.render(**context)

        with open(output_file + '.html', "w") as file:
            file.write(html)

        result = cStringIO.StringIO()
        h = logging.FileHandler(output_file + '.log', 'w')
        pisa.log.addHandler(h)
        pdf = pisa.pisaDocument(
            cStringIO.StringIO(html), result, encoding='utf-8')
        pisa.log.removeHandler(h)
        h.close()

        with open(output_file, 'wb') as file:
            file.write(result.getvalue())
            file.close()

        if pdf.err:
            raise Exception("pisa.pisaDocument.err is %r" % pdf.err)

class PortraitPrintTableActionPisa(PrintTableActionPisa):
    label = _("Table (portrait)")
    sort_index = -9
    template_name = "table-portrait.pisa.html"

AbstractTable.as_pdf = PrintTableActionPisa()
AbstractTable.as_pdf_p = PortraitPrintTableActionPisa()
<commit_msg>Fix exception when printing unicode characters<commit_after>import logging
import os
import cStringIO
from django.conf import settings
from ho import pisa
from lino.core.web import extend_context
from lino.utils.appy_pod import PrintTableAction
from lino.core.tables import AbstractTable
from lino.utils.media import TmpMediaFile
from lino.utils.xmlgen import html as xghtml
from django.utils.translation import ugettext_lazy as _

class PrintTableActionPisa(PrintTableAction):
    template_name = "table.pisa.html"

    def run_from_ui(self, ar, **kw):
        # Prepare tmp file
        mf = TmpMediaFile(ar, self.target_file_format)
        settings.SITE.makedirs_if_missing(os.path.dirname(mf.name))

        # Render
        self.render(ar, mf.name)

        ar.response.update(success=True)
        ar.response.update(open_url=mf.url)

    def as_html_table(self, ar):
        t = xghtml.Table()
        ar.dump2html(t, ar.data_iterator)

        return xghtml.E.tostring(t.as_element())

    def render(self, ar, output_file):
        context = {
            'action': self,
            'ar': ar,
            'as_html_table': self.as_html_table,
        }

        extend_context(context)

        template = settings.SITE.jinja_env.get_template(self.template_name)
        html = template.render(**context).encode('utf-8')

        with open(output_file + '.html', "w") as file:
            file.write(html)

        result = cStringIO.StringIO()
        h = logging.FileHandler(output_file + '.log', 'w')
        pisa.log.addHandler(h)
        pdf = pisa.pisaDocument(
            cStringIO.StringIO(html), result, encoding='utf-8')
        pisa.log.removeHandler(h)
        h.close()

        with open(output_file, 'wb') as file:
            file.write(result.getvalue())
            file.close()

        if pdf.err:
            raise Exception("pisa.pisaDocument.err is %r" % pdf.err)

class PortraitPrintTableActionPisa(PrintTableActionPisa):
    label = _("Table (portrait)")
    sort_index = -9
    template_name = "table-portrait.pisa.html"

AbstractTable.as_pdf = PrintTableActionPisa()
AbstractTable.as_pdf_p = PortraitPrintTableActionPisa()
<|endoftext|>


<commit_before># Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright 2012 Red Hat, Inc.
# Copyright 2013 NTT corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Command-line flag library.

Emulates gflags by wrapping cfg.ConfigOpts.

The idea is to move fully to cfg eventually, and this wrapper is a
stepping stone.

"""

import socket

from oslo_config import cfg
from oslo_log import log as logging
from oslo_middleware import cors
from oslo_policy import opts as policy_opts
from oslo_utils import netutils

from cinder.policy import DEFAULT_POLICY_FILENAME

CONF = cfg.CONF
logging.register_options(CONF)

core_opts = [
    cfg.StrOpt('state_path',
               default='/var/lib/cinder',
               help="Top-level directory for maintaining cinder's state"), ]

CONF.register_cli_opts(core_opts)

api_opts = [
    cfg.BoolOpt('api_rate_limit',
                default=True,
                help='Enables or disables rate limit of the API.'),
    cfg.StrOpt('group_api_class',
               default='cinder.group.api.API',
               help='The full class name of the group API class'),
    cfg.ListOpt('osapi_volume_ext_list',
                default=[],
                help='Specify list of extensions to load when using osapi_'
                     'volume_extension option with cinder.api.contrib.'
                     'select_extensions'),
    cfg.MultiStrOpt('osapi_volume_extension',
                    default=['cinder.api.contrib.standard_extensions'],
                    help='osapi volume extension to load'),
    cfg.StrOpt('volume_api_class',
               default='cinder.volume.api.API',
               help='The full class name of the volume API class to use'),
]

global_opts = [
    cfg.HostAddressOpt('my_ip',
                       sample_default='<HOST_IP_ADDRESS>',
                       default=netutils.get_my_ipv4(),
                       help='IP address of this host'),
    cfg.StrOpt('volume_manager',
               default='cinder.volume.manager.VolumeManager',
               help='Full class name for the Manager for volume'),
    cfg.StrOpt('scheduler_manager',
               default='cinder.scheduler.manager.SchedulerManager',
               help='Full class name for the Manager for scheduler'),
    cfg.StrOpt('host',
               sample_default='localhost',
               default=socket.gethostname(),
               help='Name of this node.  This can be an opaque '
                    'identifier. It is not necessarily a host name, '
                    'FQDN, or IP address.'),
    # NOTE(vish): default to nova for compatibility with nova installs
    cfg.StrOpt('storage_availability_zone',
               default='nova',
               help='Availability zone of this node. Can be overridden per '
                    'volume backend with the option '
                    '"backend_availability_zone".'),
    cfg.StrOpt('default_availability_zone',
               help='Default availability zone for new volumes. If not set, '
                    'the storage_availability_zone option value is used as '
                    'the default for new volumes.'),
    cfg.BoolOpt('allow_availability_zone_fallback',
                default=False,
                help='If the requested Cinder availability zone is '
                     'unavailable, fall back to the value of '
                     'default_availability_zone, then '
                     'storage_availability_zone, instead of failing.'),
    cfg.StrOpt('default_volume_type',
               default='__DEFAULT__',
               required=True,
               help='Default volume type to use'),
    cfg.StrOpt('default_group_type',
               help='Default group type to use'),
    cfg.StrOpt('volume_usage_audit_period',
               default='month',
               help='Time period for which to generate volume usages. '
                    'The options are hour, day, month, or year.'),
    cfg.StrOpt('rootwrap_config',
               default='/etc/cinder/rootwrap.conf',
               help='Path to the rootwrap configuration file to use for '
                    'running commands as root'),
    cfg.BoolOpt('monkey_patch',
                default=False,
                help='Enable monkey patching'),
    cfg.ListOpt('monkey_patch_modules',
                default=[],
                help='List of modules/decorators to monkey patch'),
    cfg.IntOpt('service_down_time',
               default=60,
               help='Maximum time since last check-in for a service to be '
                    'considered up'),
    cfg.ListOpt('enabled_backends',
                help='A list of backend names to use. These backend names '
                     'should be backed by a unique [CONFIG] group '
                     'with its options'),
    cfg.BoolOpt('no_snapshot_gb_quota',
                default=False,
                help='Whether snapshots count against gigabyte quota'),
    cfg.StrOpt('transfer_api_class',
               default='cinder.transfer.api.API',
               help='The full class name of the volume transfer API class'),
    cfg.StrOpt('consistencygroup_api_class',
               default='cinder.consistencygroup.api.API',
               help='The full class name of the consistencygroup API class'),
    cfg.BoolOpt('split_loggers',
                default=False,
                help='Log requests to multiple loggers.')
]

auth_opts = [
    cfg.StrOpt('auth_strategy',
               default='keystone',
               choices=[('noauth', 'Do not perform authentication'),
                        ('keystone', 'Authenticate using keystone')],
               help='The strategy to use for auth. Supports noauth or '
                    'keystone.'),
]

backup_opts = [
    cfg.StrOpt('backup_api_class',
               default='cinder.backup.api.API',
               help='The full class name of the volume backup API class'),
    cfg.StrOpt('backup_manager',
               default='cinder.backup.manager.BackupManager',
               help='Full class name for the Manager for volume backup'),
]

image_opts = [
    cfg.ListOpt('glance_api_servers',
                default=None,
                help='A list of the URLs of glance API servers available to '
                     'cinder ([http[s]://][hostname|ip]:port). If protocol '
                     'is not specified it defaults to http.'),
    cfg.IntOpt('glance_num_retries',
               min=0,
               default=3,
               help='Number retries when downloading an image from glance'),
    cfg.BoolOpt('glance_api_insecure',
                default=False,
                help='Allow to perform insecure SSL (https) requests to '
                     'glance (https will be used but cert validation will '
                     'not be performed).'),
    cfg.BoolOpt('glance_api_ssl_compression',
                default=False,
                help='Enables or disables negotiation of SSL layer '
                     'compression. In some cases disabling compression '
                     'can improve data throughput, such as when high '
                     'network bandwidth is available and you use '
                     'compressed image formats like qcow2.'),
    cfg.StrOpt('glance_ca_certificates_file',
               help='Location of ca certificates file to use for glance '
                    'client requests.'),
    cfg.StrOpt('glance_certfile',
               help='Location of certificate file to use for glance '
                    'client requests.'),
    cfg.StrOpt('glance_keyfile',
               help='Location of certificate key file to use for glance '
                    'client requests.'),
    cfg.IntOpt('glance_request_timeout',
               help='http/https timeout value for glance operations. If no '
                    'value (None) is supplied here, the glanceclient default '
                    'value is used.'),
]

compression_opts = [
    cfg.StrOpt('compression_format',
               default='gzip',
               choices=[('gzip', 'GNUzip format')],
               help='Image compression format on image upload'),
    cfg.BoolOpt('allow_compression_on_image_upload',
                default=False,
                help='The strategy to use for image compression on upload. '
                     'Default is disallow compression.'),
]

CONF.register_opts(api_opts)
CONF.register_opts(core_opts)
CONF.register_opts(auth_opts)
CONF.register_opts(backup_opts)
CONF.register_opts(image_opts)
CONF.register_opts(global_opts)
CONF.register_opts(compression_opts)

def set_middleware_defaults():
    """Update default configuration options for oslo.middleware."""

    cors.set_defaults(
        allow_headers=['X-Auth-Token',
                       'X-Identity-Status',
                       'X-Roles',
                       'X-Service-Catalog',
                       'X-User-Id',
                       'X-Tenant-Id',
                       'X-OpenStack-Request-ID',
                       'X-Trace-Info',
                       'X-Trace-HMAC',
                       'OpenStack-API-Version'],
        expose_headers=['X-Auth-Token',
                        'X-Subject-Token',
                        'X-Service-Token',
                        'X-OpenStack-Request-ID',
                        'OpenStack-API-Version'],
        allow_methods=['GET',
                       'PUT',
                       'POST',
                       'DELETE',
                       'PATCH',
                       'HEAD']
    )

def set_external_library_defaults():
    """Set default configuration options for external openstack libraries."""
    # This function is required so that our settings will override the defaults
    # set by the libraries when the Cinder config files are generated.  This
    # function is declared as an entry point for oslo.config.opts.defaults in
    # setup.cfg.

    set_middleware_defaults()
    policy_opts.set_defaults(CONF, policy_file=DEFAULT_POLICY_FILENAME)
<commit_msg>Improve no_snapshot_gb_quota description<commit_after># Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright 2012 Red Hat, Inc.
# Copyright 2013 NTT corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Command-line flag library.

Emulates gflags by wrapping cfg.ConfigOpts.

The idea is to move fully to cfg eventually, and this wrapper is a
stepping stone.

"""

import socket

from oslo_config import cfg
from oslo_log import log as logging
from oslo_middleware import cors
from oslo_policy import opts as policy_opts
from oslo_utils import netutils

from cinder.policy import DEFAULT_POLICY_FILENAME

CONF = cfg.CONF
logging.register_options(CONF)

core_opts = [
    cfg.StrOpt('state_path',
               default='/var/lib/cinder',
               help="Top-level directory for maintaining cinder's state"), ]

CONF.register_cli_opts(core_opts)

api_opts = [
    cfg.BoolOpt('api_rate_limit',
                default=True,
                help='Enables or disables rate limit of the API.'),
    cfg.StrOpt('group_api_class',
               default='cinder.group.api.API',
               help='The full class name of the group API class'),
    cfg.ListOpt('osapi_volume_ext_list',
                default=[],
                help='Specify list of extensions to load when using osapi_'
                     'volume_extension option with cinder.api.contrib.'
                     'select_extensions'),
    cfg.MultiStrOpt('osapi_volume_extension',
                    default=['cinder.api.contrib.standard_extensions'],
                    help='osapi volume extension to load'),
    cfg.StrOpt('volume_api_class',
               default='cinder.volume.api.API',
               help='The full class name of the volume API class to use'),
]

global_opts = [
    cfg.HostAddressOpt('my_ip',
                       sample_default='<HOST_IP_ADDRESS>',
                       default=netutils.get_my_ipv4(),
                       help='IP address of this host'),
    cfg.StrOpt('volume_manager',
               default='cinder.volume.manager.VolumeManager',
               help='Full class name for the Manager for volume'),
    cfg.StrOpt('scheduler_manager',
               default='cinder.scheduler.manager.SchedulerManager',
               help='Full class name for the Manager for scheduler'),
    cfg.StrOpt('host',
               sample_default='localhost',
               default=socket.gethostname(),
               help='Name of this node.  This can be an opaque '
                    'identifier. It is not necessarily a host name, '
                    'FQDN, or IP address.'),
    # NOTE(vish): default to nova for compatibility with nova installs
    cfg.StrOpt('storage_availability_zone',
               default='nova',
               help='Availability zone of this node. Can be overridden per '
                    'volume backend with the option '
                    '"backend_availability_zone".'),
    cfg.StrOpt('default_availability_zone',
               help='Default availability zone for new volumes. If not set, '
                    'the storage_availability_zone option value is used as '
                    'the default for new volumes.'),
    cfg.BoolOpt('allow_availability_zone_fallback',
                default=False,
                help='If the requested Cinder availability zone is '
                     'unavailable, fall back to the value of '
                     'default_availability_zone, then '
                     'storage_availability_zone, instead of failing.'),
    cfg.StrOpt('default_volume_type',
               default='__DEFAULT__',
               required=True,
               help='Default volume type to use'),
    cfg.StrOpt('default_group_type',
               help='Default group type to use'),
    cfg.StrOpt('volume_usage_audit_period',
               default='month',
               help='Time period for which to generate volume usages. '
                    'The options are hour, day, month, or year.'),
    cfg.StrOpt('rootwrap_config',
               default='/etc/cinder/rootwrap.conf',
               help='Path to the rootwrap configuration file to use for '
                    'running commands as root'),
    cfg.BoolOpt('monkey_patch',
                default=False,
                help='Enable monkey patching'),
    cfg.ListOpt('monkey_patch_modules',
                default=[],
                help='List of modules/decorators to monkey patch'),
    cfg.IntOpt('service_down_time',
               default=60,
               help='Maximum time since last check-in for a service to be '
                    'considered up'),
    cfg.ListOpt('enabled_backends',
                help='A list of backend names to use. These backend names '
                     'should be backed by a unique [CONFIG] group '
                     'with its options'),
    cfg.BoolOpt('no_snapshot_gb_quota',
                default=False,
                help="Whether snapshots sizes count against global and per "
                     "volume type gigabyte quotas. By default snapshots' "
                     "sizes are counted."),
    cfg.StrOpt('transfer_api_class',
               default='cinder.transfer.api.API',
               help='The full class name of the volume transfer API class'),
    cfg.StrOpt('consistencygroup_api_class',
               default='cinder.consistencygroup.api.API',
               help='The full class name of the consistencygroup API class'),
    cfg.BoolOpt('split_loggers',
                default=False,
                help='Log requests to multiple loggers.')
]

auth_opts = [
    cfg.StrOpt('auth_strategy',
               default='keystone',
               choices=[('noauth', 'Do not perform authentication'),
                        ('keystone', 'Authenticate using keystone')],
               help='The strategy to use for auth. Supports noauth or '
                    'keystone.'),
]

backup_opts = [
    cfg.StrOpt('backup_api_class',
               default='cinder.backup.api.API',
               help='The full class name of the volume backup API class'),
    cfg.StrOpt('backup_manager',
               default='cinder.backup.manager.BackupManager',
               help='Full class name for the Manager for volume backup'),
]

image_opts = [
    cfg.ListOpt('glance_api_servers',
                default=None,
                help='A list of the URLs of glance API servers available to '
                     'cinder ([http[s]://][hostname|ip]:port). If protocol '
                     'is not specified it defaults to http.'),
    cfg.IntOpt('glance_num_retries',
               min=0,
               default=3,
               help='Number retries when downloading an image from glance'),
    cfg.BoolOpt('glance_api_insecure',
                default=False,
                help='Allow to perform insecure SSL (https) requests to '
                     'glance (https will be used but cert validation will '
                     'not be performed).'),
    cfg.BoolOpt('glance_api_ssl_compression',
                default=False,
                help='Enables or disables negotiation of SSL layer '
                     'compression. In some cases disabling compression '
                     'can improve data throughput, such as when high '
                     'network bandwidth is available and you use '
                     'compressed image formats like qcow2.'),
    cfg.StrOpt('glance_ca_certificates_file',
               help='Location of ca certificates file to use for glance '
                    'client requests.'),
    cfg.StrOpt('glance_certfile',
               help='Location of certificate file to use for glance '
                    'client requests.'),
    cfg.StrOpt('glance_keyfile',
               help='Location of certificate key file to use for glance '
                    'client requests.'),
    cfg.IntOpt('glance_request_timeout',
               help='http/https timeout value for glance operations. If no '
                    'value (None) is supplied here, the glanceclient default '
                    'value is used.'),
]

compression_opts = [
    cfg.StrOpt('compression_format',
               default='gzip',
               choices=[('gzip', 'GNUzip format')],
               help='Image compression format on image upload'),
    cfg.BoolOpt('allow_compression_on_image_upload',
                default=False,
                help='The strategy to use for image compression on upload. '
                     'Default is disallow compression.'),
]

CONF.register_opts(api_opts)
CONF.register_opts(core_opts)
CONF.register_opts(auth_opts)
CONF.register_opts(backup_opts)
CONF.register_opts(image_opts)
CONF.register_opts(global_opts)
CONF.register_opts(compression_opts)

def set_middleware_defaults():
    """Update default configuration options for oslo.middleware."""

    cors.set_defaults(
        allow_headers=['X-Auth-Token',
                       'X-Identity-Status',
                       'X-Roles',
                       'X-Service-Catalog',
                       'X-User-Id',
                       'X-Tenant-Id',
                       'X-OpenStack-Request-ID',
                       'X-Trace-Info',
                       'X-Trace-HMAC',
                       'OpenStack-API-Version'],
        expose_headers=['X-Auth-Token',
                        'X-Subject-Token',
                        'X-Service-Token',
                        'X-OpenStack-Request-ID',
                        'OpenStack-API-Version'],
        allow_methods=['GET',
                       'PUT',
                       'POST',
                       'DELETE',
                       'PATCH',
                       'HEAD']
    )

def set_external_library_defaults():
    """Set default configuration options for external openstack libraries."""
    # This function is required so that our settings will override the defaults
    # set by the libraries when the Cinder config files are generated.  This
    # function is declared as an entry point for oslo.config.opts.defaults in
    # setup.cfg.

    set_middleware_defaults()
    policy_opts.set_defaults(CONF, policy_file=DEFAULT_POLICY_FILENAME)
<|endoftext|>


<commit_before>import logging
import json
import requests
import datetime
from requests.exceptions import ConnectionError
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from ckan.plugins import toolkit
from pprint import pformat

PUBLIC_ORGANIZATION_CLASSES = ['GOV', 'MUN', 'ORG']
COMPANY_CLASSES = ['COM']

DEFAULT_TIMEOUT = 3  # seconds

# Add default timeout
class TimeoutHTTPAdapter(HTTPAdapter):
    def __init__(self, *args, **kwargs):
        self.timeout = DEFAULT_TIMEOUT
        if "timeout" in kwargs:
            self.timeout = kwargs["timeout"]
            del kwargs["timeout"]
        super(TimeoutHTTPAdapter, self).__init__(*args, **kwargs)

retry_strategy = Retry(
    total=3,
    backoff_factor=1
)

adapter = TimeoutHTTPAdapter(max_retries=retry_strategy)
http = requests.Session()
http.mount("http://", adapter)

log = logging.getLogger(__name__)

def update_xroad_organizations(context, data_dict):
    harvest_source_list = toolkit.get_action('harvest_source_list')
    organization_list = toolkit.get_action('organization_list')
    organization_show = toolkit.get_action('organization_show')
    organization_patch = toolkit.get_action('organization_patch')

    harvest_sources = harvest_source_list(context, {})
    organization_names = organization_list(context, {})
    timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')

    for harvest_source in harvest_sources:
        if harvest_source.get('type') != 'xroad':
            continue

        source_url = harvest_source.get('url')
        source_title = harvest_source.get('title')

        for organization_name in organization_names:
            organization = organization_show(context, {'id': organization_name})

            if not organization.get('xroad_membercode'):
                continue

            last_updated = organization.get('metadata_updated_from_xroad_timestamp') or '2011-01-01T00:00:00'
            patch = _prepare_xroad_organization_patch(organization, source_url, last_updated)
            if patch is not None:
                log.debug('Updating organization %s data from %s', organization_name, source_title)
                patch['metadata_updated_from_xroad_timestamp'] = timestamp
                try:
                    organization_patch(context, patch)
                except toolkit.ValidationError:
                    log.debug('Validation error updating %s from %s: %s', organization_name, source_title, pformat(patch))

            else:
                log.debug('Nothing to do for %s from %s', organization_name, source_title)

def _prepare_xroad_organization_patch(organization, source_url, last_updated):

    member_class = organization.get('xroad_memberclass')
    member_code = organization.get('xroad_membercode')
    organization_name = organization.get('name')

    if not member_code:
        log.info('Organization %s has no X-Road member code, skipping...', organization_name)
        return None

    organization_dict = {'id': organization['id']}

    if member_class in PUBLIC_ORGANIZATION_CLASSES:
        try:
            organization_changed = not last_updated or _get_organization_changes(source_url, member_code, last_updated)

            if not organization_changed:
                log.debug('No changes to organization %s since last update at %s, skipping...', organization_name, last_updated)
                return None

            org_information_list = _get_organization_information(source_url, member_code)

            organization_info = None

            if not org_information_list:
                return None

            else:
                # If PTV has only one matching organization
                if type(org_information_list) is dict:
                    log.info("Only one matching organization in ptv for organization %s" % organization_name)
                    organization_info = org_information_list
                else:
                    log.info("Multiple organizations found in ptv for organization %s" % organization_name)
                    # Match only if PTV title matches our organization title
                    organization_info = _parse_organization_info(org_information_list, organization_name)

                if not organization_info:
                    return None
                else:
                    log.info("Parsing organization information for %s" % organization_name)

                    if organization_info.get('organizationNames', {}):
                        org_names = _convert_xroad_value_to_uniform_list(
                                organization_info.get('organizationNames', {}).get('organizationName', {}))

                        org_names_translated = {
                                "fi": next((name.get('value', '') for name in org_names if (name.get('language') == 'fi') and name.get('type') == "Name"), ""),
                                "sv": next((name.get('value', '') for name in org_names if (name.get('language') == 'sv') and name.get('type') == "Name"), ""),
                                "en": next((name.get('value', '') for name in org_names if (name.get('language') == 'en') and name.get('type') == "Name"), ""),
                                }

                        organization_dict['title_translated'] = org_names_translated

                    if organization_info.get('organizationDescriptions', {}):
                        org_descriptions = _convert_xroad_value_to_uniform_list(
                                organization_info.get('organizationDescriptions', {}).get('organizationDescription', {}))

                        org_descriptions_translated = {
                                "fi": next((description.get('value', '') for description in org_descriptions if description.get('language') == 'fi'), ""),
                                "sv": next((description.get('value', '') for description in org_descriptions if description.get('language') == 'sv'), ""),
                                "en": next((description.get('value', '') for description in org_descriptions if description.get('language') == 'en'), "")
                                }

                        organization_dict['description_translated'] = org_descriptions_translated

                    if organization_info.get('webPages', {}):
                        webpages = _convert_xroad_value_to_uniform_list(
                                organization_info.get('webPages', {}).get('webPage', {}))

                        webpage_addresses = {
                                "fi": next((webpage.get('url', '') for webpage in webpages if webpage.get('language') == 'fi'), ""),
                                "sv": next((webpage.get('url', '') for webpage in webpages if webpage.get('language') == 'sv'), ""),
                                "en": next((webpage.get('url', '') for webpage in webpages if webpage.get('language') == 'en'), "")
                                }

                        webpage_descriptions = {
                                "fi": next((webpage.get('value', '') for webpage in webpages if webpage.get('language') == 'fi'), ""),
                                "sv": next((webpage.get('value', '') for webpage in webpages if webpage.get('language') == 'sv'), ""),
                                "en": next((webpage.get('value', '') for webpage in webpages if webpage.get('language') == 'en'), "")
                                }

                        organization_dict['webpage_address'] = webpage_addresses
                        organization_dict['webpage_description'] = webpage_descriptions

                    emails_field = organization_info.get('emails', {})
                    if emails_field:
                        email_data = _convert_xroad_value_to_uniform_list(emails_field.get('email'))
                        if email_data:
                            languages = set(item['language'] for item in email_data)
                            emails = {lang: [item['value']
                                             for item in email_data
                                             if 'value' in item and item.get('language') == lang]
                                      for lang in languages}
                            if emails:
                                organization_dict['email_address'] = emails

                        organization_dict['organization_guid'] = organization_info.get('guid', '')

        except Exception:
            log.warn("Failed to fetch organization information with id %s", member_code)

    elif member_class in COMPANY_CLASSES:
        try:
            company_changed = not last_updated or _get_company_changes(source_url, member_code, last_updated)

            if not company_changed:
                log.debug('No changes to company %s since last update at %s, skipping...', organization_name, last_updated)
                return None

            company = _get_companies_information(source_url, member_code)

            if not company:
                return None

            if type(company) is dict:
                if company.get('companyForms'):
                    company_forms = _convert_xroad_value_to_uniform_list(company.get('companyForms', {}).get('companyForm', {}))
                    forms = {
                            "fi": next((form.get('name') for form in company_forms if form.get('language') == 'FI'), ""),
                            "sv": next((form.get('name') for form in company_forms if form.get('language') == 'SE'), ""),
                            "en": next((form.get('name') for form in company_forms if form.get('language') == 'EN'), "")
                            }

                    organization_dict['company_type'] = forms

                if company.get('businessAddresses'):
                    business_addresses = _convert_xroad_value_to_uniform_list(company.get('businessAddresses', {}).get('businessAddress', {}))

                    business_address = business_addresses[0] if business_addresses else None

                    # TODO: language should be country
                    organization_dict['postal_address'] = business_address.get('street') + ', ' + str(business_address.get('postCode'))\
                            + ', ' + business_address.get('city') + ', ' + business_address.get('language')

                if company.get('languages'):
                    languages = _convert_xroad_value_to_uniform_list(company.get('languages', {}).get('language', {}))
                    company_languages = {
                            "fi": next((language.get('name', '') for language in languages if language.get('language') == 'FI'), ""),
                            "sv": next((language.get('name', '') for language in languages if language.get('language') == 'SE'), ""),
                            "en": next((language.get('name', '') for language in languages if language.get('language') == 'EN'), "")
                            }

                    organization_dict['company_language'] = company_languages

                # Convert "2001-06-11T00:00:00.000+03:00" to "2001-06-11T00:00:00"
                organization_dict['company_registration_date'] = company.get('registrationDate').split(".")[0]

                if company.get('businessIdChanges'):
                    business_id_changes = _convert_xroad_value_to_uniform_list(
                            company.get('businessIdChanges', {}).get('businessIdChange', {}))

                    old_business_ids = [str(business_id_change.get('oldBusinessId')) for business_id_change in business_id_changes]
                    organization_dict['old_business_ids'] = json.dumps(old_business_ids)

        except Exception:
            log.warn("Failed to fetch company information with id %s", member_code)

    else:
        log.debug('Skipping %s because of class %s', organization_name, member_class)
        return None

    return organization_dict

def _get_organization_information(url, business_code):
    try:
        r = http.get(url + '/Consumer/GetOrganizations', params = {'businessCode': business_code},
                headers = {'Accept': 'application/json'})

        response_json = r.json()
        if response_json.get("error"):
            log.info(response_json.get("error").get("string"))
            return []

        if response_json.get('organizationList', {}).get('organization') is dict:
            return [response_json['organizationList']['organization']]

        return response_json.get('organizationList', {}).get('organization')
    except ConnectionError:
        log.error("Calling XRoad service GetOrganizations failed")
        return None

def _parse_organization_info(data, organization_name):
    organization_info = next((org_info for org_info in data for name in
        _convert_xroad_value_to_uniform_list(org_info.get('organizationNames', {})
            .get('organizationName'))
        if name.get('value', {}) == organization_name), None)

    return  organization_info

def _get_companies_information(url, business_id):
    try:
        r = http.get(url + '/Consumer/GetCompanies', params = {'businessId': business_id},
                headers = {'Accept': 'application/json'})

        response_json = r.json()
        if response_json.get("error"):
            log.info(response_json.get("error").get("string"))
            return ""

        return response_json.get('companyList', {}).get('company')
    except ConnectionError:
        log.error("Calling XRoad service GetCompanies failed")
        return None

def _get_organization_changes(url, guid, changed_after):
    try:
        r = http.get(url + '/Consumer/HasOrganizationChanged', params = {'guid': guid, 'changedAfter': changed_after},
                headers = {'Accept': 'application/json'})

        response_json = r.json()
        if response_json.get("error"):
            log.info(response_json.get("error").get("string"))
            return ""

        return r.json()
    except ConnectionError:
        log.error("Calling XRoad service HasOrganizationChanged failed")
        return None

def _get_company_changes(url, business_id, changed_after ):
    try:
        r = http.get(url + '/Consumer/HasCompanyChanged', params = {'businessId': business_id, 'changedAfter': changed_after},
                headers = {'Accept': 'application/json'})

        response_json = r.json()
        if response_json.get("error"):
            log.info(response_json.get("error").get("string"))
            return ""

        return r.json()
    except ConnectionError:
        log.error("Calling XRoad service HasCompanyChanged failed")
        return None

def _convert_xroad_value_to_uniform_list(value):
    if isinstance(value, basestring):
        return []

    if type(value) is dict:
        return [value]

    return value
<commit_msg>LIKA-215: Check if organization info has been fetched at least once<commit_after>import logging
import json
import requests
import datetime
from requests.exceptions import ConnectionError
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from ckan.plugins import toolkit
from pprint import pformat

PUBLIC_ORGANIZATION_CLASSES = ['GOV', 'MUN', 'ORG']
COMPANY_CLASSES = ['COM']

DEFAULT_TIMEOUT = 3  # seconds

# Add default timeout
class TimeoutHTTPAdapter(HTTPAdapter):
    def __init__(self, *args, **kwargs):
        self.timeout = DEFAULT_TIMEOUT
        if "timeout" in kwargs:
            self.timeout = kwargs["timeout"]
            del kwargs["timeout"]
        super(TimeoutHTTPAdapter, self).__init__(*args, **kwargs)

retry_strategy = Retry(
    total=3,
    backoff_factor=1
)

adapter = TimeoutHTTPAdapter(max_retries=retry_strategy)
http = requests.Session()
http.mount("http://", adapter)

log = logging.getLogger(__name__)

def update_xroad_organizations(context, data_dict):
    harvest_source_list = toolkit.get_action('harvest_source_list')
    organization_list = toolkit.get_action('organization_list')
    organization_show = toolkit.get_action('organization_show')
    organization_patch = toolkit.get_action('organization_patch')

    harvest_sources = harvest_source_list(context, {})
    organization_names = organization_list(context, {})
    timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')

    for harvest_source in harvest_sources:
        if harvest_source.get('type') != 'xroad':
            continue

        source_url = harvest_source.get('url')
        source_title = harvest_source.get('title')

        for organization_name in organization_names:
            organization = organization_show(context, {'id': organization_name})

            if not organization.get('xroad_membercode'):
                continue

            last_updated = organization.get('metadata_updated_from_xroad_timestamp') or '2011-01-01T00:00:00'
            patch = _prepare_xroad_organization_patch(organization, source_url, last_updated)
            if patch is not None:
                log.debug('Updating organization %s data from %s', organization_name, source_title)
                patch['metadata_updated_from_xroad_timestamp'] = timestamp
                try:
                    organization_patch(context, patch)
                except toolkit.ValidationError:
                    log.debug('Validation error updating %s from %s: %s', organization_name, source_title, pformat(patch))

            else:
                log.debug('Nothing to do for %s from %s', organization_name, source_title)

def _prepare_xroad_organization_patch(organization, source_url, last_updated):

    member_class = organization.get('xroad_memberclass')
    member_code = organization.get('xroad_membercode')
    organization_name = organization.get('name')

    if not member_code:
        log.info('Organization %s has no X-Road member code, skipping...', organization_name)
        return None

    organization_dict = {'id': organization['id']}

    if member_class in PUBLIC_ORGANIZATION_CLASSES:
        try:
            if 'organization_guid' in organization:
                organization_changed = not last_updated or _get_organization_changes(source_url, organization.get('organization_guid'), last_updated)

                if not organization_changed:
                    log.info('No changes to organization %s since last update at %s, skipping...', organization_name, last_updated)
                    return None

            org_information_list = _get_organization_information(source_url, member_code)

            organization_info = None

            if not org_information_list:
                return None

            else:
                # If PTV has only one matching organization
                if type(org_information_list) is dict:
                    log.info("Only one matching organization in ptv for organization %s" % organization_name)
                    organization_info = org_information_list
                else:
                    log.info("Multiple organizations found in ptv for organization %s" % organization_name)
                    # Match only if PTV title matches our organization title
                    organization_info = _parse_organization_info(org_information_list, organization_name)

                if not organization_info:
                    return None
                else:
                    log.info("Parsing organization information for %s" % organization_name)

                    if organization_info.get('organizationNames', {}):
                        org_names = _convert_xroad_value_to_uniform_list(
                                organization_info.get('organizationNames', {}).get('organizationName', {}))

                        org_names_translated = {
                                "fi": next((name.get('value', '') for name in org_names if (name.get('language') == 'fi') and name.get('type') == "Name"), ""),
                                "sv": next((name.get('value', '') for name in org_names if (name.get('language') == 'sv') and name.get('type') == "Name"), ""),
                                "en": next((name.get('value', '') for name in org_names if (name.get('language') == 'en') and name.get('type') == "Name"), ""),
                                }

                        organization_dict['title_translated'] = org_names_translated

                    if organization_info.get('organizationDescriptions', {}):
                        org_descriptions = _convert_xroad_value_to_uniform_list(
                                organization_info.get('organizationDescriptions', {}).get('organizationDescription', {}))

                        org_descriptions_translated = {
                                "fi": next((description.get('value', '') for description in org_descriptions if description.get('language') == 'fi'), ""),
                                "sv": next((description.get('value', '') for description in org_descriptions if description.get('language') == 'sv'), ""),
                                "en": next((description.get('value', '') for description in org_descriptions if description.get('language') == 'en'), "")
                                }

                        organization_dict['description_translated'] = org_descriptions_translated

                    if organization_info.get('webPages', {}):
                        webpages = _convert_xroad_value_to_uniform_list(
                                organization_info.get('webPages', {}).get('webPage', {}))

                        webpage_addresses = {
                                "fi": next((webpage.get('url', '') for webpage in webpages if webpage.get('language') == 'fi'), ""),
                                "sv": next((webpage.get('url', '') for webpage in webpages if webpage.get('language') == 'sv'), ""),
                                "en": next((webpage.get('url', '') for webpage in webpages if webpage.get('language') == 'en'), "")
                                }

                        webpage_descriptions = {
                                "fi": next((webpage.get('value', '') for webpage in webpages if webpage.get('language') == 'fi'), ""),
                                "sv": next((webpage.get('value', '') for webpage in webpages if webpage.get('language') == 'sv'), ""),
                                "en": next((webpage.get('value', '') for webpage in webpages if webpage.get('language') == 'en'), "")
                                }

                        organization_dict['webpage_address'] = webpage_addresses
                        organization_dict['webpage_description'] = webpage_descriptions

                    emails_field = organization_info.get('emails', {})
                    if emails_field:
                        email_data = _convert_xroad_value_to_uniform_list(emails_field.get('email'))
                        if email_data:
                            languages = set(item['language'] for item in email_data)
                            emails = {lang: [item['value']
                                             for item in email_data
                                             if 'value' in item and item.get('language') == lang]
                                      for lang in languages}
                            if emails:
                                organization_dict['email_address'] = emails

                        organization_dict['organization_guid'] = organization_info.get('guid', '')

        except Exception:
            log.warn("Failed to fetch organization information with id %s", member_code)

    elif member_class in COMPANY_CLASSES:
        try:
            company_changed = not last_updated or _get_company_changes(source_url, member_code, last_updated)

            if not company_changed:
                log.debug('No changes to company %s since last update at %s, skipping...', organization_name, last_updated)
                return None

            company = _get_companies_information(source_url, member_code)

            if not company:
                return None

            if type(company) is dict:
                if company.get('companyForms'):
                    company_forms = _convert_xroad_value_to_uniform_list(company.get('companyForms', {}).get('companyForm', {}))
                    forms = {
                            "fi": next((form.get('name') for form in company_forms if form.get('language') == 'FI'), ""),
                            "sv": next((form.get('name') for form in company_forms if form.get('language') == 'SE'), ""),
                            "en": next((form.get('name') for form in company_forms if form.get('language') == 'EN'), "")
                            }

                    organization_dict['company_type'] = forms

                if company.get('businessAddresses'):
                    business_addresses = _convert_xroad_value_to_uniform_list(company.get('businessAddresses', {}).get('businessAddress', {}))

                    business_address = business_addresses[0] if business_addresses else None

                    # TODO: language should be country
                    organization_dict['postal_address'] = business_address.get('street') + ', ' + str(business_address.get('postCode'))\
                            + ', ' + business_address.get('city') + ', ' + business_address.get('language')

                if company.get('languages'):
                    languages = _convert_xroad_value_to_uniform_list(company.get('languages', {}).get('language', {}))
                    company_languages = {
                            "fi": next((language.get('name', '') for language in languages if language.get('language') == 'FI'), ""),
                            "sv": next((language.get('name', '') for language in languages if language.get('language') == 'SE'), ""),
                            "en": next((language.get('name', '') for language in languages if language.get('language') == 'EN'), "")
                            }

                    organization_dict['company_language'] = company_languages

                # Convert "2001-06-11T00:00:00.000+03:00" to "2001-06-11T00:00:00"
                organization_dict['company_registration_date'] = company.get('registrationDate').split(".")[0]

                if company.get('businessIdChanges'):
                    business_id_changes = _convert_xroad_value_to_uniform_list(
                            company.get('businessIdChanges', {}).get('businessIdChange', {}))

                    old_business_ids = [str(business_id_change.get('oldBusinessId')) for business_id_change in business_id_changes]
                    organization_dict['old_business_ids'] = json.dumps(old_business_ids)

        except Exception:
            log.warn("Failed to fetch company information with id %s", member_code)

    else:
        log.debug('Skipping %s because of class %s', organization_name, member_class)
        return None

    return organization_dict

def _get_organization_information(url, business_code):
    try:
        r = http.get(url + '/Consumer/GetOrganizations', params = {'businessCode': business_code},
                headers = {'Accept': 'application/json'})

        response_json = r.json()
        if response_json.get("error"):
            log.info(response_json.get("error").get("string"))
            return []

        if response_json.get('organizationList', {}).get('organization') is dict:
            return [response_json['organizationList']['organization']]

        return response_json.get('organizationList', {}).get('organization')
    except ConnectionError:
        log.error("Calling XRoad service GetOrganizations failed")
        return None

def _parse_organization_info(data, organization_name):
    organization_info = next((org_info for org_info in data for name in
        _convert_xroad_value_to_uniform_list(org_info.get('organizationNames', {})
            .get('organizationName'))
        if name.get('value', {}) == organization_name), None)

    return  organization_info

def _get_companies_information(url, business_id):
    try:
        r = http.get(url + '/Consumer/GetCompanies', params = {'businessId': business_id},
                headers = {'Accept': 'application/json'})

        response_json = r.json()
        if response_json.get("error"):
            log.info(response_json.get("error").get("string"))
            return ""

        return response_json.get('companyList', {}).get('company')
    except ConnectionError:
        log.error("Calling XRoad service GetCompanies failed")
        return None

def _get_organization_changes(url, guid, changed_after):
    try:
        r = http.get(url + '/Consumer/HasOrganizationChanged', params = {'guid': guid, 'changedAfter': changed_after},
                headers = {'Accept': 'application/json'})

        response_json = r.json()
        if response_json.get("error"):
            log.info(response_json.get("error").get("string"))
            return ""

        return r.json()
    except ConnectionError:
        log.error("Calling XRoad service HasOrganizationChanged failed")
        return None

def _get_company_changes(url, business_id, changed_after ):
    try:
        r = http.get(url + '/Consumer/HasCompanyChanged', params = {'businessId': business_id, 'changedAfter': changed_after},
                headers = {'Accept': 'application/json'})

        response_json = r.json()
        if response_json.get("error"):
            log.info(response_json.get("error").get("string"))
            return ""

        return r.json()
    except ConnectionError:
        log.error("Calling XRoad service HasCompanyChanged failed")
        return None

def _convert_xroad_value_to_uniform_list(value):
    if isinstance(value, basestring):
        return []

    if type(value) is dict:
        return [value]

    return value
<|endoftext|>


<commit_before>from items import *
import sys

class Inventory(object):
    def __init__(self, slot_count = 27):
        self.slot_count = slot_count
        self.slots = [None] * self.slot_count
        self.sort_mode = 0

    def find_empty_slot(self):
        return next((index for index,value in enumerate(self.slots) if not value), -1)

    def add_item(self, item_id, quantity = 1):
        self.sort()
        if quantity < 1:
            return False

        item_stack = self.get_item(item_id)
        if item_id >= ITEM_ID_MIN:
            max_size = ITEMS_DIR[item_id].max_stack_size
        else:
            max_size = BLOCKS_DIR[item_id].max_stack_size

        if item_stack:
            retval = False
            while quantity > 0:
                # can't find an unfilled slot
                if not item_stack:
                    # find an empty slot to store these items
                    index = self.find_empty_slot()

                    if index == -1 and len(self.slots) == self.slot_count:
                        return retval

                    # overflow ?
                    if quantity > max_size:
                        quantity -= max_size
                        item_stack = ItemStack(type=item_id, amount=max_size)
                    else:
                        item_stack = ItemStack(type=item_id, amount=quantity)
                        quantity = 0

                    self.slots.insert(index, item_stack)
                    retval = True
                else:
                    capacity = max_size - item_stack.amount
                    if quantity < capacity:     # there is a slot with enough space
                        item_stack.change_amount(quantity)
                        self.sort()
                        return True
                    else:   # overflow
                        quantity -= capacity
                        item_stack.change_amount(capacity)
                        # find next unfilled slot
                        item_stack = self.get_unfilled_item(item_id)

        else:
            while quantity > 0:
                index = self.find_empty_slot()

                retval = False
                if index == -1 and len(self.slots) == self.slot_count:
                    return retval

                # overflow ?
                if quantity > max_size:
                    quantity -= max_size
                    item_stack = ItemStack(type=item_id, amount=max_size)
                else:
                    item_stack = ItemStack(type=item_id, amount=quantity)
                    quantity = 0

                self.slots.insert(index, item_stack)
                retval = True
        self.sort()
        return True

    def remove_item(self, item_id, quantity = 1):
        self.sort()
        if quantity < 1:
            return False

        index = self.get_index(item_id)
        if index >= 0:
            self.slots[index].change_amount(quantity*-1)
            if self.slots[index].amount == 0:
                self.slots[index] = None
                self.sort()
                return True
        self.sort()
        return False

    def sort(self, reverse=True):
        if self.sort_mode == 0:
            self.sort_with_key(key=lambda x: x.id() if x != None else -sys.maxint - 1, reverse=True)
        elif self.sort_mode == 1:
            self.sort_with_key(key=lambda x: x.id() if x != None else sys.maxint - 1, reverse=False)
        if self.sort_mode == 2:
            self.sort_with_key(key=lambda x: x.amount if x != None else -sys.maxint - 1, reverse=True)
        elif self.sort_mode == 3:
            self.sort_with_key(key=lambda x: x.amount if x != None else sys.maxint - 1, reverse=False)

    def sort_with_key(self, key, reverse=True):
        self.slots = sorted(self.slots, key=key, reverse=reverse)

    def change_sort_mode(self, change=1):
        self.sort_mode += change
        if self.sort_mode > 3:
            self.sort_mode = 0
        elif self.sort_mode < 0:
            self.sort_mode = 3
        self.sort()

    def at(self, index):
        index = int(index)
        if index >= 0 and index < self.slot_count:
            return self.slots[index]
        return None

    def get_index(self, item_id):
        return next((index for index, x in enumerate(self.slots) if x and x.type == item_id), -1)

    def get_item(self, item_id):
        return next((x for x in self.slots if x and x.type == item_id), None)

    def get_unfilled_item(self, item_id):
        return next((x for x in self.slots if x and (x.type == item_id) and (x.amount < x.max_stack_size)), None)

    def get_items(self):
        return self.slots
<commit_msg>Little fix (2).<commit_after>from items import *
import sys

class Inventory(object):
    def __init__(self, slot_count = 27):
        self.slot_count = slot_count
        self.slots = [None] * self.slot_count
        self.sort_mode = 0

    def find_empty_slot(self):
        return next((index for index,value in enumerate(self.slots) if not value), -1)

    def add_item(self, item_id, quantity = 1):
        self.sort()
        if quantity < 1:
            return False

        item_stack = self.get_item(item_id)
        if item_id >= ITEM_ID_MIN:
            max_size = ITEMS_DIR[item_id].max_stack_size
        else:
            max_size = BLOCKS_DIR[item_id].max_stack_size

        if item_stack:
            retval = False
            while quantity > 0:
                # can't find an unfilled slot
                if not item_stack:
                    # find an empty slot to store these items
                    index = self.find_empty_slot()

                    if index == -1 and len(self.slots) == self.slot_count:
                        return retval

                    # overflow ?
                    if quantity > max_size:
                        quantity -= max_size
                        item_stack = ItemStack(type=item_id, amount=max_size)
                    else:
                        item_stack = ItemStack(type=item_id, amount=quantity)
                        quantity = 0

                    self.slots.insert(index, item_stack)
                    retval = True
                else:
                    capacity = max_size - item_stack.amount
                    if quantity < capacity:     # there is a slot with enough space
                        item_stack.change_amount(quantity)
                        self.sort()
                        return True
                    else:   # overflow
                        quantity -= capacity
                        item_stack.change_amount(capacity)
                        # find next unfilled slot
                        item_stack = self.get_unfilled_item(item_id)

        else:
            while quantity > 0:
                index = self.find_empty_slot()

                retval = False
                if index == -1 and len(self.slots) == self.slot_count:
                    return retval

                # overflow ?
                if quantity > max_size:
                    quantity -= max_size
                    item_stack = ItemStack(type=item_id, amount=max_size)
                else:
                    item_stack = ItemStack(type=item_id, amount=quantity)
                    quantity = 0

                self.slots.insert(index, item_stack)
                retval = True
        self.sort()
        return True

    def remove_item(self, item_id, quantity = 1):
        self.sort()
        if quantity < 1:
            return False

        index = self.get_index(item_id)
        if index >= 0:
            self.slots[index].change_amount(quantity*-1)
            if self.slots[index].amount == 0:
                self.slots[index] = None
                self.sort()
                return True
        self.sort()
        return False

    def sort(self, reverse=True):
        if self.sort_mode == 0:
            self.sort_with_key(key=lambda x: x.id() if x != None else -sys.maxint - 1, reverse=True)
        if self.sort_mode == 1:
            self.sort_with_key(key=lambda x: x.amount if x != None else -sys.maxint - 1, reverse=True)
        elif self.sort_mode == 2:
            self.sort_with_key(key=lambda x: x.amount if x != None else sys.maxint - 1, reverse=False)

    def sort_with_key(self, key, reverse=True):
        self.slots = sorted(self.slots, key=key, reverse=reverse)

    def change_sort_mode(self, change=1):
        self.sort_mode += change
        if self.sort_mode > 2:
            self.sort_mode = 0
        elif self.sort_mode < 0:
            self.sort_mode = 2
        self.sort()

    def at(self, index):
        index = int(index)
        if index >= 0 and index < self.slot_count:
            return self.slots[index]
        return None

    def get_index(self, item_id):
        return next((index for index, x in enumerate(self.slots) if x and x.type == item_id), -1)

    def get_item(self, item_id):
        return next((x for x in self.slots if x and x.type == item_id), None)

    def get_unfilled_item(self, item_id):
        return next((x for x in self.slots if x and (x.type == item_id) and (x.amount < x.max_stack_size)), None)

    def get_items(self):
        return self.slots
<|endoftext|>