<commit_before># -*- coding: utf-8 -*-

# Copyright (C) 2014 Johannes Baiter <johannes.baiter@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging
import logging.handlers
import os
import sys
from itertools import chain
from threading import Thread

from spreads.vendor.huey import SqliteHuey
from spreads.vendor.huey.consumer import Consumer
from flask import Flask
from tornado.wsgi import WSGIContainer
from tornado.ioloop import IOLoop
from tornado.web import FallbackHandler, Application

import spreads.workflow
import spreads.plugin as plugin
from spreads.util import is_os
from spreads.config import OptionTemplate
from spreads.main import add_argument_from_template

app = Flask('spreadsplug.web', static_url_path='/static',
            static_folder='./client/build', template_folder='./client')
task_queue = None
import web
import util
import handlers
app.json_encoder = util.CustomJSONEncoder
from discovery import DiscoveryListener

logger = logging.getLogger('spreadsplug.web')

try:
    # Most reliable way to get IP address, requires third-party package with
    # C extension
    import netifaces

    def get_ip_address():
        try:
            iface = next(dev for dev in netifaces.interfaces()
                         if 'wlan' in dev or 'eth' in dev
                         or dev.startswith('br'))
        except StopIteration:
            return None
        addresses = netifaces.ifaddresses(iface)
        if netifaces.AF_INET in addresses:
            return addresses[netifaces.AF_INET][0]['addr']
except ImportError:
    # Solution with built-ins, not as reliable
    import socket

    def get_ip_address():
        try:
            return next(
                ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
                if not ip.startswith("127."))
        except (StopIteration, socket.gaierror):
            return None

class WebCommands(plugin.HookPlugin, plugin.SubcommandHookMixin):
    __name__ = 'web'

    @classmethod
    def add_command_parser(cls, rootparser, config):
        cmdparser = rootparser.add_parser(
            'web', help="Start the web interface")
        cmdparser.set_defaults(subcommand=cls.run)

        if is_os('windows'):
            wincmdparser = rootparser.add_parser(
                'web-service', help="Start the web interface as a service."
            )
            wincmdparser.set_defaults(subcommand=cls.run_windows_service)

        for key, option in cls.configuration_template().iteritems():
            try:
                add_argument_from_template('web', key, option, cmdparser,
                                           config['web'][key].get())
                if is_os('windows'):
                    add_argument_from_template('web', key, option,
                                               wincmdparser,
                                               config['web'][key].get())
            except TypeError:
                continue

    @classmethod
    def configuration_template(cls):
        return {
            'mode': OptionTemplate(
                value=["full", "scanner", "processor"],
                docstring="Mode to run server in",
                selectable=True),
            'debug': OptionTemplate(
                value=False,
                docstring="Run server in debugging mode",
                selectable=False),
            'project_dir': OptionTemplate(
                value=u"~/scans",
                docstring="Directory for project folders",
                selectable=False),
            'postprocessing_server': OptionTemplate(
                value=u"",  # Cannot be None because of type deduction in
                            # option parser
                docstring="Address of the postprocessing server",
                selectable=False),
            'standalone_device': OptionTemplate(
                value=False,
                docstring="Server runs on a standalone device dedicated to "
                          "scanning (e.g. 'spreadpi').",
                selectable=False),
            'port': OptionTemplate(
                value=5000,
                docstring="TCP-Port to listen on",
                selectable=False),
        }

    @staticmethod
    def run(config):
        app = WebApplication(config)
        app.run_server()

    @staticmethod
    def run_windows_service(config):
        import webbrowser
        from winservice import SysTrayIcon

        app = WebApplication(config)
        server_thread = Thread(target=app.run_server)
        server_thread.start()

        def on_quit(systray):
            IOLoop.instance().stop()
            server_thread.join()

        listening_port = config['web']['port'].get(int)
        open_browser = (
            lambda x: webbrowser.open_new_tab("http://127.0.0.1:{0}"
                                              .format(listening_port)))
        menu_options = (('Open in browser', None, open_browser),)

        SysTrayIcon(
            icon=os.path.join(os.path.dirname(sys.argv[0]), 'spreads.ico'),
            hover_text="Spreads Web Service",
            menu_options=menu_options,
            on_quit=on_quit,
            default_menu_index=1,
            on_click=open_browser)

class WebApplication(object):
    def __init__(self, config):
        self.global_config = config
        self.config = config['web']
        mode = self.config['mode'].get()
        logger.debug("Starting scanning station server in \"{0}\" mode"
                     .format(mode))
        project_dir = os.path.expanduser(self.config['project_dir'].get())
        if not os.path.exists(project_dir):
            os.mkdir(project_dir)

        self._debug = self.config['debug'].get(bool)
        app.config['debug'] = self._debug
        app.config['mode'] = mode
        app.config['base_path'] = project_dir
        app.config['default_config'] = config
        app.config['standalone'] = self.config['standalone_device'].get()
        app.config['postprocessing_server'] = (
            self.config['postprocessing_server'].get() or None)
        if not self._debug:
            app.error_handler_spec[None][500] = web.handle_general_exception

    def setup_task_queue(self):
        # Initialize huey task queue
        global task_queue
        db_location = self.global_config.cfg_path.parent / 'queue.db'
        task_queue = SqliteHuey(location=unicode(db_location))
        self.consumer = Consumer(task_queue)

    def setup_logging(self):
        # Add in-memory log handler
        memoryhandler = logging.handlers.BufferingHandler(1024*10)
        memoryhandler.setLevel(logging.DEBUG)
        logger.root.addHandler(memoryhandler)

        # Silence some rather annoying loggers
        logging.getLogger('huey.consumer').setLevel(logging.INFO)
        logging.getLogger('huey.consumer.ConsumerThread').setLevel(
            logging.INFO)
        logging.getLogger('bagit').setLevel(logging.ERROR)
        logging.getLogger('isbnlib.dev.webservice').setLevel(logging.ERROR)
        logging.getLogger('tornado.access').setLevel(logging.ERROR)

    def setup_signals(self):
        def get_signal_callback_http(signal):
            def signal_callback(sender, **kwargs):
                event = util.Event(signal, sender, kwargs)
                handlers.event_buffer.new_events([event])
            return signal_callback

        def get_signal_callback_websockets(signal):
            def signal_callback(sender, **kwargs):
                handlers.WebSocketHandler.send_event(
                    util.Event(signal, sender, kwargs))
            return signal_callback

        # Register event handlers
        import tasks
        signals_ = chain(*(x.signals.values()
                           for x in (spreads.workflow, util.EventHandler,
                                     tasks, handlers)))

        for signal in signals_:
            signal.connect(get_signal_callback_http(signal), weak=False)
            signal.connect(get_signal_callback_websockets(signal), weak=False)

    def setup_tornado(self):
        if self._debug:
            logger.info("Starting server in debugging mode")
            from werkzeug.debug import DebuggedApplication
            container = WSGIContainer(DebuggedApplication(app, evalex=True))
        else:
            container = WSGIContainer(app)
        self.application = Application([
            (r"/ws", handlers.WebSocketHandler),
            (r"/api/workflow/([0-9a-z-]+)/download/(.*)\.zip",
             handlers.ZipDownloadHandler,
             dict(base_path=app.config['base_path'])),
            (r"/api/workflow/([0-9a-z-]+)/download/(.*).\.tar",
             handlers.TarDownloadHandler,
             dict(base_path=app.config['base_path'])),
            (r"/api/workflow/upload",
             handlers.StreamingUploadHandler,
             dict(base_path=app.config['base_path'])),
            (r"/api/poll", handlers.EventLongPollingHandler),
            (r".*", FallbackHandler, dict(fallback=container))
        ], debug=self._debug)

    def run_server(self):
        self.setup_logging()
        self.setup_task_queue()
        self.setup_signals()
        self.setup_tornado()

        listening_port = self.config['port'].get(int)

        ip_address = get_ip_address()
        if (app.config['standalone'] and ip_address
                and self.global_config['driver'].get() == 'chdkcamera'):
            # Display the address of the web interface on the camera displays
            try:
                for cam in plugin.get_devices(self.global_config):
                    cam.show_textbox(
                        "\n    You can now access the web interface at:"
                        "\n\n\n         http://{0}:{1}"
                        .format(ip_address, listening_port))
            except plugin.DeviceException:
                logger.warn("No devices could be found at startup.")

        # Start task consumer
        self.consumer.start()

        # Start discovery listener
        if app.config['mode'] in ('processor', 'full'):
            discovery_listener = DiscoveryListener(listening_port)
            discovery_listener.start()

        # Spin up WSGI server
        self.application.listen(listening_port)

        try:
            IOLoop.instance().start()
        finally:
            self.consumer.shutdown()
            if app.config['mode'] in ('processor', 'full'):
                discovery_listener.stop()
<commit_msg>web: If IP can't be shown at startup, periodically check for them and display address (fixes #122)<commit_after># -*- coding: utf-8 -*-

# Copyright (C) 2014 Johannes Baiter <johannes.baiter@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging
import logging.handlers
import os
import sys
from itertools import chain
from threading import Thread

from spreads.vendor.huey import SqliteHuey
from spreads.vendor.huey.consumer import Consumer
from flask import Flask
from tornado.wsgi import WSGIContainer
from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.web import FallbackHandler, Application

import spreads.workflow
import spreads.plugin as plugin
from spreads.util import is_os
from spreads.config import OptionTemplate
from spreads.main import add_argument_from_template

app = Flask('spreadsplug.web', static_url_path='/static',
            static_folder='./client/build', template_folder='./client')
task_queue = None
import web
import util
import handlers
app.json_encoder = util.CustomJSONEncoder
from discovery import DiscoveryListener

logger = logging.getLogger('spreadsplug.web')

try:
    # Most reliable way to get IP address, requires third-party package with
    # C extension
    import netifaces

    def get_ip_address():
        try:
            iface = next(dev for dev in netifaces.interfaces()
                         if 'wlan' in dev or 'eth' in dev
                         or dev.startswith('br'))
        except StopIteration:
            return None
        addresses = netifaces.ifaddresses(iface)
        if netifaces.AF_INET in addresses:
            return addresses[netifaces.AF_INET][0]['addr']
except ImportError:
    # Solution with built-ins, not as reliable
    import socket

    def get_ip_address():
        try:
            return next(
                ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
                if not ip.startswith("127."))
        except (StopIteration, socket.gaierror):
            return None

class WebCommands(plugin.HookPlugin, plugin.SubcommandHookMixin):
    __name__ = 'web'

    @classmethod
    def add_command_parser(cls, rootparser, config):
        cmdparser = rootparser.add_parser(
            'web', help="Start the web interface")
        cmdparser.set_defaults(subcommand=cls.run)

        if is_os('windows'):
            wincmdparser = rootparser.add_parser(
                'web-service', help="Start the web interface as a service."
            )
            wincmdparser.set_defaults(subcommand=cls.run_windows_service)

        for key, option in cls.configuration_template().iteritems():
            try:
                add_argument_from_template('web', key, option, cmdparser,
                                           config['web'][key].get())
                if is_os('windows'):
                    add_argument_from_template('web', key, option,
                                               wincmdparser,
                                               config['web'][key].get())
            except TypeError:
                continue

    @classmethod
    def configuration_template(cls):
        return {
            'mode': OptionTemplate(
                value=["full", "scanner", "processor"],
                docstring="Mode to run server in",
                selectable=True),
            'debug': OptionTemplate(
                value=False,
                docstring="Run server in debugging mode",
                selectable=False),
            'project_dir': OptionTemplate(
                value=u"~/scans",
                docstring="Directory for project folders",
                selectable=False),
            'postprocessing_server': OptionTemplate(
                value=u"",  # Cannot be None because of type deduction in
                            # option parser
                docstring="Address of the postprocessing server",
                selectable=False),
            'standalone_device': OptionTemplate(
                value=False,
                docstring="Server runs on a standalone device dedicated to "
                          "scanning (e.g. 'spreadpi').",
                selectable=False),
            'port': OptionTemplate(
                value=5000,
                docstring="TCP-Port to listen on",
                selectable=False),
        }

    @staticmethod
    def run(config):
        app = WebApplication(config)
        app.run_server()

    @staticmethod
    def run_windows_service(config):
        import webbrowser
        from winservice import SysTrayIcon

        app = WebApplication(config)
        server_thread = Thread(target=app.run_server)
        server_thread.start()

        def on_quit(systray):
            IOLoop.instance().stop()
            server_thread.join()

        listening_port = config['web']['port'].get(int)
        open_browser = (
            lambda x: webbrowser.open_new_tab("http://127.0.0.1:{0}"
                                              .format(listening_port)))
        menu_options = (('Open in browser', None, open_browser),)

        SysTrayIcon(
            icon=os.path.join(os.path.dirname(sys.argv[0]), 'spreads.ico'),
            hover_text="Spreads Web Service",
            menu_options=menu_options,
            on_quit=on_quit,
            default_menu_index=1,
            on_click=open_browser)

class WebApplication(object):
    def __init__(self, config):
        self.global_config = config
        self.config = config['web']
        mode = self.config['mode'].get()
        logger.debug("Starting scanning station server in \"{0}\" mode"
                     .format(mode))
        project_dir = os.path.expanduser(self.config['project_dir'].get())
        if not os.path.exists(project_dir):
            os.mkdir(project_dir)

        self._debug = self.config['debug'].get(bool)
        app.config['debug'] = self._debug
        app.config['mode'] = mode
        app.config['base_path'] = project_dir
        app.config['default_config'] = config
        app.config['standalone'] = self.config['standalone_device'].get()
        app.config['postprocessing_server'] = (
            self.config['postprocessing_server'].get() or None)
        if not self._debug:
            app.error_handler_spec[None][500] = web.handle_general_exception

    def setup_task_queue(self):
        # Initialize huey task queue
        global task_queue
        db_location = self.global_config.cfg_path.parent / 'queue.db'
        task_queue = SqliteHuey(location=unicode(db_location))
        self.consumer = Consumer(task_queue)

    def setup_logging(self):
        # Add in-memory log handler
        memoryhandler = logging.handlers.BufferingHandler(1024*10)
        memoryhandler.setLevel(logging.DEBUG)
        logger.root.addHandler(memoryhandler)

        # Silence some rather annoying loggers
        logging.getLogger('huey.consumer').setLevel(logging.INFO)
        logging.getLogger('huey.consumer.ConsumerThread').setLevel(
            logging.INFO)
        logging.getLogger('bagit').setLevel(logging.ERROR)
        logging.getLogger('isbnlib.dev.webservice').setLevel(logging.ERROR)
        logging.getLogger('tornado.access').setLevel(logging.ERROR)

    def setup_signals(self):
        def get_signal_callback_http(signal):
            def signal_callback(sender, **kwargs):
                event = util.Event(signal, sender, kwargs)
                handlers.event_buffer.new_events([event])
            return signal_callback

        def get_signal_callback_websockets(signal):
            def signal_callback(sender, **kwargs):
                handlers.WebSocketHandler.send_event(
                    util.Event(signal, sender, kwargs))
            return signal_callback

        # Register event handlers
        import tasks
        signals_ = chain(*(x.signals.values()
                           for x in (spreads.workflow, util.EventHandler,
                                     tasks, handlers)))

        for signal in signals_:
            signal.connect(get_signal_callback_http(signal), weak=False)
            signal.connect(get_signal_callback_websockets(signal), weak=False)

    def setup_tornado(self):
        if self._debug:
            logger.info("Starting server in debugging mode")
            from werkzeug.debug import DebuggedApplication
            container = WSGIContainer(DebuggedApplication(app, evalex=True))
        else:
            container = WSGIContainer(app)
        self.application = Application([
            (r"/ws", handlers.WebSocketHandler),
            (r"/api/workflow/([0-9a-z-]+)/download/(.*)\.zip",
             handlers.ZipDownloadHandler,
             dict(base_path=app.config['base_path'])),
            (r"/api/workflow/([0-9a-z-]+)/download/(.*).\.tar",
             handlers.TarDownloadHandler,
             dict(base_path=app.config['base_path'])),
            (r"/api/workflow/upload",
             handlers.StreamingUploadHandler,
             dict(base_path=app.config['base_path'])),
            (r"/api/poll", handlers.EventLongPollingHandler),
            (r".*", FallbackHandler, dict(fallback=container))
        ], debug=self._debug)

    def display_ip(self):
        # Display the address of the web interface on the camera displays
        try:
            for cam in plugin.get_devices(self.global_config):
                cam.show_textbox(
                    "\n    You can now access the web interface at:"
                    "\n\n\n         http://{0}:{1}"
                    .format(self._ip_address, self._listening_port))
            self._display_callback.stop()
        except plugin.DeviceException:
            # Try again next time...
            return

    def run_server(self):
        self.setup_logging()
        self.setup_task_queue()
        self.setup_signals()
        self.setup_tornado()

        self._listening_port = self.config['port'].get(int)

        self._ip_address = get_ip_address()
        device_driver = plugin.get_driver(self.global_config['driver'].get())
        should_display_ip = (app.config['standalone'] and self._ip_address
                             and plugin.DeviceFeatures.CAN_DISPLAY_TEXT in
                             device_driver.features)
        if should_display_ip:
            # Every 30 seconds, see if there are devices attached and display
            # IP address and port on them, then disable the callback
            self._display_callback = PeriodicCallback(
                self.display_ip, 30*10**3)
            # Run once immediately
            self.display_ip()

        # Start task consumer
        self.consumer.start()

        # Start discovery listener
        if app.config['mode'] in ('processor', 'full'):
            discovery_listener = DiscoveryListener(self._listening_port)
            discovery_listener.start()

        # Spin up WSGI server
        self.application.listen(self._listening_port)

        try:
            IOLoop.instance().start()
        finally:
            self.consumer.shutdown()
            if app.config['mode'] in ('processor', 'full'):
                discovery_listener.stop()
<|endoftext|>


<commit_before>#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

import logging
import sys
if (sys.version_info > (3,)):
  from urllib.parse import quote
else:
  from urllib import quote
import jsonpickle
from cairis.core.Asset import Asset
from cairis.core.ObjectSummary import ObjectSummary
from cairis.core.AssetEnvironmentProperties import AssetEnvironmentProperties
from cairis.core.ValueType import ValueType
from cairis.test.CairisDaemonTestCase import CairisDaemonTestCase
from cairis.tools.JsonConverter import json_deserialize
from cairis.tools.ModelDefinitions import AssetEnvironmentPropertiesModel, SecurityAttribute
from cairis.tools.SessionValidator import check_required_keys
from cairis.mio.ModelImport import importModelFile
import os

__author__ = 'Robin Quetin, Shamal Faily'

class AssetAPITests(CairisDaemonTestCase):

  @classmethod
  def setUpClass(cls):
    importModelFile(os.environ['CAIRIS_SRC'] + '/../examples/exemplars/NeuroGrid/NeuroGrid.xml',1,'test')

  def setUp(self):
    # region Class fields
    self.logger = logging.getLogger(__name__)
    self.existing_asset_name = 'Data node'
    self.new_asset = Asset(
      assetId=-1,
      assetName='Test',
      shortCode='TST',
      assetDescription='This is a new test asset',
      assetSig='Very significant',
      assetType='Hardware',
      cFlag=0,
      cRationale=None,
      tags=[],
      ifs=[],
      cProps=[]
    )
    self.new_asset_sec_attr = [
      SecurityAttribute(
        'Accountability',
        'Low',
        'None'
      ),
      SecurityAttribute(
        'Confidentiality',
        'Medium',
        'None'
      )
    ]
    self.new_asset_props = [
      AssetEnvironmentProperties(
        environmentName='Stroke',
        associations=[[0,"Association","1..*","","","1","Association",0,"Grid meta-data"]],
        syProperties=self.new_asset_sec_attr,
        pRationale=[]
      )
    ]
    self.new_asset_dict = {
      'session_id': 'test',
      'object': self.new_asset
    }
    # endregion

  def test_get_all(self):
    method = 'test_get_all'
    rv = self.app.get('/api/assets?session_id=test')
    if (sys.version_info > (3,)):
      assets = json_deserialize(rv.data.decode('utf-8'))
    else:
      assets = json_deserialize(rv.data)
    self.assertIsNotNone(assets, 'No results after deserialization')
    self.assertIsInstance(assets, dict, 'The result is not a dictionary as expected')
    self.assertGreater(len(assets), 0, 'No assets in the dictionary')
    self.assertIsInstance(list(assets.values())[0], dict)
    self.logger.info('[%s] Assets found: %d', method, len(assets))
    self.logger.info('[%s] First asset: %s\n', method, list(assets.values())[0]['theName'])

  def test_get_all_summary(self):
    method = 'test_get_all_summary'
    rv = self.app.get('/api/assets/summary?session_id=test')
    if (sys.version_info > (3,)):
      assets = json_deserialize(rv.data.decode('utf-8'))
    else:
      assets = json_deserialize(rv.data)
    self.assertIsNotNone(assets, 'No results after deserialization')
    self.assertGreater(len(assets), 0, 'No asset summaries')
    self.assertIsInstance(assets[0], dict)
    self.logger.info('[%s] Assets found: %d', method, len(assets))
    self.logger.info('[%s] First asset summary: %s [%d]\n', method, assets[0]['theName'], assets[0]['theType'])

  def test_post(self):
    method = 'test_post_new'
    rv = self.app.post('/api/assets', content_type='application/json', data=jsonpickle.encode(self.new_asset_dict))
    if (sys.version_info > (3,)):
      postResponse = rv.data.decode('utf-8')
    else:
      postResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, postResponse)
    json_resp = json_deserialize(postResponse)
    self.assertIsNotNone(postResponse, 'No results after deserialization')
    url = '/api/assets/name/' + quote(self.new_asset_dict['object'].theName) + '?session_id=test'
    rv = self.app.get(url)
    if (sys.version_info > (3,)):
      asset = json_deserialize(rv.data.decode('utf-8'))
    else:
      asset = json_deserialize(rv.data)
    self.logger.info('[%s] Asset: %s\n', method, asset['theName'])

  def test_get_invalid_name(self):
    method = 'test_get_name'
    url = '/api/assets/name/invalidname?session_id=test'
    rv = self.app.get(url)
    if (sys.version_info > (3,)):
      msg = json_deserialize(rv.data.decode('utf-8'))
    else:
      msg = json_deserialize(rv.data)
    self.assertIsNotNone(msg, 'No results after deserialization')
    self.assertEqual(msg['code'],404)

  def test_get_name(self):
    method = 'test_get_name'
    url = '/api/assets/name/%s?session_id=test' % quote(self.existing_asset_name)
    rv = self.app.get(url)
    if (sys.version_info > (3,)):
      asset = json_deserialize(rv.data.decode('utf-8'))
    else:
      asset = json_deserialize(rv.data)
    self.assertIsNotNone(asset, 'No results after deserialization')
    self.assertEqual(asset['theName'],self.existing_asset_name)

  def test_put_name(self):
    method = 'test_put_name'
    rv = self.app.post('/api/assets', content_type='application/json', data=jsonpickle.encode(self.new_asset_dict))

    url = '/api/assets/name/%s' % quote(self.new_asset.theName)
    upd_asset = self.new_asset
    upd_asset.theName = 'Test2'
    upd_asset_dict = self.new_asset_dict
    upd_asset_dict['object'] = upd_asset
    upd_asset_body = jsonpickle.encode(upd_asset_dict)
    self.logger.info('[%s] JSON data: %s', method, upd_asset_body)

    rv = self.app.put(url, content_type='application/json', data=upd_asset_body)
    if (sys.version_info > (3,)):
      putResponse = rv.data.decode('utf-8')
    else:
      putResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, putResponse)
    json_resp = json_deserialize(putResponse)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message returned')

    rv = self.app.get('/api/assets/name/Test2?session_id=test')
    if (sys.version_info > (3,)):
      asset = json_deserialize(rv.data.decode('utf-8'))
    else:
      asset = json_deserialize(rv.data)
    self.logger.info('[%s] Asset: %s\n', method, asset['theName'])

  def test_delete_name(self):
    method = 'test_delete_name'
    url = '/api/assets/name/{}?session_id=test'.format(quote(self.new_asset.theName))
    rv = self.app.delete(url)
    url = '/api/assets/name/Test2?session_id=test'.format(quote(self.new_asset.theName))
    rv = self.app.delete(url)
    if (sys.version_info > (3,)):
      delResponse = rv.data.decode('utf-8')
    else:
      delResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, delResponse)
    json_resp = json_deserialize(delResponse)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message returned')
    self.logger.info('[%s] Message: %s\n', method, message)

  def test_types_get(self):
    method = 'test_types_get'
    rv = self.app.get('/api/assets/types?session_id=test')
    if (sys.version_info > (3,)):
      assets = jsonpickle.decode(rv.data.decode('utf-8'))
    else:
      assets = jsonpickle.decode(rv.data)
    self.assertIsNotNone(assets, 'No results after deserialization')
    self.assertIsInstance(assets, list, 'The result is not a dictionary as expected')
    self.assertGreater(len(assets), 0, 'No assets in the dictionary')
    self.logger.info('[%s] Asset types found: %d', method, len(assets))
    asset_type = assets[0]
    self.logger.info('[%s] First asset types: %s\n', method, asset_type['theName'])

  def test_types_delete(self):
    method = 'test_types_delete'
    url = '/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName)
    json_dict = {
      'session_id': 'test',
      'object': self.prepare_new_asset_type()
    }
    new_asset_type_body = jsonpickle.encode(json_dict, unpicklable=False)

    self.app.delete(url)
    self.logger.info('[%s] Object to delete: %s', method, new_asset_type_body)
    self.app.post('/api/assets/types', content_type='application/json', data=new_asset_type_body)
    self.logger.info('[%s] URL: %s', method, url)
    rv = self.app.delete(url)
    if (sys.version_info > (3,)):
      delResponse = rv.data.decode('utf-8')
    else:
      delResponse = rv.data
    self.logger.info('[%s] Response data: %s', method, delResponse)
    self.assertIsNotNone(delResponse, 'No response')
    json_resp = jsonpickle.decode(delResponse)
    self.assertIsInstance(json_resp, dict, 'The response cannot be converted to a dictionary')
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message in response')
    self.logger.info('[%s] Message: %s\n', method, message)

  def test_types_post(self):
    method = 'test_types_post'
    url = '/api/assets/types'
    self.logger.info('[%s] URL: %s', method, url)
    json_dict = {'session_id': 'test', 'object': self.prepare_new_asset_type()}
    new_asset_type_body = jsonpickle.encode(json_dict, unpicklable=False)
    self.logger.info('JSON data: %s', new_asset_type_body)

    self.app.delete('/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName))
    rv = self.app.post(url, content_type='application/json', data=new_asset_type_body)
    if (sys.version_info > (3,)):
      postResponse = rv.data.decode('utf-8')
    else:
      postResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, postResponse)
    json_resp = jsonpickle.decode(postResponse)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    ackMsg = json_resp.get('message', None)
    self.assertEqual(ackMsg, 'Asset type successfully added')

    rv = self.app.delete('/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName))

  def test_types_put(self):
    method = 'test_types_put'
    url = '/api/assets/types'
    self.logger.info('[%s] URL: %s', method, url)
    json_dict = {'session_id': 'test', 'object': self.prepare_new_asset_type()}
    new_asset_type_body = jsonpickle.encode(json_dict)
    self.logger.info('JSON data: %s', new_asset_type_body)

    rv = self.app.delete('/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName))
    rv = self.app.post(url, content_type='application/json', data=new_asset_type_body)
    if (sys.version_info > (3,)):
      postResponse = rv.data.decode('utf-8')
    else:
      postResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, postResponse)
    json_resp = jsonpickle.decode(postResponse)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    ackMsg = json_resp.get('message', None)
    self.assertIsNotNone(ackMsg, 'Asset type successfully updated')

    type_to_update = self.prepare_new_asset_type()
    type_to_update.theName = 'Edited test asset type'
    json_dict = {'session_id': 'test', 'object': type_to_update}
    upd_type_body = jsonpickle.encode(json_dict)
    rv = self.app.put('/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName), data=upd_type_body, content_type='application/json')
    if (sys.version_info > (3,)):
      putResponse = rv.data.decode('utf-8')
    else:
      putResponse = rv.data
    self.assertIsNotNone(rv.data, 'No response')
    json_resp = jsonpickle.decode(putResponse)
    self.assertIsNotNone(json_resp)
    self.assertIsInstance(json_resp, dict)
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message in response')
    self.logger.info('[%s] Message: %s', method, message)
    self.assertGreater(message.find('successfully updated'), -1, 'The asset was not successfully updated')

    rv = self.app.get('/api/assets/types/name/%s?session_id=test' % quote(type_to_update.theName))
    if (sys.version_info > (3,)):
      getResponse = rv.data.decode('utf-8')
    else:
      getResponse = rv.data
    upd_asset_type = jsonpickle.decode(getResponse)
    self.assertIsNotNone(upd_asset_type, 'Unable to decode JSON data')
    self.logger.debug('[%s] Response data: %s', method, getResponse)
    self.logger.info('[%s] Asset type: %s\n', method, upd_asset_type['theName'])

    rv = self.app.delete('/api/assets/types/name/%s?session_id=test' % quote(type_to_update.theName))

  def prepare_new_asset_type(self):
    new_type = ValueType(
                 valueTypeId=-1,
                 valueTypeName='Test asset type',
                 valueTypeDescription='This is a test asset type',
                 vType='asset-type',
                 vEnv='all'
               )
    new_type.theEnvironmentName = 'all'
    return new_type
<commit_msg>Fix broken Asset API test<commit_after>#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

import logging
import sys
if (sys.version_info > (3,)):
  from urllib.parse import quote
else:
  from urllib import quote
import jsonpickle
from cairis.core.Asset import Asset
from cairis.core.ObjectSummary import ObjectSummary
from cairis.core.AssetEnvironmentProperties import AssetEnvironmentProperties
from cairis.core.ValueType import ValueType
from cairis.test.CairisDaemonTestCase import CairisDaemonTestCase
from cairis.tools.JsonConverter import json_deserialize
from cairis.tools.ModelDefinitions import AssetEnvironmentPropertiesModel, SecurityAttribute
from cairis.tools.SessionValidator import check_required_keys
from cairis.mio.ModelImport import importModelFile
import os

__author__ = 'Robin Quetin, Shamal Faily'

class AssetAPITests(CairisDaemonTestCase):

  @classmethod
  def setUpClass(cls):
    importModelFile(os.environ['CAIRIS_SRC'] + '/../examples/exemplars/NeuroGrid/NeuroGrid.xml',1,'test')

  def setUp(self):
    # region Class fields
    self.logger = logging.getLogger(__name__)
    self.existing_asset_name = 'Data node'
    self.new_asset = Asset(
      assetId=-1,
      assetName='Test',
      shortCode='TST',
      assetDescription='This is a new test asset',
      assetSig='Very significant',
      assetType='Hardware',
      cFlag=0,
      cRationale=None,
      tags=[],
      ifs=[],
      cProps=[]
    )
    self.new_asset_sec_attr = [
      SecurityAttribute(
        'Accountability',
        'Low',
        'None'
      ),
      SecurityAttribute(
        'Confidentiality',
        'Medium',
        'None'
      )
    ]
    self.new_asset_props = [
      AssetEnvironmentProperties(
        environmentName='Stroke',
        associations=[[0,"Association","1..*","","","1","Association",0,"Grid meta-data"]],
        syProperties=self.new_asset_sec_attr,
        pRationale=[]
      )
    ]
    self.new_asset_dict = {
      'session_id': 'test',
      'object': self.new_asset
    }
    # endregion

  def test_get_all(self):
    method = 'test_get_all'
    rv = self.app.get('/api/assets?session_id=test')
    if (sys.version_info > (3,)):
      assets = json_deserialize(rv.data.decode('utf-8'))
    else:
      assets = json_deserialize(rv.data)
    self.assertIsNotNone(assets, 'No results after deserialization')
    self.assertIsInstance(assets, dict, 'The result is not a dictionary as expected')
    self.assertGreater(len(assets), 0, 'No assets in the dictionary')
    self.assertIsInstance(list(assets.values())[0], dict)
    self.logger.info('[%s] Assets found: %d', method, len(assets))
    self.logger.info('[%s] First asset: %s\n', method, list(assets.values())[0]['theName'])

  def test_get_all_summary(self):
    method = 'test_get_all_summary'
    rv = self.app.get('/api/assets/summary?session_id=test')
    if (sys.version_info > (3,)):
      assets = json_deserialize(rv.data.decode('utf-8'))
    else:
      assets = json_deserialize(rv.data)
    self.assertIsNotNone(assets, 'No results after deserialization')
    self.assertGreater(len(assets), 0, 'No asset summaries')
    self.assertIsInstance(assets[0], dict)
    self.logger.info('[%s] Assets found: %d', method, len(assets))
    self.logger.info('[%s] First asset summary: %s [%d]\n', method, assets[0]['theName'], assets[0]['theType'])

  def test_post(self):
    method = 'test_post_new'
    rv = self.app.post('/api/assets', content_type='application/json', data=jsonpickle.encode(self.new_asset_dict))
    if (sys.version_info > (3,)):
      postResponse = rv.data.decode('utf-8')
    else:
      postResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, postResponse)
    json_resp = json_deserialize(postResponse)
    self.assertIsNotNone(postResponse, 'No results after deserialization')
    url = '/api/assets/name/' + quote(self.new_asset_dict['object'].theName) + '?session_id=test'
    rv = self.app.get(url)
    if (sys.version_info > (3,)):
      asset = json_deserialize(rv.data.decode('utf-8'))
    else:
      asset = json_deserialize(rv.data)
    self.logger.info('[%s] Asset: %s\n', method, asset['theName'])

  def test_get_invalid_name(self):
    method = 'test_get_name'
    url = '/api/assets/name/invalidname?session_id=test'
    rv = self.app.get(url)
#    if (sys.version_info > (3,)):
#      msg = json_deserialize(rv.data.decode('utf-8'))
#    else:
    msg = json_deserialize(rv.data)
    self.assertIsNotNone(msg, 'No results after deserialization')
    self.assertEqual(msg['code'],404)

  def test_get_name(self):
    method = 'test_get_name'
    url = '/api/assets/name/%s?session_id=test' % quote(self.existing_asset_name)
    rv = self.app.get(url)
    if (sys.version_info > (3,)):
      asset = json_deserialize(rv.data.decode('utf-8'))
    else:
      asset = json_deserialize(rv.data)
    self.assertIsNotNone(asset, 'No results after deserialization')
    self.assertEqual(asset['theName'],self.existing_asset_name)

  def test_put_name(self):
    method = 'test_put_name'
    rv = self.app.post('/api/assets', content_type='application/json', data=jsonpickle.encode(self.new_asset_dict))

    url = '/api/assets/name/%s' % quote(self.new_asset.theName)
    upd_asset = self.new_asset
    upd_asset.theName = 'Test2'
    upd_asset_dict = self.new_asset_dict
    upd_asset_dict['object'] = upd_asset
    upd_asset_body = jsonpickle.encode(upd_asset_dict)
    self.logger.info('[%s] JSON data: %s', method, upd_asset_body)

    rv = self.app.put(url, content_type='application/json', data=upd_asset_body)
    if (sys.version_info > (3,)):
      putResponse = rv.data.decode('utf-8')
    else:
      putResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, putResponse)
    json_resp = json_deserialize(putResponse)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message returned')

    rv = self.app.get('/api/assets/name/Test2?session_id=test')
    if (sys.version_info > (3,)):
      asset = json_deserialize(rv.data.decode('utf-8'))
    else:
      asset = json_deserialize(rv.data)
    self.logger.info('[%s] Asset: %s\n', method, asset['theName'])

  def test_delete_name(self):
    method = 'test_delete_name'
    url = '/api/assets/name/{}?session_id=test'.format(quote(self.new_asset.theName))
    rv = self.app.delete(url)
    url = '/api/assets/name/Test2?session_id=test'.format(quote(self.new_asset.theName))
    rv = self.app.delete(url)
    if (sys.version_info > (3,)):
      delResponse = rv.data.decode('utf-8')
    else:
      delResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, delResponse)
    json_resp = json_deserialize(delResponse)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message returned')
    self.logger.info('[%s] Message: %s\n', method, message)

  def test_types_get(self):
    method = 'test_types_get'
    rv = self.app.get('/api/assets/types?session_id=test')
    if (sys.version_info > (3,)):
      assets = jsonpickle.decode(rv.data.decode('utf-8'))
    else:
      assets = jsonpickle.decode(rv.data)
    self.assertIsNotNone(assets, 'No results after deserialization')
    self.assertIsInstance(assets, list, 'The result is not a dictionary as expected')
    self.assertGreater(len(assets), 0, 'No assets in the dictionary')
    self.logger.info('[%s] Asset types found: %d', method, len(assets))
    asset_type = assets[0]
    self.logger.info('[%s] First asset types: %s\n', method, asset_type['theName'])

  def test_types_delete(self):
    method = 'test_types_delete'
    url = '/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName)
    json_dict = {
      'session_id': 'test',
      'object': self.prepare_new_asset_type()
    }
    new_asset_type_body = jsonpickle.encode(json_dict, unpicklable=False)

    self.app.delete(url)
    self.logger.info('[%s] Object to delete: %s', method, new_asset_type_body)
    self.app.post('/api/assets/types', content_type='application/json', data=new_asset_type_body)
    self.logger.info('[%s] URL: %s', method, url)
    rv = self.app.delete(url)
    if (sys.version_info > (3,)):
      delResponse = rv.data.decode('utf-8')
    else:
      delResponse = rv.data
    self.logger.info('[%s] Response data: %s', method, delResponse)
    self.assertIsNotNone(delResponse, 'No response')
    json_resp = jsonpickle.decode(delResponse)
    self.assertIsInstance(json_resp, dict, 'The response cannot be converted to a dictionary')
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message in response')
    self.logger.info('[%s] Message: %s\n', method, message)

  def test_types_post(self):
    method = 'test_types_post'
    url = '/api/assets/types'
    self.logger.info('[%s] URL: %s', method, url)
    json_dict = {'session_id': 'test', 'object': self.prepare_new_asset_type()}
    new_asset_type_body = jsonpickle.encode(json_dict, unpicklable=False)
    self.logger.info('JSON data: %s', new_asset_type_body)

    self.app.delete('/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName))
    rv = self.app.post(url, content_type='application/json', data=new_asset_type_body)
    if (sys.version_info > (3,)):
      postResponse = rv.data.decode('utf-8')
    else:
      postResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, postResponse)
    json_resp = jsonpickle.decode(postResponse)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    ackMsg = json_resp.get('message', None)
    self.assertEqual(ackMsg, 'Asset type successfully added')

    rv = self.app.delete('/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName))

  def test_types_put(self):
    method = 'test_types_put'
    url = '/api/assets/types'
    self.logger.info('[%s] URL: %s', method, url)
    json_dict = {'session_id': 'test', 'object': self.prepare_new_asset_type()}
    new_asset_type_body = jsonpickle.encode(json_dict)
    self.logger.info('JSON data: %s', new_asset_type_body)

    rv = self.app.delete('/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName))
    rv = self.app.post(url, content_type='application/json', data=new_asset_type_body)
    if (sys.version_info > (3,)):
      postResponse = rv.data.decode('utf-8')
    else:
      postResponse = rv.data
    self.logger.debug('[%s] Response data: %s', method, postResponse)
    json_resp = jsonpickle.decode(postResponse)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    ackMsg = json_resp.get('message', None)
    self.assertIsNotNone(ackMsg, 'Asset type successfully updated')

    type_to_update = self.prepare_new_asset_type()
    type_to_update.theName = 'Edited test asset type'
    json_dict = {'session_id': 'test', 'object': type_to_update}
    upd_type_body = jsonpickle.encode(json_dict)
    rv = self.app.put('/api/assets/types/name/%s?session_id=test' % quote(self.prepare_new_asset_type().theName), data=upd_type_body, content_type='application/json')
    if (sys.version_info > (3,)):
      putResponse = rv.data.decode('utf-8')
    else:
      putResponse = rv.data
    self.assertIsNotNone(rv.data, 'No response')
    json_resp = jsonpickle.decode(putResponse)
    self.assertIsNotNone(json_resp)
    self.assertIsInstance(json_resp, dict)
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message in response')
    self.logger.info('[%s] Message: %s', method, message)
    self.assertGreater(message.find('successfully updated'), -1, 'The asset was not successfully updated')

    rv = self.app.get('/api/assets/types/name/%s?session_id=test' % quote(type_to_update.theName))
    if (sys.version_info > (3,)):
      getResponse = rv.data.decode('utf-8')
    else:
      getResponse = rv.data
    upd_asset_type = jsonpickle.decode(getResponse)
    self.assertIsNotNone(upd_asset_type, 'Unable to decode JSON data')
    self.logger.debug('[%s] Response data: %s', method, getResponse)
    self.logger.info('[%s] Asset type: %s\n', method, upd_asset_type['theName'])

    rv = self.app.delete('/api/assets/types/name/%s?session_id=test' % quote(type_to_update.theName))

  def prepare_new_asset_type(self):
    new_type = ValueType(
                 valueTypeId=-1,
                 valueTypeName='Test asset type',
                 valueTypeDescription='This is a test asset type',
                 vType='asset-type',
                 vEnv='all'
               )
    new_type.theEnvironmentName = 'all'
    return new_type
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

"""
Utilities for generating perfect hash functions for integer keys.

This module implements the first fit decreasing method, described in
Gettys01_. It is **not** guaranteed to generate a *minimal* perfect hash,
though by no means is it impossible. See for example:

>>> phash = hash_parameters('+-<>[].,', to_int=ord)
>>> len(phash.slots)
8
>>> phash.slots
('+', ',', '-', '.', '<', '[', '>', ']')

.. _Gettys01: http://www.drdobbs.com/architecture-and-design/generating-perfect-hash-functions/184404506

"""

import math
import collections
import heapq

HashInfo = collections.namedtuple('HashInfo', 't slots r offset to_int')

def __identity(x):
    return x

def hash_parameters(keys, minimize=True, to_int=None):
    """
    Calculates the parameters for a perfect hash. The result is returned as a
    HashInfo tuple which has the following fields:

        t       -- table parameter. In practice, this means the maximum size
                   of the output
        slots   -- the output slots in the domain
        r       -- displacement vector
        offset  -- the amount by which to offset all values (once converted to
                   ints)
        to_int  -- function to convert input to int (if given)

    >>> hash_parameters([1, 5, 7], minimize=False)
    HashInfo(t=3, slots=(1, 5, 7), r=(-1, -1, 1), offset=0, to_int=None)

    >>> hash_parameters([1, 5, 7])
    HashInfo(t=3, slots=(1, 5, 7), r=(0, 0, 2), offset=-1, to_int=None)

    >>> l = (0, 3, 4, 7 ,10, 13, 15, 18, 19, 21, 22, 24, 26, 29, 30, 34)
    >>> phash = hash_parameters(l)
    >>> phash.slots
    (18, 19, 0, 21, 22, 3, 4, 24, 7, 26, 30, 10, 29, 13, 34, 15)
    """

    # If to_int is not assigned, simply use the identity function.
    if to_int is None:
        to_int = __identity

    key_to_original = {to_int(original): original for original in keys}

    # Create a set of all items to be hashed.
    items = key_to_original.keys()

    if minimize:
        offset = 0 - min(items)
        items = frozenset(x + offset for x in items)
    else:
        offset = 0

    # 1. Start with a square array (not stored) that is t units on each side.
    # Choose a t such that t * t >= max(S)
    t = choose_best_t(items)
    assert t * t >= max(items) and t * t >= len(items)

    # 2. Place each key K in the square at location (x,y), where
    # x = K mod t, y = K / t.
    row_queue = place_items_in_square(items, t)

    # 3. Arrange rows so that they'll fit into one row and generate a
    # displacement vector.
    final_row, displacement_vector = arrange_rows(row_queue, t)

    # Translate the internal keys to their original items.
    slots = tuple(key_to_original[item - offset] if item is not None else None
                  for item in final_row)

    # Return the parameters
    return HashInfo(
        t=t,
        slots=slots,
        r=displacement_vector,
        offset=offset,
        to_int=to_int if to_int is not __identity else None
    )

def choose_best_t(items):
    minimum_allowable = int(math.ceil(math.sqrt(max(items))))
    if minimum_allowable ** 2 < len(items):
        return len(items)
    else:
        return minimum_allowable

def place_items_in_square(items, t):
    """
    Returns a list of rows that are stored as a priority queue to be used
    with heapq functions.

    >>> place_items_in_square([1,5,7], 4)
    [(2, 1, [(1, 5), (3, 7)]), (3, 0, [(1, 1)])]
    >>> place_items_in_square([1,5,7], 3)
    [(2, 0, [(1, 1)]), (2, 1, [(2, 5)]), (2, 2, [(1, 7)])]
    """

    # A minheap (because that's all that heapq supports :/)
    # of the length of each row. Why this is important is because
    # we'll be popping the largest rows when figuring out row displacements.
    # Each item is a tuple of (t - |row|, y, [(xpos_1, item_1), ...]).
    # Until the call to heapq.heapify(), the rows are ordered in
    # increasing row number (y).
    rows = [(t, y, []) for y in xrange(t)]

    for item in items:
        # Calculate the cell the item should fall in.
        x = item % t
        y = item / t

        # Push the item to its corresponding row...
        inverse_length, _, row_contents = rows[y]
        heapq.heappush(row_contents, (x, item))

        # Ensure the heap key is kept intact.
        rows[y] = inverse_length - 1, y, row_contents

    assert all(inv_len == t - len(rows) for inv_len, _, rows in rows)

    heapq.heapify(rows)

    # Return only rows that are populated.
    return [row for row in rows if row[2]]

def arrange_rows(row_queue, t):
    """
    Takes a priority queue as generated by place_items_in_square().
    Arranges the items from its conceptual square to one list.
    Returns both the resultant vector, plus the displacement vector, to be
    used in the final output hash function.

    >>> rows = [(2, 1, [(0, 1), (1, 5)]), (3, 3, [(1, 7)])]
    >>> result, displacements = arrange_rows(rows, 4)
    >>> result
    (1, 5, 7)
    >>> displacements
    (None, 0, None, 1)

    >>> rows = [(1, 1, [(0, 1), (2, 7)]), (2, 2, [(1, 5)])]
    >>> result, displacements = arrange_rows(rows, 3)
    >>> result
    (1, 5, 7)
    >>> displacements
    (None, 0, 0)
    """

    # Create a set of all of the unoccupied columns.
    max_columns = t ** 2
    cols = ((x, True) for x in xrange(max_columns))
    unoccupied_columns = collections.OrderedDict(cols)

    # Create the resultant and displacement vectors.
    result = [None] * max_columns
    displacements = [None] * t

    while row_queue:
        # Get the next row to place.
        _inverse_length, y, row = heapq.heappop(row_queue)

        offset = find_first_fit(unoccupied_columns, row, max_columns)
        # Calculate the offset of the first item.
        first_item_x = row[0][0]

        displacements[y] = offset
        for x, item in row:
            actual_x = x + offset
            result[actual_x] = item
            del unoccupied_columns[actual_x]

    return tuple(trim_nones_from_right(result)), tuple(displacements)

def find_first_fit(unoccupied_columns, row, row_length):
    """
    Finds the first index that the row's items can fit.

    """
    for free_col in unoccupied_columns:
        # The offset is that such that the first item goes in the free column.
        first_item_x = row[0][0]
        offset = free_col - first_item_x
        if check_columns_fit(unoccupied_columns, row, offset, row_length):
            return offset

    raise ValueError("Row cannot bossily fit in %r: %r"
                     % (unoccupied_columns.keys(), row))

def check_columns_fit(unoccupied_columns, row, offset, row_length):
    """
    Checks if all the occupied columns in the row fit in the indices given by
    free columns.

    >>> check_columns_fit({0,1,2,3}, [(0, True), (2, True)], 0, 4)
    True
    >>> check_columns_fit({0,2,3}, [(2, True), (3, True)], 0, 4)
    True
    >>> check_columns_fit({}, [(2, True), (3, True)], 0, 4)
    False
    >>> check_columns_fit({0}, [(2, True)], 2, 4)
    True
    >>> check_columns_fit({0}, [(3, True)], 2, 4)
    False

    """
    for index, item in row:
        adjusted_index = (index + offset) % row_length

        # Check if the index is in the appropriate place.
        if adjusted_index not in unoccupied_columns:
            return False

    return True

def print_square(row_queue, t):
    """
    Prints a row queue as its conceptual square array.
    """
    occupied_rows = {y: row for _, y, row in row_queue}

    empty_row = ', '.join('...' for _ in xrange(t))
    for y in xrange(t):
        print '|',
        if y not in occupied_rows:
            print empty_row,
        else:
            row = dict(occupied_rows[y])
            all_cols = ('%3d' % row[x] if x in row else '...'
                        for x in xrange(t))
            print ', '.join(all_cols),

        print "|"

def trim_nones_from_right(xs):
    """
    Returns the list without all the Nones at the right end.

    >>> trim_nones_from_right([1, 2, None, 4, None, 5, None, None])
    [1, 2, None, 4, None, 5]

    """
    # Find the first element that does not contain none.
    for i, item in enumerate(reversed(xs)):
        if item is not None:
            break

    return xs[:-i]

def make_hash(keys, **kwargs):
    """
    Creates a perfect hash function from the given keys. For a description of
    the keyword args see :py:func:`hash_parameters`.

    >>> l = (0, 3, 4, 7 ,10, 13, 15, 18, 19, 21, 22, 24, 26, 29, 30, 34)
    >>> hf = make_hash(l)
    >>> hf(19)
    1
    >>> hash_parameters(l).slots[1]
    19

    :py:func:

    """
    params = hash_parameters(keys, **kwargs)

    t = params.t
    r = params.r
    offset = params.offset
    to_int = params.to_int if params.to_int else __identity

    def perfect_hash(x):
        val = to_int(x) + offset
        x = val % t
        y = val / t
        return x + r[y]

    return perfect_hash

__all__ = ['hash_parameters', 'make_hash']

if __name__ == '__main__':
    # TODO: Make this main more useful.
    import doctest
    doctest.testmod(verbose=False)
<commit_msg>Added make_dict() to dynamically generate mutable mapping subclasses.<commit_after>#!/usr/bin/env python
# coding: utf-8

"""
Utilities for generating perfect hash functions for integer keys.

This module implements the first fit decreasing method, described in
Gettys01_. It is **not** guaranteed to generate a *minimal* perfect hash,
though by no means is it impossible. See for example:

>>> phash = hash_parameters('+-<>[].,', to_int=ord)
>>> len(phash.slots)
8
>>> phash.slots
('+', ',', '-', '.', '<', '[', '>', ']')

.. _Gettys01: http://www.drdobbs.com/architecture-and-design/generating-perfect-hash-functions/184404506

"""

import math
import collections
import heapq

__all__ = ['hash_parameters', 'make_hash', 'make_dict']

HashInfo = collections.namedtuple('HashInfo', 't slots r offset to_int')

__identity = lambda x: x

def hash_parameters(keys, minimize=True, to_int=None):
    """
    Calculates the parameters for a perfect hash. The result is returned as a
    HashInfo tuple which has the following fields:

    t
       The "table parameter". This is the minimum side length of the table
       used to create the hash. In practice, t**2 is the maximum size of the
       output hash.
    slots
       The original inputs mapped to a vector. This is the hash function.
    r
       The displacement vector. This is the displacement of the given row in
       the result vector. To find a given value, use ``x + r[y]``.
    offset
       The amount by which to offset all values (once converted to ints)
    to_int
       A function that converts the input to an int (if given).

    Keyword parameters:

    ``minimize``
        Whether or not offset all integer keys internally by the minimum
        value. This typically results in smaller output.
    ``to_int``
        A callable that converts the input keys to ints. If not specified, all
        keys should be given as ints.

    >>> hash_parameters([1, 5, 7], minimize=False)
    HashInfo(t=3, slots=(1, 5, 7), r=(-1, -1, 1), offset=0, to_int=None)

    >>> hash_parameters([1, 5, 7])
    HashInfo(t=3, slots=(1, 5, 7), r=(0, 0, 2), offset=-1, to_int=None)

    >>> l = (0, 3, 4, 7 ,10, 13, 15, 18, 19, 21, 22, 24, 26, 29, 30, 34)
    >>> phash = hash_parameters(l)
    >>> phash.slots
    (18, 19, 0, 21, 22, 3, 4, 24, 7, 26, 30, 10, 29, 13, 34, 15)
    """

    # If to_int is not assigned, simply use the identity function.
    if to_int is None:
        to_int = __identity

    key_to_original = {to_int(original): original for original in keys}

    # Create a set of all items to be hashed.
    items = key_to_original.keys()

    if minimize:
        offset = 0 - min(items)
        items = frozenset(x + offset for x in items)
    else:
        offset = 0

    # 1. Start with a square array (not stored) that is t units on each side.
    # Choose a t such that t * t >= max(S)
    t = choose_best_t(items)
    assert t * t >= max(items) and t * t >= len(items)

    # 2. Place each key K in the square at location (x,y), where
    # x = K mod t, y = K / t.
    row_queue = place_items_in_square(items, t)

    # 3. Arrange rows so that they'll fit into one row and generate a
    # displacement vector.
    final_row, displacement_vector = arrange_rows(row_queue, t)

    # Translate the internal keys to their original items.
    slots = tuple(key_to_original[item - offset] if item is not None else None
                  for item in final_row)

    # Return the parameters
    return HashInfo(
        t=t,
        slots=slots,
        r=displacement_vector,
        offset=offset,
        to_int=to_int if to_int is not __identity else None
    )

def choose_best_t(items):
    minimum_allowable = int(math.ceil(math.sqrt(max(items))))
    if minimum_allowable ** 2 < len(items):
        return len(items)
    else:
        return minimum_allowable

def place_items_in_square(items, t):
    """
    Returns a list of rows that are stored as a priority queue to be used
    with heapq functions.

    >>> place_items_in_square([1,5,7], 4)
    [(2, 1, [(1, 5), (3, 7)]), (3, 0, [(1, 1)])]
    >>> place_items_in_square([1,5,7], 3)
    [(2, 0, [(1, 1)]), (2, 1, [(2, 5)]), (2, 2, [(1, 7)])]
    """

    # A minheap (because that's all that heapq supports :/)
    # of the length of each row. Why this is important is because
    # we'll be popping the largest rows when figuring out row displacements.
    # Each item is a tuple of (t - |row|, y, [(xpos_1, item_1), ...]).
    # Until the call to heapq.heapify(), the rows are ordered in
    # increasing row number (y).
    rows = [(t, y, []) for y in xrange(t)]

    for item in items:
        # Calculate the cell the item should fall in.
        x = item % t
        y = item // t

        # Push the item to its corresponding row...
        inverse_length, _, row_contents = rows[y]
        heapq.heappush(row_contents, (x, item))

        # Ensure the heap key is kept intact.
        rows[y] = inverse_length - 1, y, row_contents

    assert all(inv_len == t - len(rows) for inv_len, _, rows in rows)

    heapq.heapify(rows)

    # Return only rows that are populated.
    return [row for row in rows if row[2]]

def arrange_rows(row_queue, t):
    """
    Takes a priority queue as generated by place_items_in_square().
    Arranges the items from its conceptual square to one list.
    Returns both the resultant vector, plus the displacement vector, to be
    used in the final output hash function.

    >>> rows = [(2, 1, [(0, 1), (1, 5)]), (3, 3, [(1, 7)])]
    >>> result, displacements = arrange_rows(rows, 4)
    >>> result
    (1, 5, 7)
    >>> displacements
    (None, 0, None, 1)

    >>> rows = [(1, 1, [(0, 1), (2, 7)]), (2, 2, [(1, 5)])]
    >>> result, displacements = arrange_rows(rows, 3)
    >>> result
    (1, 5, 7)
    >>> displacements
    (None, 0, 0)
    """

    # Create a set of all of the unoccupied columns.
    max_columns = t ** 2
    cols = ((x, True) for x in xrange(max_columns))
    unoccupied_columns = collections.OrderedDict(cols)

    # Create the resultant and displacement vectors.
    result = [None] * max_columns
    displacements = [None] * t

    while row_queue:
        # Get the next row to place.
        _inverse_length, y, row = heapq.heappop(row_queue)

        offset = find_first_fit(unoccupied_columns, row, max_columns)
        # Calculate the offset of the first item.
        first_item_x = row[0][0]

        displacements[y] = offset
        for x, item in row:
            actual_x = x + offset
            result[actual_x] = item
            del unoccupied_columns[actual_x]

    return tuple(trim_nones_from_right(result)), tuple(displacements)

def find_first_fit(unoccupied_columns, row, row_length):
    """
    Finds the first index that the row's items can fit.

    """
    for free_col in unoccupied_columns:
        # The offset is that such that the first item goes in the free column.
        first_item_x = row[0][0]
        offset = free_col - first_item_x
        if check_columns_fit(unoccupied_columns, row, offset, row_length):
            return offset

    raise ValueError("Row cannot bossily fit in %r: %r"
                     % (unoccupied_columns.keys(), row))

def check_columns_fit(unoccupied_columns, row, offset, row_length):
    """
    Checks if all the occupied columns in the row fit in the indices given by
    free columns.

    >>> check_columns_fit({0,1,2,3}, [(0, True), (2, True)], 0, 4)
    True
    >>> check_columns_fit({0,2,3}, [(2, True), (3, True)], 0, 4)
    True
    >>> check_columns_fit({}, [(2, True), (3, True)], 0, 4)
    False
    >>> check_columns_fit({0}, [(2, True)], 2, 4)
    True
    >>> check_columns_fit({0}, [(3, True)], 2, 4)
    False

    """
    for index, item in row:
        adjusted_index = (index + offset) % row_length

        # Check if the index is in the appropriate place.
        if adjusted_index not in unoccupied_columns:
            return False

    return True

def print_square(row_queue, t):
    """
    Prints a row queue as its conceptual square array.
    """
    occupied_rows = {y: row for _, y, row in row_queue}

    empty_row = ', '.join('...' for _ in xrange(t))
    for y in xrange(t):
        print '|',
        if y not in occupied_rows:
            print empty_row,
        else:
            row = dict(occupied_rows[y])
            all_cols = ('%3d' % row[x] if x in row else '...'
                        for x in xrange(t))
            print ', '.join(all_cols),

        print "|"

def trim_nones_from_right(xs):
    """
    Returns the list without all the Nones at the right end.

    >>> trim_nones_from_right([1, 2, None, 4, None, 5, None, None])
    [1, 2, None, 4, None, 5]

    """
    # Find the first element that does not contain none.
    for i, item in enumerate(reversed(xs)):
        if item is not None:
            break

    return xs[:-i]

def make_hash(keys, **kwargs):
    """
    Creates a perfect hash function from the given keys. For a description of
    the keyword arguments see :py:func:`hash_parameters`.

    >>> l = (0, 3, 4, 7 ,10, 13, 15, 18, 19, 21, 22, 24, 26, 29, 30, 34)
    >>> hf = make_hash(l)
    >>> hf(19)
    1
    >>> hash_parameters(l).slots[1]
    19
    """
    params = hash_parameters(keys, **kwargs)

    t = params.t
    r = params.r
    offset = params.offset
    to_int = params.to_int if params.to_int else __identity

    def perfect_hash(x):
        val = to_int(x) + offset
        x = val % t
        y = val / t
        return x + r[y]

    # Undocumented properties, but used in make_dict()...
    perfect_hash.length = len(params.slots)
    perfect_hash.slots = params.slots

    return perfect_hash

def make_dict(name, keys, **kwargs):
    """
    Creates a dictionary-like mapping class that uses perfect hashing.
    ``name`` is the proper class name of the returned class. See
    ``hash_parameters()`` for documentation on all arguments after ``name``.

    >>> MyDict = make_dict('MyDict', '+-<>[],.', to_int=ord)
    >>> d = MyDict([('+', 1), ('-', 2)])
    >>> d[','] = 3
    >>> d
    MyDict([('+', 1), (',', 3), ('-', 2)])
    >>> del d['+']
    >>> del d['.']
    Traceback (most recent call last):
    ...
    KeyError: '.'
    >>> len(d)
    2
    """

    hash_func = make_hash(keys, **kwargs)

    # Returns array index or ra
    def index_or_key_error(key):
        index = hash_func(key)
        # Make sure the key is **exactly** the same.
        if key != hash_func.slots[index]:
            raise KeyError(key)
        return index

    def init(self, *args, **kwargs):
        self._arr = [None] * hash_func.length
        self._len = 0

        # Delegate iniaitlization to update provided by MutableMapping:
        self.update(*args, **kwargs)

    def getitem(self, key):
        index = index_or_key_error(key)
        if self._arr[index] is None:
            raise KeyError(key)
        return self._arr[index][1]

    def setitem(self, key, value):
        index = index_or_key_error(key)
        self._arr[index] = (key, value)

    def delitem(self, key):
        index = index_or_key_error(key)
        if self._arr[index] is None:
            raise KeyError(key)
        self._arr[index] = None

    def dict_iter(self):
        return (pair[0] for pair in self._arr if pair is not None)

    def dict_len(self):
        # TODO: Make this O(1) using auxillary state?
        return sum(1 for _ in self)

    def dict_repr(self):
        arr_repr = (repr(pair) for pair in self._arr if pair is not None)
        return ''.join((name, '([', ', '.join(arr_repr), '])'))

    # Create a docstring that at least describes where the class came from...
    doc = """
        Dictionary-like object that uses perfect hashing. This class was
        generated by `%s.%s(%r, ...)`.
        """ % (__name__, make_dict.__name__, name)

    # Inheriting from MutableMapping gives us a whole whackload of methods for
    # free.
    bases = (collections.MutableMapping,)

    return type(name, bases, {
        '__init__': init,
        '__doc__': doc,

        '__getitem__': getitem,
        '__setitem__': setitem,
        '__delitem__': delitem,
        '__iter__': dict_iter,
        '__len__': dict_len,

        '__repr__': dict_repr,
    })

if __name__ == '__main__':
    # TODO: Make this main more useful.
    import doctest
    doctest.testmod(verbose=False)
<|endoftext|>


<commit_before>from functools import wraps

from messente.verigator import routes, models, client

def _validate_input(func):
    # decorator for validating that passed arguments are all string
    @wraps(func)
    def wrapper(*args):
        for arg in args[1:]:
            if not isinstance(arg, basestring):
                raise ValueError("{} should be string".format(arg))
        return func(*args)

    return wrapper

def _validate_client(func):
    # decorator for validating that passed client is RestClient
    @wraps(func)
    def wrapper(self, rest_client):
        if not isinstance(rest_client, client.RestClient):
            raise ValueError("client should be RestClient")
        return func(self, rest_client)

    return wrapper

class Services(object):
    """
    Controller for service resource
    """

    @_validate_client
    def __init__(self, rest_client):
        """

        Args:
            rest_client (client.RestClient):
        """
        self.rest_client = rest_client

    @_validate_input
    def create(self, domain, name):
        """Creates new service.

        Args:
            domain (str): The domain name.
            name (str): The name of the service.

        Returns:
            models.Service: created service

        """

        response = self.rest_client.post(routes.CREATE_SERVICE, json={
            'fqdn': domain,
            'name': name
        })
        return self.__service_from_json(response)

    @_validate_input
    def get(self, id):
        """Fetches service with a given id from the server.

        Args:
            id (str): The id of the service

        Returns:
            models.Service: Fetched service

        """
        response = self.rest_client.get(routes.GET_SERVICE.format(id))

        return self.__service_from_json(response)

    @_validate_input
    def delete(self, id):
        """Deletes service with id

        Args:
            id (str): service id

        Returns:
            bool:

        """
        self.rest_client.delete(routes.DELETE_SERVICE.format(id))
        return True

    @staticmethod
    def __service_from_json(json):
        return models.Service(json['id'], json['ctime'], json['name'])

# noinspection PyShadowingBuiltins
class Users(object):
    """Controller for service resource
    """

    @_validate_client
    def __init__(self, rest_client):
        """

        Args:
            rest_client (client.RestClient):
        """
        self.rest_client = rest_client

    @_validate_input
    def get_all(self, service_id):
        """Fetches all users for the given service

        Args:
            service_id (str): service id to search users for

        Returns:
            list[models.User]: list of users

        """
        response = self.rest_client.get(routes.GET_USERS.format(service_id))

        return [self.__user_from_json(user) for user in response['users']]

    @_validate_input
    def get(self, service_id, id):
        """Fetches single user with given id for the given service

        Args:
            service_id (str): service id
            id (str): user id

        Returns:
            models.User: fetched user

        """
        response = self.rest_client.get(routes.GET_USER.format(service_id, id))
        return self.__user_from_json(response)

    @_validate_input
    def create(self, service_id, number, username):
        """Creates new user for the given service

        Args:
            service_id (str): service id
            number (str): users phone number, used for 2fa
            username (str): username

        Returns:
            models.User: created user

        """

        response = self.rest_client.post(routes.CREATE_USER.format(service_id), json={
            "phone_number": number,
            "id_in_service": username
        })
        return self.__user_from_json(response)

    @_validate_input
    def delete(self, service_id, id):
        """Deleted user with given id for the given service

        Args:
            service_id (str): service id
            id (str): user id

        Returns:
            bool: True on success raises exception on error

        """
        self.rest_client.delete(routes.DELETE_USER.format(service_id, id))

        return True

    @staticmethod
    def __user_from_json(json):
        return models.User(json['id'], json['ctime'], json['id_in_service'])

class Auth(object):
    """Controller for service resource

    """

    METHOD_SMS = "sms"
    METHOD_TOTP = "totp"

    @_validate_client
    def __init__(self, rest_client):
        """

        Args:
            rest_client (client.RestClient):
        """
        self.rest_client = rest_client

    @_validate_input
    def initiate(self, service_id, user_id, method):
        """Initiates authentication process
        sends sms in case of sms authentication

        Args:
            service_id (str): service id
            user_id (str): user id
            method (str): auth method (sms or totp) use Auth.METHOD_SMS or Auth.METHOD_TOTP

        Note:
            System will automatically fall back from TOTP to SMS if user has no devices attached to the number

        Returns:
            str: auth_id if sms auth else None, raises exception on error

        """
        response = self.rest_client.post(routes.AUTH_INITIATE.format(service_id, user_id), json={
            "method": method
        })

        if response['method'] == "sms":
            return response['auth_id']

    @_validate_input
    def verify(self, service_id, user_id, method, token, auth_id=None):
        """Verifies user input validity

        Args:
            service_id (str): service id
            user_id (str): user id
            method (str): auth method (sms or totp) use Auth.METHOD_SMS or Auth.METHOD_TOTP
            token (str): user provided token
            auth_id (str): in case of sms auth, auth_id you got when initiated authentication

        Returns:
            (bool, dict): boolean indicating verification status and error dict in case verification fails

        """
        json = {"method": method, "token": token}

        if auth_id:
            json["auth_id"] = auth_id

        response = self.rest_client.put(routes.AUTH_VERIFY.format(service_id, user_id), json=json)

        verified = response['verified']
        error = response.get('status', None)

        return verified, error
<commit_msg>Fix Python 2 compatibility<commit_after>import sys
from functools import wraps

from messente.verigator import routes, models, client

PY2 = sys.version_info.major == 2

def _validate_input(func):
    # decorator for validating that passed arguments are all string
    @wraps(func)
    def wrapper(*args):
        for arg in args[1:]:
            if not isinstance(arg, str if not PY2 else basestring):
                raise ValueError("{} should be string".format(arg))
        return func(*args)

    return wrapper

def _validate_client(func):
    # decorator for validating that passed client is RestClient
    @wraps(func)
    def wrapper(self, rest_client):
        if not isinstance(rest_client, client.RestClient):
            raise ValueError("client should be RestClient")
        return func(self, rest_client)

    return wrapper

class Services(object):
    """
    Controller for service resource
    """

    @_validate_client
    def __init__(self, rest_client):
        """

        Args:
            rest_client (client.RestClient):
        """
        self.rest_client = rest_client

    @_validate_input
    def create(self, domain, name):
        """Creates new service.

        Args:
            domain (str): The domain name.
            name (str): The name of the service.

        Returns:
            models.Service: created service

        """

        response = self.rest_client.post(routes.CREATE_SERVICE, json={
            'fqdn': domain,
            'name': name
        })
        return self.__service_from_json(response)

    @_validate_input
    def get(self, id):
        """Fetches service with a given id from the server.

        Args:
            id (str): The id of the service

        Returns:
            models.Service: Fetched service

        """
        response = self.rest_client.get(routes.GET_SERVICE.format(id))

        return self.__service_from_json(response)

    @_validate_input
    def delete(self, id):
        """Deletes service with id

        Args:
            id (str): service id

        Returns:
            bool:

        """
        self.rest_client.delete(routes.DELETE_SERVICE.format(id))
        return True

    @staticmethod
    def __service_from_json(json):
        return models.Service(json['id'], json['ctime'], json['name'])

# noinspection PyShadowingBuiltins
class Users(object):
    """Controller for service resource
    """

    @_validate_client
    def __init__(self, rest_client):
        """

        Args:
            rest_client (client.RestClient):
        """
        self.rest_client = rest_client

    @_validate_input
    def get_all(self, service_id):
        """Fetches all users for the given service

        Args:
            service_id (str): service id to search users for

        Returns:
            list[models.User]: list of users

        """
        response = self.rest_client.get(routes.GET_USERS.format(service_id))

        return [self.__user_from_json(user) for user in response['users']]

    @_validate_input
    def get(self, service_id, id):
        """Fetches single user with given id for the given service

        Args:
            service_id (str): service id
            id (str): user id

        Returns:
            models.User: fetched user

        """
        response = self.rest_client.get(routes.GET_USER.format(service_id, id))
        return self.__user_from_json(response)

    @_validate_input
    def create(self, service_id, number, username):
        """Creates new user for the given service

        Args:
            service_id (str): service id
            number (str): users phone number, used for 2fa
            username (str): username

        Returns:
            models.User: created user

        """

        response = self.rest_client.post(routes.CREATE_USER.format(service_id), json={
            "phone_number": number,
            "id_in_service": username
        })
        return self.__user_from_json(response)

    @_validate_input
    def delete(self, service_id, id):
        """Deleted user with given id for the given service

        Args:
            service_id (str): service id
            id (str): user id

        Returns:
            bool: True on success raises exception on error

        """
        self.rest_client.delete(routes.DELETE_USER.format(service_id, id))

        return True

    @staticmethod
    def __user_from_json(json):
        return models.User(json['id'], json['ctime'], json['id_in_service'])

class Auth(object):
    """Controller for service resource

    """

    METHOD_SMS = "sms"
    METHOD_TOTP = "totp"

    @_validate_client
    def __init__(self, rest_client):
        """

        Args:
            rest_client (client.RestClient):
        """
        self.rest_client = rest_client

    @_validate_input
    def initiate(self, service_id, user_id, method):
        """Initiates authentication process
        sends sms in case of sms authentication

        Args:
            service_id (str): service id
            user_id (str): user id
            method (str): auth method (sms or totp) use Auth.METHOD_SMS or Auth.METHOD_TOTP

        Note:
            System will automatically fall back from TOTP to SMS if user has no devices attached to the number

        Returns:
            str: auth_id if sms auth else None, raises exception on error

        """
        response = self.rest_client.post(routes.AUTH_INITIATE.format(service_id, user_id), json={
            "method": method
        })

        if response['method'] == "sms":
            return response['auth_id']

    @_validate_input
    def verify(self, service_id, user_id, method, token, auth_id=None):
        """Verifies user input validity

        Args:
            service_id (str): service id
            user_id (str): user id
            method (str): auth method (sms or totp) use Auth.METHOD_SMS or Auth.METHOD_TOTP
            token (str): user provided token
            auth_id (str): in case of sms auth, auth_id you got when initiated authentication

        Returns:
            (bool, dict): boolean indicating verification status and error dict in case verification fails

        """
        json = {"method": method, "token": token}

        if auth_id:
            json["auth_id"] = auth_id

        response = self.rest_client.put(routes.AUTH_VERIFY.format(service_id, user_id), json=json)

        verified = response['verified']
        error = response.get('status', None)

        return verified, error
<|endoftext|>


<commit_before>"""A hook for modifying parameter values read from the WMT client."""

import os
from wmt.models.models import get_model_upload_dir
from wmt.utils.hook import yaml_dump
from permafrost_benchmark_system.file import (IlambConfigFile,
                                              get_region_labels_txt,
                                              get_region_labels_ncdf)

gfed_region_names = ['global', 'bona', 'tena', 'ceam', 'nhsa', 'shsa',
                     'euro', 'mide', 'nhaf', 'shaf', 'boas', 'ceas',
                     'seas', 'eqas', 'aust']

def load_custom_regions(regions_file):
    """Get a list of custom region labels from a file.

    Parameters
    ----------
    regions_file : str
        The path to an ILAMB custom regions file.

    Returns
    -------
    list
        A list of custom region names.

    """
    try:
        return get_region_labels_ncdf(regions_file)
    except:
        return get_region_labels_txt(regions_file)
    else:
        return []

def execute(env):
    """Perform pre-stage tasks for running a component.

    Parameters
    ----------
    env : dict
      A dict of component parameter values from WMT.

    """
    var_list = []
    for k, v in env.items():
        if k.startswith('_variable') and v != '':
            var_list.append(str(v))

    has_relationships = (env['_relationships'] == 'Yes') and \
                        (len(var_list) > 1)

    f = IlambConfigFile(var_list,
                        relationships=has_relationships,
                        title=env['simulation_name'])
    f.setup()
    f.write()

    model_list = []
    for k, v in env.items():
        if k.startswith('_model_') and v == 'On':
            model_list.append(k.lstrip('_model_').encode('utf-8'))
    env['models'] = model_list

    regions = []
    for r in gfed_region_names:
        if env['_region_' + r] == 'On':
            regions.append(r)
    env['regions'] = regions

    if env['_define_regions_file'] != 'Off':
        env['define_regions'] = str(env['_define_regions_file'])
        upload_dir = get_model_upload_dir(env['_model_id'])
        regions_file = os.path.join(upload_dir, env['define_regions'])
        custom_regions = load_custom_regions(regions_file)
        env['regions'].extend(custom_regions)

    # For debugging.
    env['_sources_file'] = f.sources_file
    env['_var_list'] = var_list
    env['_has_relationships'] = has_relationships
    yaml_dump('_env.yaml', env)
<commit_msg>Make sure to copy the custom regions file from the uploads directory<commit_after>"""A hook for modifying parameter values read from the WMT client."""

import os
import shutil
from wmt.models.models import get_model_upload_dir
from wmt.utils.hook import yaml_dump
from permafrost_benchmark_system.file import (IlambConfigFile,
                                              get_region_labels_txt,
                                              get_region_labels_ncdf)

gfed_region_names = ['global', 'bona', 'tena', 'ceam', 'nhsa', 'shsa',
                     'euro', 'mide', 'nhaf', 'shaf', 'boas', 'ceas',
                     'seas', 'eqas', 'aust']

def load_custom_regions(regions_file):
    """Get a list of custom region labels from a file.

    Parameters
    ----------
    regions_file : str
        The path to an ILAMB custom regions file.

    Returns
    -------
    list
        A list of custom region names.

    """
    try:
        return get_region_labels_ncdf(regions_file)
    except:
        return get_region_labels_txt(regions_file)
    else:
        return []

def execute(env):
    """Perform pre-stage tasks for running a component.

    Parameters
    ----------
    env : dict
      A dict of component parameter values from WMT.

    """
    var_list = []
    for k, v in env.items():
        if k.startswith('_variable') and v != '':
            var_list.append(str(v))

    has_relationships = (env['_relationships'] == 'Yes') and \
                        (len(var_list) > 1)

    f = IlambConfigFile(var_list,
                        relationships=has_relationships,
                        title=env['simulation_name'])
    f.setup()
    f.write()

    model_list = []
    for k, v in env.items():
        if k.startswith('_model_') and v == 'On':
            model_list.append(k.lstrip('_model_').encode('utf-8'))
    env['models'] = model_list

    regions = []
    for r in gfed_region_names:
        if env['_region_' + r] == 'On':
            regions.append(r)
    env['regions'] = regions

    if env['_define_regions_file'] != 'Off':
        env['define_regions'] = str(env['_define_regions_file'])
        upload_dir = get_model_upload_dir(env['_model_id'])
        regions_file = os.path.join(upload_dir, env['define_regions'])
        shutil.copy(regions_file, os.getcwd())
        custom_regions = load_custom_regions(env['define_regions'])
        env['regions'].extend(custom_regions)

    # For debugging.
    env['_sources_file'] = f.sources_file
    env['_var_list'] = var_list
    env['_has_relationships'] = has_relationships
    yaml_dump('_env.yaml', env)
<|endoftext|>


<commit_before># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Colab-specific file helpers."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import base64
import collections
import os
import socket
import threading
import uuid

import IPython
import portpicker
import six
from six.moves import SimpleHTTPServer
from six.moves import socketserver

from google.colab import output

def upload():
  """Renders widget to upload local (to the browser) files to the kernel.

  Blocks until the files are available.

  Returns:
    A map of the form {<filename>: <file contents>} for all uploaded files.
  """
  upload_id = str(uuid.uuid4())
  input_id = 'files-' + upload_id
  output_id = 'result-' + upload_id

  IPython.display.display(
      IPython.core.display.HTML("""
     <input type="file" id="{input_id}" name="files[]" multiple disabled />
     <output id="{output_id}">
      Upload widget is only available when the cell has been executed in the
      current browser session. Please rerun this cell to enable.
      </output>
      <script src="/nbextensions/google.colab/files.js"</script> """.format(
          input_id=input_id, output_id=output_id)))

  # First result is always an indication that the file picker has completed.
  result = output.eval_js(
      'google.colab._files._uploadFiles("{input_id}", "{output_id}")'.format(
          input_id=input_id, output_id=output_id))
  files = collections.defaultdict(six.binary_type)

  while result['action'] != 'complete':
    result = output.eval_js(
        'google.colab._files._uploadFilesContinue("{output_id}")'.format(
            output_id=output_id))
    if result['action'] != 'append':
      # JS side uses a generator of promises to process all of the files- some
      # steps may not produce data for the Python side, so just proceed onto the
      # next message.
      continue
    files[result['file']] += base64.b64decode(result['data'])

  return dict(files)

class _V6Server(socketserver.TCPServer):
  address_family = socket.AF_INET6

class _FileHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  """SimpleHTTPRequestHandler with a couple tweaks."""

  def translate_path(self, path):
    # Client specifies absolute paths.
    return path

  def log_message(self, fmt, *args):
    # Suppress logging since it's on the background. Any errors will be reported
    # via the handler.
    pass

  def end_headers(self):
    # Do not cache the response in the notebook, since it may be quite large.
    self.send_header('x-colab-notebook-cache-control', 'no-cache')
    SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self)

def download(filename):
  """Downloads the file to the user's local disk via a browser download action.

  Args:
    filename: Name of the file on disk to be downloaded.
  """

  started = threading.Event()
  port = portpicker.pick_unused_port()

  def server_entry():
    httpd = _V6Server(('::', port), _FileHandler)
    started.set()
    # Handle a single request then exit the thread.
    httpd.handle_request()

  thread = threading.Thread(target=server_entry)
  thread.start()
  started.wait()

  output.eval_js(
      """
      (async function() {
        const response = await fetch('https://localhost:%(port)d%(path)s');
        if (!response.ok) {
          throw new Error('Failed to download: ' + response.statusText);
        }
        const blob = await response.blob();

        const a = document.createElement('a');
        a.href = window.URL.createObjectURL(blob);
        a.download = '%(name)s';
        a.click();
      })();
  """ % {
      'port': port,
      'path': os.path.abspath(filename),
      'name': os.path.basename(filename),
  })
<commit_msg>Internal update.<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Colab-specific file helpers."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import base64
import collections
import os
import socket
import threading
import uuid

import IPython
import portpicker
import six
from six.moves import SimpleHTTPServer
from six.moves import socketserver

from google.colab import output

def upload():
  """Renders widget to upload local (to the browser) files to the kernel.

  Blocks until the files are available.

  Returns:
    A map of the form {<filename>: <file contents>} for all uploaded files.
  """
  upload_id = str(uuid.uuid4())
  input_id = 'files-' + upload_id
  output_id = 'result-' + upload_id

  IPython.display.display(
      IPython.core.display.HTML("""
     <input type="file" id="{input_id}" name="files[]" multiple disabled />
     <output id="{output_id}">
      Upload widget is only available when the cell has been executed in the
      current browser session. Please rerun this cell to enable.
      </output>
      <script src="/nbextensions/google.colab/files.js"</script> """.format(
          input_id=input_id, output_id=output_id)))

  # First result is always an indication that the file picker has completed.
  result = output.eval_js(
      'google.colab._files._uploadFiles("{input_id}", "{output_id}")'.format(
          input_id=input_id, output_id=output_id))
  files = collections.defaultdict(six.binary_type)
  # Mapping from original filename to filename as saved locally.
  local_filenames = dict()

  while result['action'] != 'complete':
    result = output.eval_js(
        'google.colab._files._uploadFilesContinue("{output_id}")'.format(
            output_id=output_id))
    if result['action'] != 'append':
      # JS side uses a generator of promises to process all of the files- some
      # steps may not produce data for the Python side, so just proceed onto the
      # next message.
      continue
    data = base64.b64decode(result['data'])
    filename = result['file']

    files[filename] += data
    local_filename = local_filenames.get(filename)
    if not local_filename:
      local_filename = _get_unique_filename(filename)
      local_filenames[filename] = local_filename
      print('Saving {filename} to {local_filename}'.format(
          filename=filename, local_filename=local_filename))
    with open(local_filename, 'ab') as f:
      f.write(data)

  return dict(files)

def _get_unique_filename(filename):
  if not os.path.lexists(filename):
    return filename
  counter = 1
  while True:
    path, ext = os.path.splitext(filename)
    new_filename = '{} ({}){}'.format(path, counter, ext)
    if not os.path.lexists(new_filename):
      return new_filename
    counter += 1

class _V6Server(socketserver.TCPServer):
  address_family = socket.AF_INET6

class _FileHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  """SimpleHTTPRequestHandler with a couple tweaks."""

  def translate_path(self, path):
    # Client specifies absolute paths.
    return path

  def log_message(self, fmt, *args):
    # Suppress logging since it's on the background. Any errors will be reported
    # via the handler.
    pass

  def end_headers(self):
    # Do not cache the response in the notebook, since it may be quite large.
    self.send_header('x-colab-notebook-cache-control', 'no-cache')
    SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self)

def download(filename):
  """Downloads the file to the user's local disk via a browser download action.

  Args:
    filename: Name of the file on disk to be downloaded.
  """

  started = threading.Event()
  port = portpicker.pick_unused_port()

  def server_entry():
    httpd = _V6Server(('::', port), _FileHandler)
    started.set()
    # Handle a single request then exit the thread.
    httpd.handle_request()

  thread = threading.Thread(target=server_entry)
  thread.start()
  started.wait()

  output.eval_js(
      """
      (async function() {
        const response = await fetch('https://localhost:%(port)d%(path)s');
        if (!response.ok) {
          throw new Error('Failed to download: ' + response.statusText);
        }
        const blob = await response.blob();

        const a = document.createElement('a');
        a.href = window.URL.createObjectURL(blob);
        a.download = '%(name)s';
        a.click();
      })();
  """ % {
      'port': port,
      'path': os.path.abspath(filename),
      'name': os.path.basename(filename),
  })
<|endoftext|>