<commit_before>#!/usr/bin/python
# Copyright (c) 2011 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import pyauto_nacl  # Must be imported before pyauto
import pyauto
import nacl_utils
import random

class NaClTest(pyauto.PyUITest):
  """Tests for NaCl."""
  nexes = [# TODO(cstefansen): Figure out why multiple nexes in a page flakes
           # on Linux 32 bit (issue 1851); then uncomment the following line
           # (and similar line below)
           # 'basic_object.html',
           'earth_c.html',
           'earth_cc.html',
           'ppapi_core.html',
           'ppapi_example_audio.html#mute',
           'ppapi_file_system.html',
           'ppapi_geturl.html',
           'ppapi_progress_events.html',
           # TODO(nfullagar): graphics 2d appears to lock up while waiting
           # for the replace contents flush callback. Enable this test when
           # we've figured out why this occurs and the problem is fixed.
           # 'ppapi_ppb_graphics2d.html',
           'srpc_basic.html',
           'srpc_hw.html',
           # TODO(nfullagar): enable this test when it works on trybots.
           # 'srpc_hw_fd.html',
           # TODO(cstefansen): Figure out why multiple nexes in a page flakes
           # on Linux 32 bit (issue 1851); then uncomment the following line.
           # 'srpc_nrd_xfer.html',
           'srpc_plugin.html',
           'srpc_shm.html',
           'srpc_sockaddr.html',
           'srpc_url_as_nacl_desc.html']

  def testLoadNexesInMultipleTabs(self):
    """Load nexes in multiple tabs and surf away from all of them."""

    # Prime each tab by navigating to about:version.
    # TODO(mcgrathr): Reduced from 10 to 5 because 256MB*10 is too
    # much /dev/shm space for the bots to handle.
    # See http://code.google.com/p/nativeclient/issues/detail?id=503
    num_tabs = 5
    num_iterations = 10
    self.NavigateToURL('about:version')
    original_title = self.GetActiveTabTitle()
    for i in range(1, num_tabs):
      self.AppendTab(pyauto.GURL('about:version'))

    for j in range(0, num_iterations):
      # Pick a nexe for each tab and navigate to it.
      for i in range(0, num_tabs):
        page_url = random.choice(NaClTest.nexes)
        self.GetBrowserWindow(0).GetTab(i).NavigateToURL(pyauto.GURL(
            self.GetHttpURLForDataPath(page_url)))

      # Wait for all the nexes to fully load.
      for i in range(0, num_tabs):
        nacl_utils.WaitForNexeLoad(self, tab_index=i)

      # Make sure every nexe successfully passed test(s).
      for i in range(0, num_tabs):
        nacl_utils.VerifyAllTestsPassed(self, tab_index=i)

      # Surf away from each nexe and verify that the tab didn't crash.
      for i in range(0, num_tabs):
        self.GetBrowserWindow(0).GetTab(i).GoBack()
        self.assertEqual(original_title, self.GetActiveTabTitle())

  def testLoadMultipleNexesInOneTab(self):
    """Load multiple nexes in one tab and load them one after another."""

    # Prime a tab by navigating to about:version.
    self.NavigateToURL('about:version')
    original_title = self.GetActiveTabTitle()

    # Navigate to a nexe and make sure it loads. Repeate for all nexes.
    for page_url in NaClTest.nexes:
      self.NavigateToURL(self.GetHttpURLForDataPath(page_url))
      nacl_utils.WaitForNexeLoad(self)
      nacl_utils.VerifyAllTestsPassed(self)

    # Keep hitting the back button and make sure all the nexes load.
    for i in range(0, len(NaClTest.nexes) - 1):
      self.GetBrowserWindow(0).GetTab(0).GoBack()
      nacl_utils.WaitForNexeLoad(self)
      nacl_utils.VerifyAllTestsPassed(self)

    # Go back one last time and make sure we ended up where we started.
    self.GetBrowserWindow(0).GetTab(0).GoBack()
    self.assertEqual(original_title, self.GetActiveTabTitle())

if __name__ == '__main__':
  pyauto_nacl.Main()
<commit_msg>limit the number of nexes when running multi-tab re-enable the tests that have multiple nexes per page BUG=code red TEST=this is the test Review URL: http://codereview.chromium.org/6987027<commit_after>#!/usr/bin/python
# Copyright (c) 2011 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import pyauto_nacl  # Must be imported before pyauto
import pyauto
import nacl_utils
import random

class NaClTest(pyauto.PyUITest):
  """Tests for NaCl."""
  # (test.html, number of nexes loaded concurrently on page)
  nexes = [('basic_object.html', 2),
           ('earth_c.html', 1),
           ('earth_cc.html', 1),
           ('ppapi_core.html', 1),
           ('ppapi_example_audio.html#mute', 1),
           ('ppapi_file_system.html', 1),
           ('ppapi_geturl.html', 1),
           ('ppapi_progress_events.html', 1),
           # TODO(nfullagar): graphics 2d appears to lock up while waiting
           # for the replace contents flush callback. Enable this test when
           # we've figured out why this occurs and the problem is fixed.
           # ('ppapi_ppb_graphics2d.html', 1),
           ('srpc_basic.html', 1),
           ('srpc_hw.html', 1),
           # TODO(nfullagar): enable this test when it works on trybots.
           # ('srpc_hw_fd.html', 1),
           ('srpc_nrd_xfer.html', 2),
           ('srpc_plugin.html', 1),
           ('srpc_shm.html', 1),
           ('srpc_sockaddr.html', 1),
           ('srpc_url_as_nacl_desc.html', 1)]

  def testLoadNexesInMultipleTabs(self):
    """Load nexes in multiple tabs and surf away from all of them."""

    # Prime each tab by navigating to about:version.
    # TODO(mcgrathr): Reduced from 10 to 6 because 256MB*10 is too
    # much /dev/shm space for the bots to handle.
    # See http://code.google.com/p/nativeclient/issues/detail?id=503
    max_nexes = 6
    max_tabs = 6
    max_attempts = max_nexes * 10
    num_iterations = 10
    self.NavigateToURL('about:version')
    original_title = self.GetActiveTabTitle()
    for i in range(1, max_tabs):
      self.AppendTab(pyauto.GURL('about:version'))

    for j in range(0, num_iterations):
      # Pick a test for each tab and navigate to it.
      # Some tests have more than one nexe on the page, limit the total to
      # max_nexes, due to mcgrathr's /dev/shm note above.
      num_nexes = 0
      num_tabs = 0
      num_attempts = 0
      while (num_nexes < max_nexes) and (num_attempts < max_attempts):
        page_url, page_nexes = random.choice(NaClTest.nexes)
        if num_nexes + page_nexes <= max_nexes:
          self.GetBrowserWindow(0).GetTab(num_tabs).NavigateToURL(pyauto.GURL(
            self.GetHttpURLForDataPath(page_url)))
          num_nexes = num_nexes + page_nexes
          num_tabs = num_tabs + 1
        num_attempts = num_attempts + 1

      # Wait for all the tabs to fully load.
      for i in range(0, num_tabs):
        nacl_utils.WaitForNexeLoad(self, tab_index=i)

      # Make sure every tab successfully passed test(s).
      for i in range(0, num_tabs):
        nacl_utils.VerifyAllTestsPassed(self, tab_index=i)

      # Surf away from each tab and verify no crash occurred.
      for i in range(0, num_tabs):
        self.GetBrowserWindow(0).GetTab(i).GoBack()
        self.assertEqual(original_title, self.GetActiveTabTitle())

  def testLoadMultipleNexesInOneTab(self):
    """Load multiple nexes in one tab and load them one after another."""

    # Prime a tab by navigating to about:version.
    self.NavigateToURL('about:version')
    original_title = self.GetActiveTabTitle()

    # Navigate to a nexe and make sure it loads. Repeate for all nexes.
    for page_info in NaClTest.nexes:
      page_url, page_nexes = page_info
      self.NavigateToURL(self.GetHttpURLForDataPath(page_url))
      nacl_utils.WaitForNexeLoad(self)
      nacl_utils.VerifyAllTestsPassed(self)

    # Keep hitting the back button and make sure all the nexes load.
    for i in range(0, len(NaClTest.nexes) - 1):
      self.GetBrowserWindow(0).GetTab(0).GoBack()
      nacl_utils.WaitForNexeLoad(self)
      nacl_utils.VerifyAllTestsPassed(self)

    # Go back one last time and make sure we ended up where we started.
    self.GetBrowserWindow(0).GetTab(0).GoBack()
    self.assertEqual(original_title, self.GetActiveTabTitle())

if __name__ == '__main__':
  pyauto_nacl.Main()
<|endoftext|>


<commit_before>#!/usr/bin/python

import plankton
import unittest

@plankton.serializable()
class Pair(object):

  @plankton.field("first")
  @plankton.field("second")
  def __init__(self, first, second):
    self.first = first
    self.second = second

  def __eq__(self, that):
    return (self is that) or (self.first == that.first) and (self.second == that.second)

  def __str__(self):
    return "(%s, %s)" % (self.first, self.second)

@plankton.serializable()
class Key(object):

  @plankton.field("id")
  def __init__(self, id=None):
    self.id = id

  def __hash__(self):
    return hash(self.id)

class PlanktonTest(unittest.TestCase):

  def transcode(self, value, resolver=None, access=None):
    encoder = plankton.Encoder().set_resolver(resolver)
    raw_encoded = encoder.encode(value)
    decoder = plankton.Decoder().set_access(access)
    return decoder.decode(raw_encoded)

  def check_transcoding(self, value):
    raw_decoded = self.transcode(value)
    self.assertEquals(value, raw_decoded)
    base64_encoded = plankton.Encoder().base64encode(value)
    base64_decoded = plankton.Decoder().base64decode(base64_encoded)
    self.assertEquals(value, base64_decoded)

  def test_object_encoding(self):
    test = self.check_transcoding
    test(Pair(1, 2))
    test(Pair(Pair(1, 2), Pair(3, 4)))
    pb = Pair("foo", None)
    pb.second = pb
    pa = self.transcode(pb)
    self.assertEquals("foo", pa.first)
    self.assertTrue(pa is pa.second)
    ab = [pb, pb, pb]
    aa = self.transcode(ab)
    self.assertEquals("foo", aa[0].first)
    self.assertTrue(aa[0] is aa[0].second)
    self.assertTrue(aa[0] is aa[1])
    self.assertTrue(aa[0] is aa[2])

  def test_environment(self):
    val_to_int = {}
    int_to_val = {}
    def add_object(value, index):
      val_to_int[value] = index
      int_to_val[index] = value
    def resolve(obj):
      return val_to_int.get(obj, None)
    def access(key):
      return int_to_val[key]
    obj0 = Key(0)
    dec0 = self.transcode(obj0, resolve, access)
    self.assertFalse(obj0 is dec0)
    add_object(obj0, 0)
    dec0p = self.transcode(obj0, resolve, access)
    self.assertTrue(obj0 is dec0p)

if __name__ == '__main__':
  runner = unittest.TextTestRunner(verbosity=0)
  unittest.main(testRunner=runner)
<commit_msg>Self-review.<commit_after>#!/usr/bin/python

import plankton
import unittest

@plankton.serializable()
class Pair(object):

  @plankton.field("first")
  @plankton.field("second")
  def __init__(self, first, second):
    self.first = first
    self.second = second

  def __eq__(self, that):
    return (self is that) or (self.first == that.first) and (self.second == that.second)

  def __str__(self):
    return "(%s, %s)" % (self.first, self.second)

@plankton.serializable()
class Key(object):

  @plankton.field("id")
  def __init__(self, id=None):
    self.id = id

  def __hash__(self):
    return hash(self.id)

class PlanktonTest(unittest.TestCase):

  def transcode(self, value, resolver=None, access=None):
    encoder = plankton.Encoder().set_resolver(resolver)
    raw_encoded = encoder.encode(value)
    decoder = plankton.Decoder().set_access(access)
    return decoder.decode(raw_encoded)

  def check_transcoding(self, value):
    raw_decoded = self.transcode(value)
    self.assertEquals(value, raw_decoded)
    base64_encoded = plankton.Encoder().base64encode(value)
    base64_decoded = plankton.Decoder().base64decode(base64_encoded)
    self.assertEquals(value, base64_decoded)

  def test_environment(self):
    val_to_int = {}
    int_to_val = {}
    def add_object(value, index):
      val_to_int[value] = index
      int_to_val[index] = value
    def resolve(obj):
      return val_to_int.get(obj, None)
    def access(key):
      return int_to_val[key]
    obj0 = Key(0)
    dec0 = self.transcode(obj0, resolve, access)
    self.assertFalse(obj0 is dec0)
    add_object(obj0, 0)
    dec0p = self.transcode(obj0, resolve, access)
    self.assertTrue(obj0 is dec0p)

if __name__ == '__main__':
  runner = unittest.TextTestRunner(verbosity=0)
  unittest.main(testRunner=runner)
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys

def get_driver(drivers, provider):
    """
    Get a driver.

    @param drivers: Dictionary containing valid providers.
    @param provider: Id of provider to get driver
    @type provider: L{libcloud.types.Provider}
    """
    if provider in drivers:
        mod_name, driver_name = drivers[provider]
        _mod = __import__(mod_name, globals(), locals(), [driver_name])
        return getattr(_mod, driver_name)

    raise AttributeError('Provider %s does not exist' % (provider))

def set_driver(drivers, provider, module, klass):
    """
    Sets a driver.

    @param drivers: Dictionary to store providers.
    @param provider: Id of provider to set driver for
    @type provider: L{libcloud.types.Provider}
    @param module: The module which contains the driver
    @type module: L
    @param klass: The driver class name
    @type klass:
    """

    if provider in drivers:
        raise AttributeError('Provider %s already registered' % (provider))

    drivers[provider] = (module, klass)

    # Check if this driver is valid
    try:
        driver = get_driver(drivers, provider)
    except (ImportError, AttributeError):
        exp = sys.exc_info()[1]
        drivers.pop(provider)
        raise exp

    return driver

def merge_valid_keys(params, valid_keys, extra):
    """
    Merge valid keys from extra into params dictionary and return
    dictionary with keys which have been merged.

    Note: params is modified in place.
    """
    merged = {}
    if not extra:
        return merged

    for key in valid_keys:
        if key in extra:
            params[key] = extra[key]
            merged[key] = extra[key]

    return merged

def get_new_obj(obj, klass, attributes):
    """
    Pass attributes from the existing object 'obj' and attributes
    dictionary to a 'klass' constructor.
    Attributes from 'attributes' dictionary are only passed to the
    constructor if they are not None.
    """
    kwargs = {}
    for key, value in list(obj.__dict__.items()):
        if isinstance(value, dict):
            kwargs[key] = value.copy()
        elif isinstance(value, (tuple, list)):
            kwargs[key] = value[:]
        else:
            kwargs[key] = value

    for key, value in list(attributes.items()):
        if value is None:
            continue

        if isinstance(value, dict):
            kwargs_value = kwargs.get(key, {})
            for key1, value2 in list(value.items()):
                if value2 is None:
                    continue

                kwargs_value[key1] = value2
            kwargs[key] = kwargs_value
        else:
            kwargs[key] = value

    return klass(**kwargs)

def str2dicts(data):
    """
    Create a list of dictionaries from a whitespace and newline delimited text.

    For example, this:
    cpu 1100
    ram 640

    cpu 2200
    ram 1024

    becomes:
    [{'cpu': '1100', 'ram': '640'}, {'cpu': '2200', 'ram': '1024'}]
    """
    list_data = []
    list_data.append({})
    d = list_data[-1]

    lines = data.split('\n')
    for line in lines:
        line = line.strip()

        if not line:
            d = {}
            list_data.append(d)
            d = list_data[-1]
            continue

        whitespace = line.find(' ')

        if not whitespace:
            continue

        key = line[0:whitespace]
        value = line[whitespace + 1:]
        d.update({key: value})

    list_data = [value for value in list_data if value != {}]
    return list_data

def str2list(data):
    """
    Create a list of values from a whitespace and newline delimited text
    (keys are ignored).

    For example, this:
    ip 1.2.3.4
    ip 1.2.3.5
    ip 1.2.3.6

    becomes:
    ['1.2.3.4', '1.2.3.5', '1.2.3.6']
    """
    list_data = []

    for line in data.split('\n'):
        line = line.strip()

        if not line:
            continue

        try:
            splitted = line.split(' ')
            # key = splitted[0]
            value = splitted[1]
        except Exception:
            continue

        list_data.append(value)

    return list_data

def dict2str(data):
    """
    Create a string with a whitespace and newline delimited text from a
    dictionary.

    For example, this:
    {'cpu': '1100', 'ram': '640', 'smp': 'auto'}

    becomes:
    cpu 1100
    ram 640
    smp auto

    cpu 2200
    ram 1024
    """
    result = ''
    for k in data:
        if data[k] != None:
            result += '%s %s\n' % (str(k), str(data[k]))
        else:
            result += '%s\n' % str(k)

    return result

def reverse_dict(dictionary):
    return dict([(value, key) for key, value in list(dictionary.items())])

def lowercase_keys(dictionary):
    return dict(((k.lower(), v) for k, v in dictionary.items()))
<commit_msg>pep8, add missing __all__.<commit_after># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__all__ = [
    'get_driver',
    'set_driver',
    'merge_valid_keys',
    'get_new_obj',
    'str2dicts',
    'dict2str',
    'reverse_dict',
    'lowercase_keys'
]

import sys

def get_driver(drivers, provider):
    """
    Get a driver.

    @param drivers: Dictionary containing valid providers.
    @param provider: Id of provider to get driver
    @type provider: L{libcloud.types.Provider}
    """
    if provider in drivers:
        mod_name, driver_name = drivers[provider]
        _mod = __import__(mod_name, globals(), locals(), [driver_name])
        return getattr(_mod, driver_name)

    raise AttributeError('Provider %s does not exist' % (provider))

def set_driver(drivers, provider, module, klass):
    """
    Sets a driver.

    @param drivers: Dictionary to store providers.
    @param provider: Id of provider to set driver for
    @type provider: L{libcloud.types.Provider}
    @param module: The module which contains the driver
    @type module: L
    @param klass: The driver class name
    @type klass:
    """

    if provider in drivers:
        raise AttributeError('Provider %s already registered' % (provider))

    drivers[provider] = (module, klass)

    # Check if this driver is valid
    try:
        driver = get_driver(drivers, provider)
    except (ImportError, AttributeError):
        exp = sys.exc_info()[1]
        drivers.pop(provider)
        raise exp

    return driver

def merge_valid_keys(params, valid_keys, extra):
    """
    Merge valid keys from extra into params dictionary and return
    dictionary with keys which have been merged.

    Note: params is modified in place.
    """
    merged = {}
    if not extra:
        return merged

    for key in valid_keys:
        if key in extra:
            params[key] = extra[key]
            merged[key] = extra[key]

    return merged

def get_new_obj(obj, klass, attributes):
    """
    Pass attributes from the existing object 'obj' and attributes
    dictionary to a 'klass' constructor.
    Attributes from 'attributes' dictionary are only passed to the
    constructor if they are not None.
    """
    kwargs = {}
    for key, value in list(obj.__dict__.items()):
        if isinstance(value, dict):
            kwargs[key] = value.copy()
        elif isinstance(value, (tuple, list)):
            kwargs[key] = value[:]
        else:
            kwargs[key] = value

    for key, value in list(attributes.items()):
        if value is None:
            continue

        if isinstance(value, dict):
            kwargs_value = kwargs.get(key, {})
            for key1, value2 in list(value.items()):
                if value2 is None:
                    continue

                kwargs_value[key1] = value2
            kwargs[key] = kwargs_value
        else:
            kwargs[key] = value

    return klass(**kwargs)

def str2dicts(data):
    """
    Create a list of dictionaries from a whitespace and newline delimited text.

    For example, this:
    cpu 1100
    ram 640

    cpu 2200
    ram 1024

    becomes:
    [{'cpu': '1100', 'ram': '640'}, {'cpu': '2200', 'ram': '1024'}]
    """
    list_data = []
    list_data.append({})
    d = list_data[-1]

    lines = data.split('\n')
    for line in lines:
        line = line.strip()

        if not line:
            d = {}
            list_data.append(d)
            d = list_data[-1]
            continue

        whitespace = line.find(' ')

        if not whitespace:
            continue

        key = line[0:whitespace]
        value = line[whitespace + 1:]
        d.update({key: value})

    list_data = [value for value in list_data if value != {}]
    return list_data

def str2list(data):
    """
    Create a list of values from a whitespace and newline delimited text
    (keys are ignored).

    For example, this:
    ip 1.2.3.4
    ip 1.2.3.5
    ip 1.2.3.6

    becomes:
    ['1.2.3.4', '1.2.3.5', '1.2.3.6']
    """
    list_data = []

    for line in data.split('\n'):
        line = line.strip()

        if not line:
            continue

        try:
            splitted = line.split(' ')
            # key = splitted[0]
            value = splitted[1]
        except Exception:
            continue

        list_data.append(value)

    return list_data

def dict2str(data):
    """
    Create a string with a whitespace and newline delimited text from a
    dictionary.

    For example, this:
    {'cpu': '1100', 'ram': '640', 'smp': 'auto'}

    becomes:
    cpu 1100
    ram 640
    smp auto

    cpu 2200
    ram 1024
    """
    result = ''
    for k in data:
        if data[k] is not None:
            result += '%s %s\n' % (str(k), str(data[k]))
        else:
            result += '%s\n' % str(k)

    return result

def reverse_dict(dictionary):
    return dict([(value, key) for key, value in list(dictionary.items())])

def lowercase_keys(dictionary):
    return dict(((k.lower(), v) for k, v in dictionary.items()))
<|endoftext|>


<commit_before>import sys
import signal
import socket
import argparse
import string
import random

import simplejson
import circus

from circus.exc import ConflictError
from zmq.eventloop import ioloop
from logbook import Logger, INFO, DEBUG, NullHandler, NestedSetup
from logbook.queues import ZeroMQHandler, ZeroMQSubscriber
from logbook.more import ColorizedStderrHandler
from tornado import gen

from .utils import connect_to_redis

@gen.coroutine
def load_drivers(*args, **kwargs):
    logger.info("Loading setup...")

    redis = connect_to_redis()
    redis.delete('ports')
    redis.delete('entries')
    redis.delete('session')
    redis.delete('events')

    try:
        with open(setup_file) as f:
            setup = simplejson.load(f)
    except simplejson.JSONDecodeError as e:
        logger.error("Error parsing '{}' : {}", setup_file, e)
        loop.stop()
    except Exception as e:
        logger.error(
            "Can't process setup file '{}' : {}", setup_file, e
        )
        loop.stop()

    if not 'name' in setup:
        # If the current setup does not have a name, we create a random one
        setup['name'] = ''.join(
            random.sample(string.letters + string.digits, 10)
        )

    if not 'entries' in setup:
        logger.warn("No entries specified in '{}'", setup_file)
        loop.stop()

    entries = setup['entries']

    redis.sadd('entries', *entries.keys())

    for name, conf in entries.items():
        logger.debug("Loading entry {}", name)

        if ':' in name:
            logger.error("Illegal character ':' in entry {}", name)
            continue

        script = 'onitu.drivers.{}'.format(conf['driver'])

        if 'options' in conf:
            redis.hmset('drivers:{}:options'.format(name), conf['options'])

        watcher = arbiter.add_watcher(
            name,
            sys.executable,
            args=['-m', script, name, log_uri],
            copy_env=True,
        )

        loop.add_callback(start_watcher, watcher)

    logger.debug("Entries loaded")

@gen.coroutine
def start_watcher(watcher):
    try:
        watcher.start()
    except ConflictError as e:
        loop.add_callback(start_watcher, watcher)
    except Exception as e:
        logger.warning("Can't start entry {} : {}", watcher.name, e)
        return

def get_logs_dispatcher(uri=None, debug=False):
    handlers = []

    if not debug:
        handlers.append(NullHandler(level=DEBUG))

    handlers.append(ColorizedStderrHandler(level=INFO))

    if not uri:
        # Find an open port.
        # This is a race condition as the port could be used between
        # the check and its binding. However, this is probably one of the
        # best solution without patching Logbook.
        tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        tmpsock.bind(('localhost', 0))
        uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
        tmpsock.close()

    subscriber = ZeroMQSubscriber(uri, multi=True)
    return uri, subscriber.dispatch_in_background(setup=NestedSetup(handlers))

if __name__ == '__main__':
    parser = argparse.ArgumentParser("onitu")
    parser.add_argument(
        '--setup', default='setup.json',
        help="A JSON file with Onitu's configuration (defaults to setup.json)"
    )
    parser.add_argument(
        '--log-uri', help="A ZMQ socket where all the logs will be sent"
    )
    parser.add_argument(
        '--debug', action='store_true', help="Enable debugging logging"
    )
    args = parser.parse_args()

    setup_file = args.setup
    log_uri = args.log_uri
    dispatcher = None

    if not args.log_uri:
        log_uri, dispatcher = get_logs_dispatcher(
            uri=log_uri, debug=args.debug
        )

    with ZeroMQHandler(log_uri, multi=True):
        logger = Logger("Onitu")

        ioloop.install()
        loop = ioloop.IOLoop.instance()

        arbiter = circus.get_arbiter(
            [
                {
                    'cmd': 'redis-server',
                    'args': 'redis/redis.conf',
                    'copy_env': True,
                    'priority': 1,
                },
                {
                    'cmd': sys.executable,
                    'args': ['-m', 'onitu.referee', log_uri],
                    'copy_env': True,
                },
            ],
            proc_name="Onitu",
            loop=loop
        )

        for s in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
            signal.signal(s, lambda *args, **kwargs: loop.stop())

        try:
            future = arbiter.start()
            loop.add_future(future, load_drivers)
            arbiter.start_io_loop()
        except (KeyboardInterrupt, SystemExit):
            pass
        finally:
            arbiter.stop()
            if dispatcher:
                dispatcher.stop()
<commit_msg>Add check for illegal chars in setup name<commit_after>import sys
import signal
import socket
import argparse
import string
import random

import simplejson
import circus

from circus.exc import ConflictError
from zmq.eventloop import ioloop
from logbook import Logger, INFO, DEBUG, NullHandler, NestedSetup
from logbook.queues import ZeroMQHandler, ZeroMQSubscriber
from logbook.more import ColorizedStderrHandler
from tornado import gen

from .utils import connect_to_redis

@gen.coroutine
def load_drivers(*args, **kwargs):
    logger.info("Loading setup...")

    redis = connect_to_redis()
    redis.delete('ports')
    redis.delete('entries')
    redis.delete('session')
    redis.delete('events')

    try:
        with open(setup_file) as f:
            setup = simplejson.load(f)
    except simplejson.JSONDecodeError as e:
        logger.error("Error parsing '{}' : {}", setup_file, e)
        loop.stop()
    except Exception as e:
        logger.error(
            "Can't process setup file '{}' : {}", setup_file, e
        )
        loop.stop()

    if not 'name' in setup:
        # If the current setup does not have a name, we create a random one
        setup['name'] = ''.join(
            random.sample(string.letters + string.digits, 10)
        )

    if ':' in setup['name']:
        logger.error("Illegal character ':' in name '{}'", setup['name'])
        loop.stop()

    if not 'entries' in setup:
        logger.warn("No entries specified in '{}'", setup_file)
        loop.stop()

    entries = setup['entries']

    redis.sadd('entries', *entries.keys())

    for name, conf in entries.items():
        logger.debug("Loading entry {}", name)

        if ':' in name:
            logger.error("Illegal character ':' in entry {}", name)
            continue

        script = 'onitu.drivers.{}'.format(conf['driver'])

        if 'options' in conf:
            redis.hmset('drivers:{}:options'.format(name), conf['options'])

        watcher = arbiter.add_watcher(
            name,
            sys.executable,
            args=['-m', script, name, log_uri],
            copy_env=True,
        )

        loop.add_callback(start_watcher, watcher)

    logger.debug("Entries loaded")

@gen.coroutine
def start_watcher(watcher):
    try:
        watcher.start()
    except ConflictError as e:
        loop.add_callback(start_watcher, watcher)
    except Exception as e:
        logger.warning("Can't start entry {} : {}", watcher.name, e)
        return

def get_logs_dispatcher(uri=None, debug=False):
    handlers = []

    if not debug:
        handlers.append(NullHandler(level=DEBUG))

    handlers.append(ColorizedStderrHandler(level=INFO))

    if not uri:
        # Find an open port.
        # This is a race condition as the port could be used between
        # the check and its binding. However, this is probably one of the
        # best solution without patching Logbook.
        tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        tmpsock.bind(('localhost', 0))
        uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
        tmpsock.close()

    subscriber = ZeroMQSubscriber(uri, multi=True)
    return uri, subscriber.dispatch_in_background(setup=NestedSetup(handlers))

if __name__ == '__main__':
    parser = argparse.ArgumentParser("onitu")
    parser.add_argument(
        '--setup', default='setup.json',
        help="A JSON file with Onitu's configuration (defaults to setup.json)"
    )
    parser.add_argument(
        '--log-uri', help="A ZMQ socket where all the logs will be sent"
    )
    parser.add_argument(
        '--debug', action='store_true', help="Enable debugging logging"
    )
    args = parser.parse_args()

    setup_file = args.setup
    log_uri = args.log_uri
    dispatcher = None

    if not args.log_uri:
        log_uri, dispatcher = get_logs_dispatcher(
            uri=log_uri, debug=args.debug
        )

    with ZeroMQHandler(log_uri, multi=True):
        logger = Logger("Onitu")

        ioloop.install()
        loop = ioloop.IOLoop.instance()

        arbiter = circus.get_arbiter(
            [
                {
                    'cmd': 'redis-server',
                    'args': 'redis/redis.conf',
                    'copy_env': True,
                    'priority': 1,
                },
                {
                    'cmd': sys.executable,
                    'args': ['-m', 'onitu.referee', log_uri],
                    'copy_env': True,
                },
            ],
            proc_name="Onitu",
            loop=loop
        )

        for s in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
            signal.signal(s, lambda *args, **kwargs: loop.stop())

        try:
            future = arbiter.start()
            loop.add_future(future, load_drivers)
            arbiter.start_io_loop()
        except (KeyboardInterrupt, SystemExit):
            pass
        finally:
            arbiter.stop()
            if dispatcher:
                dispatcher.stop()
<|endoftext|>


<commit_before># Copyright (c) 2017 Mathias Funk
# This software is released under the MIT License.
# http://opensource.org/licenses/mit-license.php

import json
import socket
import threading
import time
import uuid
from math import fsum

__author__ = 'matsfunk'

class OOCSI:

    def __init__(self, handle=None, host='localhost', port=4444, callback=None):
        if handle == None or len(handle.strip()) == 0:
            self.handle = "OOCSIClient_" + uuid.uuid4().__str__().replace('-', '')[0:15];
        else:
            self.handle = handle

        self.receivers = {self.handle: [callback]}
        self.calls = {}
        self.services = {}
        self.reconnect = True
        self.connected = False

        # Connect the socket to the port where the server is listening
        self.server_address = (host, port)
        self.log('connecting to %s port %s' % self.server_address)

        # start the connection thread
        self.runtime = OOCSIThread(self)
        self.runtime.start()

        # block till we are connected
        while not self.connected:
            {}

    def init(self):
        try:
            # Create a TCP/IP socket
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.connect(self.server_address)

            try:
                # Send data
                message = self.handle + '(JSON)'
                self.internalSend(message)

                data = self.sock.recv(1024).decode()
                if data.startswith('{'):
                    self.log('connection established')
                    # re-subscribe for channels
                    for channelName in self.receivers:
                        self.internalSend('subscribe {0}'.format(channelName))
                    self.connected = True
                else:
                    if data.startswith('error'):
                        self.log(data)
                        self.reconnect = False

                while self.connected:
                    self.loop()

            finally:
                {}

        except:
            {}

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.stop()

    def log(self, message):
        print('[{0}]: {1}'.format(self.handle, message))

    def internalSend(self, msg):
        try:
            self.sock.sendall((msg + '\n').encode())
        except:
            self.connected = False

    def loop(self):
        try:
            data = self.sock.recv(10 * 1024).decode()

            if len(data) == 0:
                self.sock.close()
                self.connected = False
            if data.startswith('ping'):
                self.internalSend('.')
            if data.startswith('{'):
                self.receive(json.loads(data))
        except :
            {}

    def receive(self, event):
        sender = event['sender']
        recipient = event['recipient']

        # clean up the event
        del event['recipient']
        del event['sender']
        del event['timestamp']
        if 'data' in event:
            del event['data']

        if '_MESSAGE_HANDLE' in event and event['_MESSAGE_HANDLE'] in self.services:
            service = self.services[event['_MESSAGE_HANDLE']]
            del event['_MESSAGE_HANDLE']
            service(event)
            self.send(sender, event)
            self.receiveChannelEvent(sender, recipient, event)

        else:

            if '_MESSAGE_ID' in event:
                myCall = self.calls[event['_MESSAGE_ID']]
                if myCall['expiration'] > time.time():
                    response = self.calls[event['_MESSAGE_ID']]
                    response['response'] = event
                    del response['expiration']
                    del response['_MESSAGE_ID']
                    del response['response']['_MESSAGE_ID']
                else:
                    del self.calls[event['_MESSAGE_ID']]

            else:
                self.receiveChannelEvent(sender, recipient, event)
#         else:
#             print('not good -------------- ', filteredEvent)

    def receiveChannelEvent(self, sender, recipient, event):
        if recipient in self.receivers and self.receivers[recipient] != None:
            for x in self.receivers[recipient]:
                x(sender, recipient, event)

    def send(self, channelName, data):
        self.internalSend('sendraw {0} {1}'.format(channelName, json.dumps(data)))

    def call(self, channelName, callName, data, timeout = 1):
        data['_MESSAGE_HANDLE'] = callName
        data['_MESSAGE_ID'] = uuid.uuid4().__str__()
        self.calls[data['_MESSAGE_ID']] = {'_MESSAGE_HANDLE': callName, '_MESSAGE_ID': data['_MESSAGE_ID'], 'expiration': time.time() + timeout}
        self.send(channelName, data)
        return self.calls[data['_MESSAGE_ID']]

    def callAndWait(self, channelName, callName, data, timeout = 1):
        call = self.call(channelName, callName, data, timeout)
        expiration = time.time() + timeout
        while time.time() < expiration:
            time.sleep(0.1)
            if 'response' in call:
                break;
            #         self.calls.append
        return call

    def register(self, channelName, callName, callback):
        self.services[callName] = callback
        self.internalSend('subscribe {0}'.format(channelName))
        self.log('registered responder on {0} for {1}'.format(channelName, callName))

    def subscribe(self, channelName, f):
        if channelName in self.receivers:
            self.receivers[channelName].append(f)
        else:
            self.receivers[channelName] = [f]
        self.internalSend('subscribe {0}'.format(channelName))
        self.log('subscribed to {0}'.format(channelName))

    def unsubscribe(self, channelName):
        del self.receivers[channelName]
        self.internalSend('unsubscribe {0}'.format(channelName))
        self.log('unsubscribed from {0}'.format(channelName))

    def variable(self, channelName, key):
        return OOCSIVariable(self, channelName, key)

    def stop(self):
        self.reconnect = False
        self.internalSend('quit')
        self.sock.close()
        self.connected = False

    def handleEvent(self, sender, receiver, message):
        {}

class OOCSIThread(threading.Thread):

    def __init__(self, parent=None):
        self.parent = parent
        super(OOCSIThread, self).__init__()

    def run(self):
        while self.parent.reconnect:
            self.parent.init()
            if self.parent.reconnect:
                self.parent.log('re-connecting to OOCSI')
                time.sleep(5)

        self.parent.log('closing connection to OOCSI')

    def __stop(self):
        self.parent.stop()
        return threading.Thread.__stop(self)

class OOCSICall:

    def __init__(self, parent=None):
        self.uuid = uuid.uuid4()
        self.expiration = time.time()

class OOCSIVariable(object):

    def __init__(self, oocsi, channelName, key):
        self.key = key
        self.channel = channelName
        oocsi.subscribe(channelName, self.internalReceiveValue)
        self.oocsi = oocsi
        self.value = None
        self.windowLength = 0
        self.values = []
        self.minvalue = None
        self.maxvalue = None
        self.sigma = None

    def get(self):
        if self.windowLength > 0 and len(self.values) > 0:
            return fsum(self.values)/float(len(self.values))
        else:
            return self.value

    def set(self, value):
        tempvalue = value
        if not self.minvalue == None and tempvalue < self.minvalue:
            tempvalue = self.minvalue
        if not self.maxvalue == None and tempvalue > self.maxvalue:
            tempvalue = self.maxvalue
        if not self.sigma == None:
            mean = self.get()
            if not mean == None:
                if abs(mean - tempvalue) > self.sigma:
                    if mean - tempvalue > 0:
                        tempvalue = mean - self.sigma/float(len(self.values))
                    else:
                        tempvalue = mean + self.sigma/float(len(self.values))

        if self.windowLength > 0:
            self.values.append(tempvalue)
            self.values = self.values[-self.windowLength:]
        else:
            self.value = tempvalue
        self.oocsi.send(self.channel, {self.key: value})

    def internalReceiveValue(self, sender, recipient, data):
        if self.key in data:
            tempvalue = data[self.key]
            if not self.minvalue == None and tempvalue < self.minvalue:
                tempvalue = self.minvalue
            if not self.maxvalue == None and tempvalue > self.maxvalue:
                tempvalue = self.maxvalue
            if not self.sigma == None:
                mean = self.get()
                if not mean == None:
                    if abs(mean - tempvalue) > self.sigma:
                        if mean - tempvalue > 0:
                            tempvalue = mean - self.sigma/float(len(self.values))
                        else:
                            tempvalue = mean + self.sigma/float(len(self.values))

            if self.windowLength > 0:
                self.values.append(tempvalue)
                self.values = self.values[-self.windowLength:]
            else:
                self.value = tempvalue

    def min(self, minvalue):
        self.minvalue = minvalue
        if self.value < self.minvalue:
            self.value = self.minvalue
        return self

    def max(self, maxvalue):
        self.maxvalue = maxvalue
        if self.value > self.maxvalue:
            self.value = self.maxvalue
        return self

    def smooth(self, windowLength, sigma=None):
        self.windowLength = windowLength
        self.sigma = sigma
        return self
        <commit_msg> - misc fixes  - increased buffer size<commit_after># Copyright (c) 2017 Mathias Funk
# This software is released under the MIT License.
# http://opensource.org/licenses/mit-license.php

import json
import socket
import threading
import time
import uuid
from math import fsum

__author__ = 'matsfunk'

class OOCSI:

    def __init__(self, handle=None, host='localhost', port=4444, callback=None):
        if handle == None or len(handle.strip()) == 0:
            self.handle = "OOCSIClient_" + uuid.uuid4().__str__().replace('-', '')[0:15];
        else:
            self.handle = handle

        self.receivers = {self.handle: [callback]}
        self.calls = {}
        self.services = {}
        self.reconnect = True
        self.connected = False

        # Connect the socket to the port where the server is listening
        self.server_address = (host, port)
        self.log('connecting to %s port %s' % self.server_address)

        # start the connection thread
        self.runtime = OOCSIThread(self)
        self.runtime.start()

        # block till we are connected
        while not self.connected:
            {}

    def init(self):
        try:
            # Create a TCP/IP socket
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.connect(self.server_address)

            try:
                # Send data
                message = self.handle + '(JSON)'
                self.internalSend(message)

                data = self.sock.recv(1024).decode()
                if data.startswith('{'):
                    self.log('connection established')
                    # re-subscribe for channels
                    for channelName in self.receivers:
                        self.internalSend('subscribe {0}'.format(channelName))
                    self.connected = True
                elif data.startswith('error'):
                    self.log(data)
                    self.reconnect = False

                while self.connected:
                    self.loop()

            finally:
                {}

        except:
            {}

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.stop()

    def log(self, message):
        print('[{0}]: {1}'.format(self.handle, message))

    def internalSend(self, msg):
        try:
            self.sock.sendall((msg + '\n').encode())
        except:
            self.connected = False

    def loop(self):
        try:
            data = self.sock.recv(4 * 1024 * 1024).decode()

            if len(data) == 0:
                self.sock.close()
                self.connected = False
            elif data.startswith('ping'):
                self.internalSend('.')
            elif data.startswith('{'):
                self.receive(json.loads(data))
        except :
            {}

    def receive(self, event):
        sender = event['sender']
        recipient = event['recipient']

        # clean up the event
        del event['recipient']
        del event['sender']
        del event['timestamp']
        if 'data' in event:
            del event['data']

        if '_MESSAGE_HANDLE' in event and event['_MESSAGE_HANDLE'] in self.services:
            service = self.services[event['_MESSAGE_HANDLE']]
            del event['_MESSAGE_HANDLE']
            service(event)
            self.send(sender, event)
            self.receiveChannelEvent(sender, recipient, event)

        else:

            if '_MESSAGE_ID' in event:
                myCall = self.calls[event['_MESSAGE_ID']]
                if myCall['expiration'] > time.time():
                    response = self.calls[event['_MESSAGE_ID']]
                    response['response'] = event
                    del response['expiration']
                    del response['_MESSAGE_ID']
                    del response['response']['_MESSAGE_ID']
                else:
                    del self.calls[event['_MESSAGE_ID']]

            else:
                self.receiveChannelEvent(sender, recipient, event)

    def receiveChannelEvent(self, sender, recipient, event):
        if recipient in self.receivers and self.receivers[recipient] != None:
            for x in self.receivers[recipient]:
                x(sender, recipient, event)

    def send(self, channelName, data):
        self.internalSend('sendraw {0} {1}'.format(channelName, json.dumps(data)))

    def call(self, channelName, callName, data, timeout = 1):
        data['_MESSAGE_HANDLE'] = callName
        data['_MESSAGE_ID'] = uuid.uuid4().__str__()
        self.calls[data['_MESSAGE_ID']] = {'_MESSAGE_HANDLE': callName, '_MESSAGE_ID': data['_MESSAGE_ID'], 'expiration': time.time() + timeout}
        self.send(channelName, data)
        return self.calls[data['_MESSAGE_ID']]

    def callAndWait(self, channelName, callName, data, timeout = 1):
        call = self.call(channelName, callName, data, timeout)
        expiration = time.time() + timeout
        while time.time() < expiration:
            time.sleep(0.1)
            if 'response' in call:
                break;
            #         self.calls.append
        return call

    def register(self, channelName, callName, callback):
        self.services[callName] = callback
        self.internalSend('subscribe {0}'.format(channelName))
        self.log('registered responder on {0} for {1}'.format(channelName, callName))

    def subscribe(self, channelName, f):
        if channelName in self.receivers:
            self.receivers[channelName].append(f)
        else:
            self.receivers[channelName] = [f]
        self.internalSend('subscribe {0}'.format(channelName))
        self.log('subscribed to {0}'.format(channelName))

    def unsubscribe(self, channelName):
        del self.receivers[channelName]
        self.internalSend('unsubscribe {0}'.format(channelName))
        self.log('unsubscribed from {0}'.format(channelName))

    def variable(self, channelName, key):
        return OOCSIVariable(self, channelName, key)

    def stop(self):
        self.reconnect = False
        self.internalSend('quit')
        self.sock.close()
        self.connected = False

    def handleEvent(self, sender, receiver, message):
        {}

class OOCSIThread(threading.Thread):

    def __init__(self, parent=None):
        self.parent = parent
        super(OOCSIThread, self).__init__()

    def run(self):
        while self.parent.reconnect:
            self.parent.init()
            if self.parent.reconnect:
                self.parent.log('re-connecting to OOCSI')
                time.sleep(5)

        self.parent.log('closing connection to OOCSI')

    def __stop(self):
        self.parent.stop()
        return threading.Thread.__stop(self)

class OOCSICall:

    def __init__(self, parent=None):
        self.uuid = uuid.uuid4()
        self.expiration = time.time()

class OOCSIVariable(object):

    def __init__(self, oocsi, channelName, key):
        self.key = key
        self.channel = channelName
        oocsi.subscribe(channelName, self.internalReceiveValue)
        self.oocsi = oocsi
        self.value = None
        self.windowLength = 0
        self.values = []
        self.minvalue = None
        self.maxvalue = None
        self.sigma = None

    def get(self):
        if self.windowLength > 0 and len(self.values) > 0:
            return fsum(self.values)/float(len(self.values))
        else:
            return self.value

    def set(self, value):
        tempvalue = value
        if not self.minvalue == None and tempvalue < self.minvalue:
            tempvalue = self.minvalue
        elif not self.maxvalue == None and tempvalue > self.maxvalue:
            tempvalue = self.maxvalue
        elif not self.sigma == None:
            mean = self.get()
            if not mean == None:
                if abs(mean - tempvalue) > self.sigma:
                    if mean - tempvalue > 0:
                        tempvalue = mean - self.sigma/float(len(self.values))
                    else:
                        tempvalue = mean + self.sigma/float(len(self.values))

        if self.windowLength > 0:
            self.values.append(tempvalue)
            self.values = self.values[-self.windowLength:]
        else:
            self.value = tempvalue
        self.oocsi.send(self.channel, {self.key: value})

    def internalReceiveValue(self, sender, recipient, data):
        if self.key in data:
            tempvalue = data[self.key]
            if not self.minvalue == None and tempvalue < self.minvalue:
                tempvalue = self.minvalue
            elif not self.maxvalue == None and tempvalue > self.maxvalue:
                tempvalue = self.maxvalue
            elif not self.sigma == None:
                mean = self.get()
                if not mean == None:
                    if abs(mean - tempvalue) > self.sigma:
                        if mean - tempvalue > 0:
                            tempvalue = mean - self.sigma/float(len(self.values))
                        else:
                            tempvalue = mean + self.sigma/float(len(self.values))

            if self.windowLength > 0:
                self.values.append(tempvalue)
                self.values = self.values[-self.windowLength:]
            else:
                self.value = tempvalue

    def min(self, minvalue):
        self.minvalue = minvalue
        if self.value < self.minvalue:
            self.value = self.minvalue
        return self

    def max(self, maxvalue):
        self.maxvalue = maxvalue
        if self.value > self.maxvalue:
            self.value = self.maxvalue
        return self

    def smooth(self, windowLength, sigma=None):
        self.windowLength = windowLength
        self.sigma = sigma
        return self
        <|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function
from future import standard_library; standard_library.install_aliases()
import logging
_L = logging.getLogger(__name__)

import json
import os
import errno
import socket
import mimetypes
import shutil

from urllib.parse import urlencode, urlparse
from hashlib import sha1

import requests

def mkdirsp(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

class CacheResult:
    cache = None
    fingerprint = None
    version = None
    elapsed = None

    # needed by openaddr.process.write_state(), for now.
    output = ''

    def __init__(self, cache, fingerprint, version, elapsed):
        self.cache = cache
        self.fingerprint = fingerprint
        self.version = version
        self.elapsed = elapsed

    @staticmethod
    def empty():
        return CacheResult(None, None, None, None)

    def todict(self):
        return dict(cache=self.cache, fingerprint=self.fingerprint, version=self.version)

class DownloadError(Exception):
    pass

class DownloadTask(object):

    def __init__(self, source_prefix):
        self.source_prefix = source_prefix

    @classmethod
    def from_type_string(clz, type_string, source_prefix=None):
        if type_string.lower() == 'http':
            return URLDownloadTask(source_prefix)
        elif type_string.lower() == 'ftp':
            return URLDownloadTask(source_prefix)
        elif type_string.lower() == 'esri':
            return EsriRestDownloadTask(source_prefix)
        else:
            raise KeyError("I don't know how to extract for type {}".format(type_string))

    def download(self, source_urls, workdir):
        raise NotImplementedError()

class URLDownloadTask(DownloadTask):
    USER_AGENT = 'openaddresses-extract/1.0 (https://github.com/openaddresses/openaddresses)'
    CHUNK = 16 * 1024

    def get_file_path(self, url, dir_path):
        ''' Return a local file path in a directory for a URL.

            May need to fill in a filename extension based on HTTP Content-Type.
        '''
        _, host, path, _, _, _ = urlparse(url)
        path_base, path_ext = os.path.splitext(path)

        if self.source_prefix is None:
            # With no source prefix like "us-ca-oakland" use the name as given.
            name_base = os.path.basename(path_base)
        else:
            # With a source prefix, create a safe and unique filename with a hash.
            hash = sha1((host + path_base).encode('utf-8'))
            name_base = '{}-{}'.format(self.source_prefix, hash.hexdigest()[:8])

        if not path_ext:
            resp = requests.head(url)
            path_ext = mimetypes.guess_extension(resp.headers['content-type'])
            _L.debug('Guessing {}{} for {}'.format(name_base, path_ext, resp.headers['content-type']))

        _L.debug('Downloading {} to {}{}'.format(path, name_base, path_ext))

        return os.path.join(dir_path, name_base + path_ext)

    def download(self, source_urls, workdir):
        output_files = []
        download_path = os.path.join(workdir, 'http')
        mkdirsp(download_path)

        for source_url in source_urls:
            file_path = self.get_file_path(source_url, download_path)

            # FIXME: For URLs with file:// scheme, simply copy the file
            # to the expected location so that os.path.exists() returns True.
            # Instead, implement a FileDownloadTask class?
            scheme, _, path, _, _, _ = urlparse(source_url)
            if scheme == 'file':
                shutil.copy(path, file_path)

            if os.path.exists(file_path):
                output_files.append(file_path)
                _L.debug("File exists %s", file_path)
                continue

            _L.debug("Requesting %s", source_url)
            headers = {'User-Agent': self.USER_AGENT}

            try:
                resp = requests.get(source_url, headers=headers, stream=True)
            except Exception as e:
                raise DownloadError("Could not connect to URL", e)

            size = 0
            with open(file_path, 'wb') as fp:
                for chunk in resp.iter_content(self.CHUNK):
                    size += len(chunk)
                    fp.write(chunk)

            output_files.append(file_path)

            _L.info("Downloaded %s bytes for file %s", size, file_path)

        return output_files

class EsriRestDownloadTask(DownloadTask):
    USER_AGENT = 'openaddresses-extract/1.0 (https://github.com/openaddresses/openaddresses)'

    def convert_esrijson_to_geojson(self, geom_type, esri_feature):
        if geom_type == 'esriGeometryPoint':
            geometry = {
                "type": "Point",
                "coordinates": [
                    esri_feature['geometry']['x'],
                    esri_feature['geometry']['y']
                ]
            }
        elif geom_type == 'esriGeometryMultipoint':
            geometry = {
                "type": "MultiPoint",
                "coordinates": [
                    [geom[0], geom[1]] for geom in esri_feature['geometry']['points']
                ]
            }
        elif geom_type == 'esriGeometryPolygon':
            geometry = {
                "type": "Polygon",
                "coordinates": [
                    [
                        [geom[0], geom[1]] for geom in ring
                    ] for ring in esri_feature['geometry']['rings']
                ]
            }
        elif geom_type == 'esriGeometryPolyline':
            geometry = {
                "type": "MultiLineString",
                "coordinates": [
                    [
                        [geom[0], geom[1]] for geom in path
                    ] for path in esri_feature['geometry']['paths']
                ]
            }
        else:
            raise KeyError("Don't know how to convert esri geometry type {}".format(geom_type))

        return {
            "type": "Feature",
            "properties": esri_feature.get('attributes'),
            "geometry": geometry
        }

    def get_file_path(self, url, dir_path):
        ''' Return a local file path in a directory for a URL.
        '''
        _, host, path, _, _, _ = urlparse(url)
        hash, path_ext = sha1((host + path).encode('utf-8')), '.json'

        # With no source prefix like "us-ca-oakland" use the host as a hint.
        name_base = '{}-{}'.format(self.source_prefix or host, hash.hexdigest()[:8])

        _L.debug('Downloading {} to {}{}'.format(path, name_base, path_ext))

        return os.path.join(dir_path, name_base + path_ext)

    def download(self, source_urls, workdir):
        output_files = []
        download_path = os.path.join(workdir, 'esri')
        mkdirsp(download_path)

        for source_url in source_urls:
            size = 0
            file_path = self.get_file_path(source_url, download_path)

            if os.path.exists(file_path):
                output_files.append(file_path)
                _L.debug("File exists %s", file_path)
                continue

            with open(file_path, 'w') as f:
                f.write('{\n"type": "FeatureCollection",\n"features": [\n')
                start = 0
                width = 500
                while True:
                    query_url = source_url + '/query'
                    query_args = urlencode({
                        'where': 'objectid >= {} and objectid < {}'.format(start, (start + width)),
                        'geometryPrecision': 7,
                        'returnGeometry': True,
                        'outSR': 4326,
                        'outFields': '*',
                        'f': 'JSON',
                    })
                    query_url += '?' + query_args

                    _L.debug("Requesting %s", query_url)
                    headers = {'User-Agent': self.USER_AGENT}

                    try:
                        data = requests.get(query_url, headers=headers).json()
                    except socket.timeout as e:
                        raise DownloadError("Timeout when connecting to URL", e)
                    except ValueError as e:
                        raise DownloadError("Could not parse JSON", e)
                    except Exception as e:
                        raise DownloadError("Could not connect to URL", e)
                    finally:
                        # Wipe out whatever we had written out so far
                        f.truncate()

                    error = data.get('error')
                    if error:
                        raise DownloadError("Problem querying ESRI dataset: %s", error['message'])

                    geometry_type = data.get('geometryType')
                    features = data.get('features')

                    f.write(',\n'.join([
                        json.dumps(self.convert_esrijson_to_geojson(geometry_type, feature)) for feature in features
                    ]))

                    size += len(features)
                    if len(features) == 0:
                        break
                    else:
                        f.write(',\n')
                        start += width

                f.write('\n]\n}\n')
            _L.info("Downloaded %s ESRI features for file %s", size, file_path)
            output_files.append(file_path)
        return output_files
<commit_msg>Change the log level on download notification<commit_after>from __future__ import absolute_import, division, print_function
from future import standard_library; standard_library.install_aliases()
import logging
_L = logging.getLogger(__name__)

import json
import os
import errno
import socket
import mimetypes
import shutil

from urllib.parse import urlencode, urlparse
from hashlib import sha1

import requests

def mkdirsp(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

class CacheResult:
    cache = None
    fingerprint = None
    version = None
    elapsed = None

    # needed by openaddr.process.write_state(), for now.
    output = ''

    def __init__(self, cache, fingerprint, version, elapsed):
        self.cache = cache
        self.fingerprint = fingerprint
        self.version = version
        self.elapsed = elapsed

    @staticmethod
    def empty():
        return CacheResult(None, None, None, None)

    def todict(self):
        return dict(cache=self.cache, fingerprint=self.fingerprint, version=self.version)

class DownloadError(Exception):
    pass

class DownloadTask(object):

    def __init__(self, source_prefix):
        self.source_prefix = source_prefix

    @classmethod
    def from_type_string(clz, type_string, source_prefix=None):
        if type_string.lower() == 'http':
            return URLDownloadTask(source_prefix)
        elif type_string.lower() == 'ftp':
            return URLDownloadTask(source_prefix)
        elif type_string.lower() == 'esri':
            return EsriRestDownloadTask(source_prefix)
        else:
            raise KeyError("I don't know how to extract for type {}".format(type_string))

    def download(self, source_urls, workdir):
        raise NotImplementedError()

class URLDownloadTask(DownloadTask):
    USER_AGENT = 'openaddresses-extract/1.0 (https://github.com/openaddresses/openaddresses)'
    CHUNK = 16 * 1024

    def get_file_path(self, url, dir_path):
        ''' Return a local file path in a directory for a URL.

            May need to fill in a filename extension based on HTTP Content-Type.
        '''
        _, host, path, _, _, _ = urlparse(url)
        path_base, path_ext = os.path.splitext(path)

        if self.source_prefix is None:
            # With no source prefix like "us-ca-oakland" use the name as given.
            name_base = os.path.basename(path_base)
        else:
            # With a source prefix, create a safe and unique filename with a hash.
            hash = sha1((host + path_base).encode('utf-8'))
            name_base = '{}-{}'.format(self.source_prefix, hash.hexdigest()[:8])

        if not path_ext:
            resp = requests.head(url)
            path_ext = mimetypes.guess_extension(resp.headers['content-type'])
            _L.debug('Guessing {}{} for {}'.format(name_base, path_ext, resp.headers['content-type']))

        _L.debug('Downloading {} to {}{}'.format(path, name_base, path_ext))

        return os.path.join(dir_path, name_base + path_ext)

    def download(self, source_urls, workdir):
        output_files = []
        download_path = os.path.join(workdir, 'http')
        mkdirsp(download_path)

        for source_url in source_urls:
            file_path = self.get_file_path(source_url, download_path)

            # FIXME: For URLs with file:// scheme, simply copy the file
            # to the expected location so that os.path.exists() returns True.
            # Instead, implement a FileDownloadTask class?
            scheme, _, path, _, _, _ = urlparse(source_url)
            if scheme == 'file':
                shutil.copy(path, file_path)

            if os.path.exists(file_path):
                output_files.append(file_path)
                _L.debug("File exists %s", file_path)
                continue

            _L.info("Requesting %s", source_url)
            headers = {'User-Agent': self.USER_AGENT}

            try:
                resp = requests.get(source_url, headers=headers, stream=True)
            except Exception as e:
                raise DownloadError("Could not connect to URL", e)

            size = 0
            with open(file_path, 'wb') as fp:
                for chunk in resp.iter_content(self.CHUNK):
                    size += len(chunk)
                    fp.write(chunk)

            output_files.append(file_path)

            _L.info("Downloaded %s bytes for file %s", size, file_path)

        return output_files

class EsriRestDownloadTask(DownloadTask):
    USER_AGENT = 'openaddresses-extract/1.0 (https://github.com/openaddresses/openaddresses)'

    def convert_esrijson_to_geojson(self, geom_type, esri_feature):
        if geom_type == 'esriGeometryPoint':
            geometry = {
                "type": "Point",
                "coordinates": [
                    esri_feature['geometry']['x'],
                    esri_feature['geometry']['y']
                ]
            }
        elif geom_type == 'esriGeometryMultipoint':
            geometry = {
                "type": "MultiPoint",
                "coordinates": [
                    [geom[0], geom[1]] for geom in esri_feature['geometry']['points']
                ]
            }
        elif geom_type == 'esriGeometryPolygon':
            geometry = {
                "type": "Polygon",
                "coordinates": [
                    [
                        [geom[0], geom[1]] for geom in ring
                    ] for ring in esri_feature['geometry']['rings']
                ]
            }
        elif geom_type == 'esriGeometryPolyline':
            geometry = {
                "type": "MultiLineString",
                "coordinates": [
                    [
                        [geom[0], geom[1]] for geom in path
                    ] for path in esri_feature['geometry']['paths']
                ]
            }
        else:
            raise KeyError("Don't know how to convert esri geometry type {}".format(geom_type))

        return {
            "type": "Feature",
            "properties": esri_feature.get('attributes'),
            "geometry": geometry
        }

    def get_file_path(self, url, dir_path):
        ''' Return a local file path in a directory for a URL.
        '''
        _, host, path, _, _, _ = urlparse(url)
        hash, path_ext = sha1((host + path).encode('utf-8')), '.json'

        # With no source prefix like "us-ca-oakland" use the host as a hint.
        name_base = '{}-{}'.format(self.source_prefix or host, hash.hexdigest()[:8])

        _L.debug('Downloading {} to {}{}'.format(path, name_base, path_ext))

        return os.path.join(dir_path, name_base + path_ext)

    def download(self, source_urls, workdir):
        output_files = []
        download_path = os.path.join(workdir, 'esri')
        mkdirsp(download_path)

        for source_url in source_urls:
            size = 0
            file_path = self.get_file_path(source_url, download_path)

            if os.path.exists(file_path):
                output_files.append(file_path)
                _L.debug("File exists %s", file_path)
                continue

            with open(file_path, 'w') as f:
                f.write('{\n"type": "FeatureCollection",\n"features": [\n')
                start = 0
                width = 500
                while True:
                    query_url = source_url + '/query'
                    query_args = urlencode({
                        'where': 'objectid >= {} and objectid < {}'.format(start, (start + width)),
                        'geometryPrecision': 7,
                        'returnGeometry': True,
                        'outSR': 4326,
                        'outFields': '*',
                        'f': 'JSON',
                    })
                    query_url += '?' + query_args

                    _L.debug("Requesting %s", query_url)
                    headers = {'User-Agent': self.USER_AGENT}

                    try:
                        data = requests.get(query_url, headers=headers).json()
                    except socket.timeout as e:
                        raise DownloadError("Timeout when connecting to URL", e)
                    except ValueError as e:
                        raise DownloadError("Could not parse JSON", e)
                    except Exception as e:
                        raise DownloadError("Could not connect to URL", e)
                    finally:
                        # Wipe out whatever we had written out so far
                        f.truncate()

                    error = data.get('error')
                    if error:
                        raise DownloadError("Problem querying ESRI dataset: %s", error['message'])

                    geometry_type = data.get('geometryType')
                    features = data.get('features')

                    f.write(',\n'.join([
                        json.dumps(self.convert_esrijson_to_geojson(geometry_type, feature)) for feature in features
                    ]))

                    size += len(features)
                    if len(features) == 0:
                        break
                    else:
                        f.write(',\n')
                        start += width

                f.write('\n]\n}\n')
            _L.info("Downloaded %s ESRI features for file %s", size, file_path)
            output_files.append(file_path)
        return output_files
<|endoftext|>


<commit_before># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import platform

if platform.system() == "Java":
    from jil import Image, ImageColor, ImageDraw, ImageFont

    class ImagePalette(object):
        def __init__(self, *args, **kw):
            raise NotImplementedError()

else:
    from PIL import Image, ImageColor, ImageDraw, ImageFont, ImagePalette<commit_msg>platform-depended quantizing<commit_after># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import platform

if platform.system() == "Java":
    from jil import Image, ImageColor, ImageDraw, ImageFont

    class ImagePalette(object):
        def __init__(self, *args, **kw):
            raise NotImplementedError()

    def quantize(img, colors=256, alpha=False, defaults=None):
        return img.convert('P', palette=Image.ADAPTIVE, colors=colors)

else:
    from PIL import Image, ImageColor, ImageDraw, ImageFont, ImagePalette

    def quantize(img, colors=256, alpha=False, defaults=None):
        if hasattr(Image, 'FASTOCTREE'):
            if not alpha:
                img = img.convert('RGB')
            img = img.quantize(colors, Image.FASTOCTREE)
        else:
            if alpha:
                alpha = img.split()[3]
                img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE, colors=colors)
                mask = Image.eval(alpha, lambda a: 255 if a <=128 else 0)
                img.paste(255, mask)
                if defaults is not None:
                    defaults['transparency'] = 255
            else:
                img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE, colors=colors)

        return img<|endoftext|>


<commit_before># coding: utf-8

from __future__ import absolute_import

from django.conf import settings

from sentry.models import (
    Organization, Project, ProjectKey, Team, User
)
from sentry.receivers.core import create_default_projects
from sentry.testutils import TestCase

class CreateDefaultProjectsTest(TestCase):
    def test_simple(self):
        user, _ = User.objects.get_or_create(is_superuser=True, defaults={
            'username': 'test'
        })
        Organization.objects.all().delete()
        Team.objects.filter(slug='sentry').delete()
        Project.objects.filter(id=settings.SENTRY_PROJECT).delete()

        create_default_projects(created_models=[Project])

        project = Project.objects.get(id=settings.SENTRY_PROJECT)
        assert project.public is False
        assert project.name == 'Backend'
        assert project.slug == 'backend'
        team = project.team
        assert team.slug == 'sentry'

        pk = ProjectKey.objects.get(project=project)
        assert not pk.roles.api
        assert pk.roles.store

        # ensure that we dont hit an error here
        create_default_projects(created_models=[Project])

    def test_without_user(self):
        User.objects.filter(is_superuser=True).delete()
        Team.objects.filter(slug='sentry').delete()
        Project.objects.filter(id=settings.SENTRY_PROJECT).delete()

        create_default_projects(created_models=[Project])

        user = User.objects.get(username='sentry')

        project = Project.objects.get(id=settings.SENTRY_PROJECT)
        assert project.public is False
        assert project.name == 'Backend'
        assert project.slug == 'backend'
        team = project.team
        assert team.slug == 'sentry'

        pk = ProjectKey.objects.get(project=project)
        assert not pk.roles.api
        assert pk.roles.store

        # ensure that we dont hit an error here
        create_default_projects(created_models=[Project])
<commit_msg>Update tests<commit_after># coding: utf-8

from __future__ import absolute_import

from django.conf import settings

from sentry.models import (
    Organization, Project, ProjectKey, Team, User
)
from sentry.receivers.core import create_default_projects
from sentry.testutils import TestCase

class CreateDefaultProjectsTest(TestCase):
    def test_simple(self):
        user, _ = User.objects.get_or_create(is_superuser=True, defaults={
            'username': 'test'
        })
        Organization.objects.all().delete()
        Team.objects.filter(slug='sentry').delete()
        Project.objects.filter(id=settings.SENTRY_PROJECT).delete()

        create_default_projects(created_models=[Project])

        project = Project.objects.get(id=settings.SENTRY_PROJECT)
        assert project.public is False
        assert project.name == 'Internal'
        assert project.slug == 'internal'
        team = project.team
        assert team.slug == 'sentry'

        pk = ProjectKey.objects.get(project=project)
        assert not pk.roles.api
        assert pk.roles.store

        # ensure that we dont hit an error here
        create_default_projects(created_models=[Project])

    def test_without_user(self):
        User.objects.filter(is_superuser=True).delete()
        Team.objects.filter(slug='sentry').delete()
        Project.objects.filter(id=settings.SENTRY_PROJECT).delete()

        create_default_projects(created_models=[Project])

        user = User.objects.get(username='sentry')

        project = Project.objects.get(id=settings.SENTRY_PROJECT)
        assert project.public is False
        assert project.name == 'Internal'
        assert project.slug == 'internal'
        team = project.team
        assert team.slug == 'sentry'

        pk = ProjectKey.objects.get(project=project)
        assert not pk.roles.api
        assert pk.roles.store

        # ensure that we dont hit an error here
        create_default_projects(created_models=[Project])
<|endoftext|>


<commit_before>from django import template
from django.conf import settings
from django.shortcuts import render_to_response
from django.template import RequestContext

from wagtail.wagtailcore import models
from wagtail.wagtailcore.models import Page
from wagtail.wagtailimages.models import Image
from wagtail.wagtailsearch.index import get_indexed_models
from wagtail.wagtailsearch.backends import get_search_backend

from portal.base.models import PortalImage
from portal.ocean_stories.models import OceanStory, OceanStories
from portal.calendar.models import Event
from portal.news.models import News, Story

from data_manager.models import Layer, Theme

register = template.Library()

def search(request, template=settings.WAGTAILSEARCH_RESULTS_TEMPLATE):
    query_string = request.GET.get('q', '')

    ocean_story_results = []
    calendar_news_results = []
    data_needs_results = []
    resources_results = []
    theme_results = []
    layer_results = []

    if len(query_string) >= 2:
        s = get_search_backend()
        models = get_indexed_models()
        # remove unnecessary models
        for i in [Page, Image, PortalImage]:
            models.remove(i)

        # search wagtail pages
        for model in models:
            results = s.search(query_string, model)
            for item in results:
                if isinstance(item, (OceanStory, OceanStories)):
                    ocean_story_results.append(item)
                elif isinstance(item, (Event, News, Story)):
                    calendar_news_results.append(item)
                elif '/data-needs-and-priorities/' in item.url:
                    data_needs_results.append(item)
                elif '/resources/' in item.url:
                    resources_results.append(item)

        # search themes from data_catalog
        for theme in Theme.objects.filter(visible=True, display_name__icontains=query_string):
            theme_results.append(theme)

        # search layers from data_catalog
        layer_results.extend(Layer.objects.exclude(layer_type='placeholder').filter(themes__visible=True, name__icontains=query_string))

    return render_to_response(template, RequestContext(request, {
        'ocean_story_results': ocean_story_results,
        'calendar_news_results': calendar_news_results,
        'data_needs_results': data_needs_results,
        'resources_results': resources_results,
        'theme_results': theme_results,
        'layer_results': layer_results
    }));<commit_msg>fixing bug in search<commit_after>from django import template
from django.conf import settings
from django.shortcuts import render_to_response
from django.template import RequestContext

from wagtail.wagtailcore import models
from wagtail.wagtailcore.models import Page
from wagtail.wagtailimages.models import Image
from wagtail.wagtailsearch.index import get_indexed_models
from wagtail.wagtailsearch.backends import get_search_backend

from portal.base.models import PortalImage
from portal.ocean_stories.models import OceanStory, OceanStories
from portal.calendar.models import Event
from portal.news.models import News, Story

from data_manager.models import Layer, Theme

register = template.Library()

def search(request, template=settings.WAGTAILSEARCH_RESULTS_TEMPLATE):
    query_string = request.GET.get('q', '')

    ocean_story_results = []
    calendar_news_results = []
    data_needs_results = []
    resources_results = []
    theme_results = []
    layer_results = []

    if len(query_string) >= 2:
        s = get_search_backend()
        models = get_indexed_models()
        # remove unnecessary models
        for i in [Page, Image, PortalImage]:
            models.remove(i)

        # search wagtail pages
        for model in models:
            results = s.search(query_string, model)
            for item in results:
                if isinstance(item, (OceanStory, OceanStories)):
                    ocean_story_results.append(item)
                elif isinstance(item, (Event, News, Story)):
                    calendar_news_results.append(item)
                elif item.url and '/data-needs-and-priorities/' in item.url:
                    data_needs_results.append(item)
                elif item.url and '/resources/' in item.url:
                    resources_results.append(item)

        # search themes from data_catalog
        for theme in Theme.objects.filter(visible=True, display_name__icontains=query_string):
            theme_results.append(theme)

        # search layers from data_catalog
        layer_results.extend(Layer.objects.exclude(layer_type='placeholder').filter(themes__visible=True, name__icontains=query_string))

    return render_to_response(template, RequestContext(request, {
        'ocean_story_results': ocean_story_results,
        'calendar_news_results': calendar_news_results,
        'data_needs_results': data_needs_results,
        'resources_results': resources_results,
        'theme_results': theme_results,
        'layer_results': layer_results
    }));
<|endoftext|>