<commit_before>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Tavendo GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from __future__ import absolute_import

from autobahn.wamp import protocol
from autobahn.wamp.types import ComponentConfig
from autobahn.websocket.protocol import parseWsUrl
from autobahn.asyncio.websocket import WampWebSocketClientFactory

try:
    import asyncio
except ImportError:
    # Trollius >= 0.3 was renamed to asyncio
    # noinspection PyUnresolvedReferences
    import trollius as asyncio

import txaio
txaio.use_asyncio()

__all__ = (
    'ApplicationSession',
    'ApplicationSessionFactory',
    'ApplicationRunner'
)

class ApplicationSession(protocol.ApplicationSession):
    """
    WAMP application session for asyncio-based applications.
    """

class ApplicationSessionFactory(protocol.ApplicationSessionFactory):
    """
    WAMP application session factory for asyncio-based applications.
    """

    session = ApplicationSession
    """
   The application session class this application session factory will use. Defaults to :class:`autobahn.asyncio.wamp.ApplicationSession`.
   """

class ApplicationRunner(object):
    """
    This class is a convenience tool mainly for development and quick hosting
    of WAMP application components.

    It can host a WAMP application component in a WAMP-over-WebSocket client
    connecting to a WAMP router.
    """

    def __init__(self, url, realm, extra=None, serializers=None,
                 debug=False, debug_wamp=False, debug_app=False,
                 ssl=None):
        """

        :param url: The WebSocket URL of the WAMP router to connect to (e.g. `ws://somehost.com:8090/somepath`)
        :type url: unicode
        :param realm: The WAMP realm to join the application session to.
        :type realm: unicode
        :param extra: Optional extra configuration to forward to the application component.
        :type extra: dict
        :param serializers: A list of WAMP serializers to use (or None for default serializers).
           Serializers must implement :class:`autobahn.wamp.interfaces.ISerializer`.
        :type serializers: list
        :param debug: Turn on low-level debugging.
        :type debug: bool
        :param debug_wamp: Turn on WAMP-level debugging.
        :type debug_wamp: bool
        :param debug_app: Turn on app-level debugging.
        :type debug_app: bool
        :param ssl: An (optional) SSL context instance or a bool. See the documentation for the
           `loop.create_connection` asyncio method, to which this value is passed.
        :type ssl: :class`ssl.SSLContext`
        """
        self.url = url
        self.realm = realm
        self.extra = extra or dict()
        self.debug = debug
        self.debug_wamp = debug_wamp
        self.debug_app = debug_app
        self.make = None
        self.serializers = serializers
        self.ssl = ssl

    def run(self, make):
        """
        Run the application component.

        :param make: A factory that produces instances of :class:`autobahn.asyncio.wamp.ApplicationSession`
           when called with an instance of :class:`autobahn.wamp.types.ComponentConfig`.
        :type make: callable
        """
        # 1) factory for use ApplicationSession
        def create():
            cfg = ComponentConfig(self.realm, self.extra)
            try:
                session = make(cfg)
            except Exception as e:
                # the app component could not be created .. fatal
                print(e)
                asyncio.get_event_loop().stop()
            else:
                session.debug_app = self.debug_app
                return session

        isSecure, host, port, resource, path, params = parseWsUrl(self.url)

        if self.ssl is None:
            ssl = isSecure
        else:
            if self.ssl and not isSecure:
                raise Exception(
                    'ssl argument value passed to %s conflicts with the "ws:" '
                    'prefix of the url argument. Did you mean to use "wss:"?' %
                    self.__class__.__name__)
            ssl = self.ssl

        # 2) create a WAMP-over-WebSocket transport client factory
        transport_factory = WampWebSocketClientFactory(create, url=self.url, serializers=self.serializers,
                                                       debug=self.debug, debug_wamp=self.debug_wamp)

        # 3) start the client
        loop = asyncio.get_event_loop()
        txaio.use_asyncio()
        txaio.config.loop = loop
        coro = loop.create_connection(transport_factory, host, port, ssl=ssl)
        loop.run_until_complete(coro)

        # 4) now enter the asyncio event loop
        loop.run_forever()
        loop.close()
<commit_msg>add whitespace to make comment more readable and clarify type<commit_after>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Tavendo GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from __future__ import absolute_import

from autobahn.wamp import protocol
from autobahn.wamp.types import ComponentConfig
from autobahn.websocket.protocol import parseWsUrl
from autobahn.asyncio.websocket import WampWebSocketClientFactory

try:
    import asyncio
except ImportError:
    # Trollius >= 0.3 was renamed to asyncio
    # noinspection PyUnresolvedReferences
    import trollius as asyncio

import txaio
txaio.use_asyncio()

__all__ = (
    'ApplicationSession',
    'ApplicationSessionFactory',
    'ApplicationRunner'
)

class ApplicationSession(protocol.ApplicationSession):
    """
    WAMP application session for asyncio-based applications.
    """

class ApplicationSessionFactory(protocol.ApplicationSessionFactory):
    """
    WAMP application session factory for asyncio-based applications.
    """

    session = ApplicationSession
    """
   The application session class this application session factory will use. Defaults to :class:`autobahn.asyncio.wamp.ApplicationSession`.
   """

class ApplicationRunner(object):
    """
    This class is a convenience tool mainly for development and quick hosting
    of WAMP application components.

    It can host a WAMP application component in a WAMP-over-WebSocket client
    connecting to a WAMP router.
    """

    def __init__(self, url, realm, extra=None, serializers=None,
                 debug=False, debug_wamp=False, debug_app=False,
                 ssl=None):
        """
        :param url: The WebSocket URL of the WAMP router to connect to (e.g. `ws://somehost.com:8090/somepath`)
        :type url: unicode

        :param realm: The WAMP realm to join the application session to.
        :type realm: unicode

        :param extra: Optional extra configuration to forward to the application component.
        :type extra: dict

        :param serializers: A list of WAMP serializers to use (or None for default serializers).
           Serializers must implement :class:`autobahn.wamp.interfaces.ISerializer`.
        :type serializers: list

        :param debug: Turn on low-level debugging.
        :type debug: bool

        :param debug_wamp: Turn on WAMP-level debugging.
        :type debug_wamp: bool

        :param debug_app: Turn on app-level debugging.
        :type debug_app: bool

        :param ssl: An (optional) SSL context instance or a bool. See
           the documentation for the `loop.create_connection` asyncio
           method, to which this value is passed as the ``ssl=``
           kwarg.
        :type ssl: :class:`ssl.SSLContext` or bool
        """
        self.url = url
        self.realm = realm
        self.extra = extra or dict()
        self.debug = debug
        self.debug_wamp = debug_wamp
        self.debug_app = debug_app
        self.make = None
        self.serializers = serializers
        self.ssl = ssl

    def run(self, make):
        """
        Run the application component.

        :param make: A factory that produces instances of :class:`autobahn.asyncio.wamp.ApplicationSession`
           when called with an instance of :class:`autobahn.wamp.types.ComponentConfig`.
        :type make: callable
        """
        # 1) factory for use ApplicationSession
        def create():
            cfg = ComponentConfig(self.realm, self.extra)
            try:
                session = make(cfg)
            except Exception as e:
                # the app component could not be created .. fatal
                print(e)
                asyncio.get_event_loop().stop()
            else:
                session.debug_app = self.debug_app
                return session

        isSecure, host, port, resource, path, params = parseWsUrl(self.url)

        if self.ssl is None:
            ssl = isSecure
        else:
            if self.ssl and not isSecure:
                raise Exception(
                    'ssl argument value passed to %s conflicts with the "ws:" '
                    'prefix of the url argument. Did you mean to use "wss:"?' %
                    self.__class__.__name__)
            ssl = self.ssl

        # 2) create a WAMP-over-WebSocket transport client factory
        transport_factory = WampWebSocketClientFactory(create, url=self.url, serializers=self.serializers,
                                                       debug=self.debug, debug_wamp=self.debug_wamp)

        # 3) start the client
        loop = asyncio.get_event_loop()
        txaio.use_asyncio()
        txaio.config.loop = loop
        coro = loop.create_connection(transport_factory, host, port, ssl=ssl)
        loop.run_until_complete(coro)

        # 4) now enter the asyncio event loop
        loop.run_forever()
        loop.close()
<|endoftext|>


<commit_before>import logging; _L = logging.getLogger('openaddr.ci.webapi')

from urllib.parse import urljoin
from operator import attrgetter
from collections import defaultdict
import json, os

from flask import Response, Blueprint, request, current_app, jsonify, url_for, redirect
from flask_cors import CORS

from .objects import (
    load_collection_zips_dict, read_latest_set, read_completed_runs_to_date,
    new_read_completed_set_runs, read_set
    )

from . import setup_logger, db_connect, db_cursor, tileindex
from .webcommon import log_application_errors, nice_domain, flask_log_level
from ..compat import expand_uri, csvIO, csvDictWriter
from .. import compat

CSV_HEADER = 'source', 'cache', 'sample', 'geometry type', 'address count', \
             'version', 'fingerprint', 'cache time', 'processed', 'process time', \
             'process hash', 'output', 'attribution required', 'attribution name', \
             'share-alike', 'code version'

webapi = Blueprint('webapi', __name__)
CORS(webapi)

@webapi.route('/index.json')
@log_application_errors
def app_index_json():
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            zips = load_collection_zips_dict(db)
            set = read_latest_set(db, 'openaddresses', 'openaddresses')

    collections = {}
    licenses = {'': 'Freely Shareable', 'sa': 'Share-Alike Required'}

    for ((collection, license), zip) in zips.items():
        if zip.content_length < 1024:
            # too small, probably empty
            continue

        if collection not in collections:
            collections[collection] = dict()

        if license not in collections[collection]:
            collections[collection][license] = dict()

        d = dict(url=nice_domain(zip.url), content_length=zip.content_length)
        d['license'] = licenses[license]
        collections[collection][license] = d

    run_states_url = url_for('webapi.app_get_state_txt')
    latest_run_processed_url = url_for('webhooks.app_get_latest_run', source='____').replace('____', '{source}')
    tileindex_url = url_for('webapi.app_get_tileindex_zip', lon='xxx', lat='yyy').replace('xxx', '{lon}').replace('yyy', '{lat}')
    licenses_url = url_for('webapi.app_licenses_json')
    latest_set_url = url_for('webapi.app_get_set_data', set_id=set.id)

    if compat.PY2:
        run_states_url = run_states_url.decode('utf8')
        latest_run_processed_url = latest_run_processed_url.decode('utf8')
        tileindex_url = tileindex_url.decode('utf8')
        licenses_url = licenses_url.decode('utf8')

    render_world_url = 'https://s3.amazonaws.com/{}/render-world.png'.format(current_app.config['AWS_S3_BUCKET'])
    render_europe_url = 'https://s3.amazonaws.com/{}/render-europe.png'.format(current_app.config['AWS_S3_BUCKET'])
    render_usa_url = 'https://s3.amazonaws.com/{}/render-usa.png'.format(current_app.config['AWS_S3_BUCKET'])
    render_geojson_url = 'https://s3.amazonaws.com/{}/render-world.geojson'.format(current_app.config['AWS_S3_BUCKET'])

    return jsonify({
        'run_states_url': urljoin(request.url, run_states_url),
        'latest_run_processed_url': urljoin(request.url, latest_run_processed_url),
        'tileindex_url': urljoin(request.url, tileindex_url),
        'licenses_url': urljoin(request.url, licenses_url),
        'latest_set_url': urljoin(request.url, latest_set_url),
        'render_world_url': nice_domain(render_world_url),
        'render_europe_url': nice_domain(render_europe_url),
        'render_usa_url': nice_domain(render_usa_url),
        'render_geojson_url': nice_domain(render_geojson_url),
        'collections': collections
        })

@webapi.route('/latest/licenses.json')
@log_application_errors
def app_licenses_json():
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            set = read_latest_set(db, 'openaddresses', 'openaddresses')
            runs = read_completed_runs_to_date(db, set.id)

    licenses = defaultdict(list)

    for run in runs:
        run_state = run.state or {}
        source = os.path.relpath(run.source_path, 'sources')

        attribution = None
        if run_state.attribution_required != 'false':
            attribution = run_state.attribution_name

        key = run_state.license, attribution
        licenses[key].append((source, run_state.website))

    licenses = [dict(license=lic, attribution=attr, sources=sorted(srcs))
                for ((lic, attr), srcs) in sorted(licenses.items(), key=repr)]

    return jsonify(licenses=licenses)

@webapi.route('/state.txt', methods=['GET'])
@log_application_errors
def app_get_state_txt():
    '''
    '''
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            set = read_latest_set(db, 'openaddresses', 'openaddresses')
            runs = read_completed_runs_to_date(db, set.id)

    buffer = csvIO()
    output = csvDictWriter(buffer, CSV_HEADER, dialect='excel-tab', encoding='utf8')
    output.writerow({col: col for col in CSV_HEADER})
    for run in sorted(runs, key=attrgetter('source_path')):
        run_state = run.state or {}
        row = {col: run_state.get(col) for col in CSV_HEADER}
        row['source'] = os.path.relpath(run.source_path, 'sources')
        row['code version'] = run.code_version
        row['cache'] = nice_domain(row['cache'])
        row['sample'] = nice_domain(row['sample'])
        row['processed'] = nice_domain(row['processed'])
        row['output'] = nice_domain(row['output'])
        output.writerow(row)

    return Response(buffer.getvalue(),
                    headers={'Content-Type': 'text/plain; charset=utf8'})

@webapi.route('/sets/<set_id>/state.txt', methods=['GET'])
@log_application_errors
def app_get_set_state_txt(set_id):
    '''
    '''
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            runs = new_read_completed_set_runs(db, set_id)

    buffer = csvIO()
    output = csvDictWriter(buffer, CSV_HEADER, dialect='excel-tab', encoding='utf8')
    output.writerow({col: col for col in CSV_HEADER})
    for run in sorted(runs, key=attrgetter('source_path')):
        run_state = run.state or {}
        row = {col: run_state.get(col) for col in CSV_HEADER}
        row['source'] = os.path.relpath(run.source_path, 'sources')
        row['code version'] = run.code_version
        row['cache'] = nice_domain(row['cache'])
        row['sample'] = nice_domain(row['sample'])
        row['processed'] = nice_domain(row['processed'])
        row['output'] = nice_domain(row['output'])
        output.writerow(row)

    return Response(buffer.getvalue(),
                    headers={'Content-Type': 'text/plain; charset=utf8'})

@webapi.route('/sets/<set_id>.json', methods=['GET'])
@log_application_errors
def app_get_set_data(set_id):
    '''
    '''
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            set = read_set(db, set_id)

    return jsonify({
        'id': set.id,
        'commit_sha': set.commit_sha,
        'datetime_start': str(set.datetime_start),
        'datetime_end': str(set.datetime_end),

        'render_world_url': nice_domain(set.render_world),
        'render_europe_url': nice_domain(set.render_europe),
        'render_usa_url': nice_domain(set.render_usa),
        'render_geojson_url': nice_domain(set.render_geojson),
        })

@webapi.route('/tiles/<lon>/<lat>.zip', methods=['GET'])
@log_application_errors
def app_get_tileindex_zip(lon, lat):
    '''
    '''
    try:
        key = tileindex.lonlat_key(float(lon), float(lat))
    except ValueError:
        return Response('"{}" and "{}" must both be numeric.\n'.format(lon, lat), status=404)

    if not (-180 <= key[0] <= 180 and -90 <= key[1] <= 90):
        return Response('"{}" and "{}" must both be on earth.\n'.format(lon, lat), status=404)

    bucket = current_app.config['AWS_S3_BUCKET']
    url = u'https://s3.amazonaws.com/{}/tiles/{:.1f}/{:.1f}.zip'.format(bucket, *key)
    return redirect(nice_domain(url), 302)

def apply_webapi_blueprint(app):
    '''
    '''
    app.register_blueprint(webapi)

    @app.before_first_request
    def app_prepare():
        setup_logger(None,
                     None,
                     os.environ.get('AWS_SNS_ARN'),
                     flask_log_level(app.config))
<commit_msg>Fixed Python 2 errors<commit_after>import logging; _L = logging.getLogger('openaddr.ci.webapi')

from urllib.parse import urljoin
from operator import attrgetter
from collections import defaultdict
import json, os

from flask import Response, Blueprint, request, current_app, jsonify, url_for, redirect
from flask_cors import CORS

from .objects import (
    load_collection_zips_dict, read_latest_set, read_completed_runs_to_date,
    new_read_completed_set_runs, read_set
    )

from . import setup_logger, db_connect, db_cursor, tileindex
from .webcommon import log_application_errors, nice_domain, flask_log_level
from ..compat import expand_uri, csvIO, csvDictWriter
from .. import compat

CSV_HEADER = 'source', 'cache', 'sample', 'geometry type', 'address count', \
             'version', 'fingerprint', 'cache time', 'processed', 'process time', \
             'process hash', 'output', 'attribution required', 'attribution name', \
             'share-alike', 'code version'

webapi = Blueprint('webapi', __name__)
CORS(webapi)

@webapi.route('/index.json')
@log_application_errors
def app_index_json():
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            zips = load_collection_zips_dict(db)
            set = read_latest_set(db, 'openaddresses', 'openaddresses')

    collections = {}
    licenses = {'': 'Freely Shareable', 'sa': 'Share-Alike Required'}

    for ((collection, license), zip) in zips.items():
        if zip.content_length < 1024:
            # too small, probably empty
            continue

        if collection not in collections:
            collections[collection] = dict()

        if license not in collections[collection]:
            collections[collection][license] = dict()

        d = dict(url=nice_domain(zip.url), content_length=zip.content_length)
        d['license'] = licenses[license]
        collections[collection][license] = d

    run_states_url = url_for('webapi.app_get_state_txt')
    latest_run_processed_url = url_for('webhooks.app_get_latest_run', source='____').replace('____', '{source}')
    tileindex_url = url_for('webapi.app_get_tileindex_zip', lon='xxx', lat='yyy').replace('xxx', '{lon}').replace('yyy', '{lat}')
    licenses_url = url_for('webapi.app_licenses_json')
    latest_set_url = url_for('webapi.app_get_set_data', set_id=set.id)

    render_world_url = 'https://s3.amazonaws.com/{}/render-world.png'.format(current_app.config['AWS_S3_BUCKET'])
    render_europe_url = 'https://s3.amazonaws.com/{}/render-europe.png'.format(current_app.config['AWS_S3_BUCKET'])
    render_usa_url = 'https://s3.amazonaws.com/{}/render-usa.png'.format(current_app.config['AWS_S3_BUCKET'])
    render_geojson_url = 'https://s3.amazonaws.com/{}/render-world.geojson'.format(current_app.config['AWS_S3_BUCKET'])

    if compat.PY2:
        run_states_url = run_states_url.decode('utf8')
        latest_run_processed_url = latest_run_processed_url.decode('utf8')
        tileindex_url = tileindex_url.decode('utf8')
        licenses_url = licenses_url.decode('utf8')
        latest_set_url = latest_set_url.decode('utf8')

        render_world_url = render_world_url.decode('utf8')
        render_europe_url = render_europe_url.decode('utf8')
        render_usa_url = render_usa_url.decode('utf8')
        render_geojson_url = render_geojson_url.decode('utf8')

    return jsonify({
        'run_states_url': urljoin(request.url, run_states_url),
        'latest_run_processed_url': urljoin(request.url, latest_run_processed_url),
        'tileindex_url': urljoin(request.url, tileindex_url),
        'licenses_url': urljoin(request.url, licenses_url),
        'latest_set_url': urljoin(request.url, latest_set_url),
        'render_world_url': nice_domain(render_world_url),
        'render_europe_url': nice_domain(render_europe_url),
        'render_usa_url': nice_domain(render_usa_url),
        'render_geojson_url': nice_domain(render_geojson_url),
        'collections': collections
        })

@webapi.route('/latest/licenses.json')
@log_application_errors
def app_licenses_json():
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            set = read_latest_set(db, 'openaddresses', 'openaddresses')
            runs = read_completed_runs_to_date(db, set.id)

    licenses = defaultdict(list)

    for run in runs:
        run_state = run.state or {}
        source = os.path.relpath(run.source_path, 'sources')

        attribution = None
        if run_state.attribution_required != 'false':
            attribution = run_state.attribution_name

        key = run_state.license, attribution
        licenses[key].append((source, run_state.website))

    licenses = [dict(license=lic, attribution=attr, sources=sorted(srcs))
                for ((lic, attr), srcs) in sorted(licenses.items(), key=repr)]

    return jsonify(licenses=licenses)

@webapi.route('/state.txt', methods=['GET'])
@log_application_errors
def app_get_state_txt():
    '''
    '''
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            set = read_latest_set(db, 'openaddresses', 'openaddresses')
            runs = read_completed_runs_to_date(db, set.id)

    buffer = csvIO()
    output = csvDictWriter(buffer, CSV_HEADER, dialect='excel-tab', encoding='utf8')
    output.writerow({col: col for col in CSV_HEADER})
    for run in sorted(runs, key=attrgetter('source_path')):
        run_state = run.state or {}
        row = {col: run_state.get(col) for col in CSV_HEADER}
        row['source'] = os.path.relpath(run.source_path, 'sources')
        row['code version'] = run.code_version
        row['cache'] = nice_domain(row['cache'])
        row['sample'] = nice_domain(row['sample'])
        row['processed'] = nice_domain(row['processed'])
        row['output'] = nice_domain(row['output'])
        output.writerow(row)

    return Response(buffer.getvalue(),
                    headers={'Content-Type': 'text/plain; charset=utf8'})

@webapi.route('/sets/<set_id>/state.txt', methods=['GET'])
@log_application_errors
def app_get_set_state_txt(set_id):
    '''
    '''
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            runs = new_read_completed_set_runs(db, set_id)

    buffer = csvIO()
    output = csvDictWriter(buffer, CSV_HEADER, dialect='excel-tab', encoding='utf8')
    output.writerow({col: col for col in CSV_HEADER})
    for run in sorted(runs, key=attrgetter('source_path')):
        run_state = run.state or {}
        row = {col: run_state.get(col) for col in CSV_HEADER}
        row['source'] = os.path.relpath(run.source_path, 'sources')
        row['code version'] = run.code_version
        row['cache'] = nice_domain(row['cache'])
        row['sample'] = nice_domain(row['sample'])
        row['processed'] = nice_domain(row['processed'])
        row['output'] = nice_domain(row['output'])
        output.writerow(row)

    return Response(buffer.getvalue(),
                    headers={'Content-Type': 'text/plain; charset=utf8'})

@webapi.route('/sets/<set_id>.json', methods=['GET'])
@log_application_errors
def app_get_set_data(set_id):
    '''
    '''
    with db_connect(current_app.config['DATABASE_URL']) as conn:
        with db_cursor(conn) as db:
            set = read_set(db, set_id)

    return jsonify({
        'id': set.id,
        'commit_sha': set.commit_sha,
        'datetime_start': str(set.datetime_start),
        'datetime_end': str(set.datetime_end),

        'render_world_url': nice_domain(set.render_world),
        'render_europe_url': nice_domain(set.render_europe),
        'render_usa_url': nice_domain(set.render_usa),
        'render_geojson_url': nice_domain(set.render_geojson),
        })

@webapi.route('/tiles/<lon>/<lat>.zip', methods=['GET'])
@log_application_errors
def app_get_tileindex_zip(lon, lat):
    '''
    '''
    try:
        key = tileindex.lonlat_key(float(lon), float(lat))
    except ValueError:
        return Response('"{}" and "{}" must both be numeric.\n'.format(lon, lat), status=404)

    if not (-180 <= key[0] <= 180 and -90 <= key[1] <= 90):
        return Response('"{}" and "{}" must both be on earth.\n'.format(lon, lat), status=404)

    bucket = current_app.config['AWS_S3_BUCKET']
    url = u'https://s3.amazonaws.com/{}/tiles/{:.1f}/{:.1f}.zip'.format(bucket, *key)
    return redirect(nice_domain(url), 302)

def apply_webapi_blueprint(app):
    '''
    '''
    app.register_blueprint(webapi)

    @app.before_first_request
    def app_prepare():
        setup_logger(None,
                     None,
                     os.environ.get('AWS_SNS_ARN'),
                     flask_log_level(app.config))
<|endoftext|>


<commit_before># Copyright ClusterHQ Inc.  See LICENSE file for details.

"""
Tests for the datasets REST API.
"""

from uuid import UUID, uuid4

from twisted.internet.task import deferLater
from twisted.trial.unittest import TestCase
from twisted.internet import reactor

from docker.utils import create_host_config

from ...testtools import random_name
from ..testtools import (
    require_cluster, require_moving_backend, create_dataset,
    REALISTIC_BLOCKDEVICE_SIZE, get_docker_client,
    post_http_server, query_http_server, assert_http_server,
)

from ..scripts import SCRIPTS

class LeaseAPITests(TestCase):
    """
    Tests for the leases API.
    """
    @require_moving_backend
    @require_cluster(2)
    def test_lease_prevents_move(self, cluster):
        """
        A dataset cannot be moved if a lease is held on
        it by a particular node.
        """
        http_port = 8080
        dataset_id = uuid4()
        dataset_path = []
        client = get_docker_client(cluster, cluster.nodes[0].public_address)
        d = create_dataset(
            self, cluster, maximum_size=REALISTIC_BLOCKDEVICE_SIZE,
            dataset_id=dataset_id
        )

        def acquire_lease(dataset):
            # Call the API to acquire a lease with the dataset ID.
            acquiring_lease = cluster.client.acquire_lease(
                dataset.dataset_id, UUID(cluster.nodes[0].uuid), expires=1000)

            def get_dataset_path(lease, created_dataset):
                getting_datasets = cluster.client.list_datasets_state()

                def extract_dataset_path(datasets):
                    dataset_path.insert(0, datasets[0].path)
                    return datasets[0].path

                getting_datasets.addCallback(extract_dataset_path)
                return getting_datasets

            acquiring_lease.addCallback(get_dataset_path, dataset)
            return acquiring_lease

        d.addCallback(acquire_lease)

        def start_http_container(dataset_path, client):
            # Launch data HTTP container and make POST requests
            # to it in a looping call every second.
            # return looping call deferred
            # import pdb;pdb.set_trace()
            script = SCRIPTS.child("datahttp.py")
            script_arguments = [u"/data"]
            docker_arguments = {
                "host_config": create_host_config(
                    binds=["{}:/data".format(dataset_path.path)],
                    port_bindings={http_port: http_port}),
                "ports": [http_port],
                "volumes": [u"/data"]}
            container = client.create_container(
                "python:2.7-slim",
                ["python", "-c", script.getContent()] + list(script_arguments),
                **docker_arguments)
            cid = container["Id"]
            client.start(container=cid)
            self.addCleanup(client.remove_container, cid, force=True)
            return cid

        d.addCallback(start_http_container, client)

        def write_data(container_id):
            data = random_name(self).encode("utf-8")
            writing = post_http_server(
                self, cluster.nodes[0].public_address, http_port,
                {"data": data}
            )
            writing.addCallback(
                lambda _: assert_http_server(
                    self, cluster.nodes[0].public_address,
                    http_port, expected_response=data
                )
            )

            writing.addCallback(lambda _: container_id)
            return writing

        d.addCallback(write_data)

        def stop_container(container_id, client, dataset_id):
            primary = cluster.nodes[1].uuid
            # This ensures Docker hasn't got a lock on the volume that
            # might prevent it being moved separate to the lock held by
            # the lease.
            client.stop(container_id)
            move_dataset_request = cluster.client.move_dataset(
                primary, dataset_id)
            move_dataset_request.addCallback(lambda _: container_id)
            return move_dataset_request

        d.addCallback(stop_container, client, dataset_id)

        def wait_five_seconds(container_id):
            return deferLater(reactor, 10, lambda: container_id)

        d.addCallback(wait_five_seconds)

        def restart_container(container_id, client, cluster):
            client.start(container=container_id)
            return container_id

        d.addCallback(restart_container, client, cluster)

        d.addCallback(write_data)

        def stop_container_again(container_id, client, dataset_id):
            client.stop(container_id)
            releasing = cluster.client.release_lease(dataset_id)
            releasing.addCallback(lambda _: container_id)
            return releasing

        d.addCallback(stop_container_again, client, dataset_id)

        d.addCallback(wait_five_seconds)

        # d.addCallback(restart_container, client, cluster)

        d.addCallback(lambda _: start_http_container(dataset_path[0], client))

        def container_did_start(container_id):
            check_datasets_config = cluster.client.list_datasets_configuration()
            def got_datasets_config(datasets):
                #ncluster = cluster
                #import pdb;pdb.set_trace()
                get_ds_state = cluster.client.list_datasets_state()
                def got_ds_state(state_datasets):
                    config_datasets = datasets
                    ncluster = cluster
                    # import pdb;pdb.set_trace()
                get_ds_state.addCallback(got_ds_state)
                return get_ds_state
            check_datasets_config.addCallback(got_datasets_config)
            #import pdb;pdb.set_trace()
            #self.fail("The container didn't fail to start :'(")
            return check_datasets_config

        def container_no_start(some_failure):
            # import pdb;pdb.set_trace()
            pass

        d.addCallback(container_did_start)
        d.addErrback(container_no_start)

        return d

    @require_moving_backend
    @require_cluster(2)
    def test_lease_prevents_delete(self, cluster):
        """
        A dataset cannot be deleted if a lease is held on
        it by a particular node.
        """
        self.fail("not implemented yet")

    @require_moving_backend
    @require_cluster(2)
    def test_delete_dataset_after_lease_release(self, cluster):
        """
        A dataset can be deleted once a lease held on it by a
        particular node is released.
        """
        self.fail("not implemented yet")

    @require_moving_backend
    @require_cluster(2)
    def test_delete_dataset_after_lease_expiry(self, cluster):
        """
        A dataset can be deleted once a lease held on it by a
        particular node has expired.
        """
        self.fail("not implemented yet")
<commit_msg>working test :)<commit_after># Copyright ClusterHQ Inc.  See LICENSE file for details.

"""
Tests for the datasets REST API.
"""

from uuid import UUID, uuid4

from twisted.internet.task import deferLater
from twisted.trial.unittest import TestCase
from twisted.internet import reactor

from docker.utils import create_host_config

from ...testtools import random_name
from ..testtools import (
    require_cluster, require_moving_backend, create_dataset,
    REALISTIC_BLOCKDEVICE_SIZE, get_docker_client,
    post_http_server, assert_http_server,
)

from ..scripts import SCRIPTS

class LeaseAPITests(TestCase):
    """
    Tests for the leases API.
    """
    @require_moving_backend
    @require_cluster(2)
    def test_lease_prevents_move(self, cluster):
        """
        A dataset cannot be moved if a lease is held on
        it by a particular node.
        """
        http_port = 8080
        dataset_id = uuid4()
        dataset_path = []
        client = get_docker_client(cluster, cluster.nodes[0].public_address)
        d = create_dataset(
            self, cluster, maximum_size=REALISTIC_BLOCKDEVICE_SIZE,
            dataset_id=dataset_id
        )

        def acquire_lease(dataset):
            # Call the API to acquire a lease with the dataset ID.
            acquiring_lease = cluster.client.acquire_lease(
                dataset.dataset_id, UUID(cluster.nodes[0].uuid), expires=1000)

            def get_dataset_path(lease, created_dataset):
                getting_datasets = cluster.client.list_datasets_state()

                def extract_dataset_path(datasets):
                    dataset_path.insert(0, datasets[0].path)
                    return datasets[0].path

                getting_datasets.addCallback(extract_dataset_path)
                return getting_datasets

            acquiring_lease.addCallback(get_dataset_path, dataset)
            return acquiring_lease

        d.addCallback(acquire_lease)

        def start_http_container(dataset_path, client):
            # Launch data HTTP container and make POST requests
            # to it in a looping call every second.
            # return looping call deferred
            # import pdb;pdb.set_trace()
            script = SCRIPTS.child("datahttp.py")
            script_arguments = [u"/data"]
            docker_arguments = {
                "host_config": create_host_config(
                    binds=["{}:/data".format(dataset_path.path)],
                    port_bindings={http_port: http_port}),
                "ports": [http_port],
                "volumes": [u"/data"]}
            container = client.create_container(
                "python:2.7-slim",
                ["python", "-c", script.getContent()] + list(script_arguments),
                **docker_arguments)
            cid = container["Id"]
            client.start(container=cid)
            self.addCleanup(client.remove_container, cid, force=True)
            return cid

        d.addCallback(start_http_container, client)

        def write_data(container_id):
            data = random_name(self).encode("utf-8")
            writing = post_http_server(
                self, cluster.nodes[0].public_address, http_port,
                {"data": data}
            )
            writing.addCallback(
                lambda _: assert_http_server(
                    self, cluster.nodes[0].public_address,
                    http_port, expected_response=data
                )
            )

            writing.addCallback(lambda _: container_id)
            return writing

        d.addCallback(write_data)

        def stop_container(container_id, client, dataset_id):
            primary = cluster.nodes[1].uuid
            # This ensures Docker hasn't got a lock on the volume that
            # might prevent it being moved separate to the lock held by
            # the lease.
            client.stop(container_id)
            move_dataset_request = cluster.client.move_dataset(
                primary, dataset_id)
            move_dataset_request.addCallback(lambda _: container_id)
            return move_dataset_request

        d.addCallback(stop_container, client, dataset_id)

        def wait_seconds(container_id):
            return deferLater(reactor, 10, lambda: container_id)

        d.addCallback(wait_seconds)

        def restart_container(container_id, client, cluster):
            client.start(container=container_id)
            return container_id

        d.addCallback(restart_container, client, cluster)

        d.addCallback(write_data)

        def stop_container_again(container_id, client, dataset_id):
            # At this point, we expect the configured dataset to reflect
            # the move from node0 to node1, but the state to retain the
            # dataset on node0.
            check_datasets_config = (
                cluster.client.list_datasets_configuration()
            )

            def got_datasets_config(datasets):
                get_ds_state = cluster.client.list_datasets_state()

                def got_ds_state(state_datasets):
                    config_datasets = datasets
                    expected_state_node = unicode(cluster.nodes[0].uuid)
                    expected_config_node = unicode(cluster.nodes[1].uuid)
                    self.assertEqual(
                        (
                            unicode(state_datasets[0].primary),
                            unicode(config_datasets[0].primary)
                        ),
                        (expected_state_node, expected_config_node))
                get_ds_state.addCallback(got_ds_state)
                return get_ds_state
            check_datasets_config.addCallback(got_datasets_config)

            def stop_and_release(_):
                client.stop(container_id)
                releasing = cluster.client.release_lease(dataset_id)
                releasing.addCallback(lambda _: container_id)
                # Now we've released the lease and stopped the running
                # container, our earlier move request should be enacted
                # after a short delay.
                return releasing

            check_datasets_config.addCallback(stop_and_release)

            return check_datasets_config

        d.addCallback(stop_container_again, client, dataset_id)

        d.addCallback(wait_seconds)

        def dataset_moved(_):
            # So at this point, we want to confirm the dataset has now moved.
            check_datasets_config = (
                cluster.client.list_datasets_configuration()
            )

            def got_datasets_config(datasets):
                get_ds_state = cluster.client.list_datasets_state()

                def got_ds_state(state_datasets):
                    config_datasets = datasets
                    expected_node = unicode(cluster.nodes[1].uuid)
                    self.assertEqual(
                        (
                            unicode(state_datasets[0].primary),
                            unicode(config_datasets[0].primary)
                        ),
                        (expected_node, expected_node))
                get_ds_state.addCallback(got_ds_state)
                return get_ds_state
            check_datasets_config.addCallback(got_datasets_config)
            return check_datasets_config

        d.addCallback(dataset_moved)

        return d

    @require_moving_backend
    @require_cluster(2)
    def test_lease_prevents_delete(self, cluster):
        """
        A dataset cannot be deleted if a lease is held on
        it by a particular node.
        """
        self.fail("not implemented yet")

    @require_moving_backend
    @require_cluster(2)
    def test_delete_dataset_after_lease_release(self, cluster):
        """
        A dataset can be deleted once a lease held on it by a
        particular node is released.
        """
        self.fail("not implemented yet")

    @require_moving_backend
    @require_cluster(2)
    def test_delete_dataset_after_lease_expiry(self, cluster):
        """
        A dataset can be deleted once a lease held on it by a
        particular node has expired.
        """
        self.fail("not implemented yet")
<|endoftext|>


<commit_before><commit_msg>Remove code which I missed on the first pass<commit_after><|endoftext|>


<commit_before>from __future__ import absolute_import
from functools import partial
import numpy as onp
import numpy.linalg as npla
from .numpy_wrapper import wrap_namespace, dot
from . import numpy_wrapper as anp
from ..core import primitive
from builtins import range

wrap_namespace(npla.__dict__, globals())

# Some formulas are from
# "An extended collection of matrix derivative results
#  for forward and reverse mode algorithmic differentiation"
# by Mike Giles
# https://people.maths.ox.ac.uk/gilesm/files/NA-08-01.pdf

# transpose by swapping last two dimensions
T = lambda x: anp.swapaxes(x, -1, -2)

# add two dimensions to the end of x
add2d = lambda x: anp.array(x)[...,None,None]

det.defgrad(lambda ans, x: lambda g: add2d(g) * add2d(ans) * T(inv(x)))
slogdet.defgrad(lambda ans, x: lambda g: add2d(g[1]) * T(inv(x)))

def make_grad_inv(ans, x):
    dot = anp.dot if ans.ndim == 2 else partial(anp.einsum, '...ij,...jk->...ik')
    return lambda g: -dot(dot(T(ans), g), T(ans))
inv.defgrad(make_grad_inv)

def make_grad_solve(argnum, ans, a, b):
    updim = lambda x: x if x.ndim == a.ndim else x[...,None]
    dot = anp.dot if a.ndim == 2 else partial(anp.einsum, '...ij,...jk->...ik')

    grad_arg0 = lambda g: -dot(updim(solve(T(a), g)), T(updim(ans)))
    grad_arg1 = lambda g: solve(T(a), g)

    return grad_arg0 if argnum == 0 else grad_arg1
solve.defgrads(make_grad_solve, [0, 1])

def make_grad_norm(ans, x, ord=None, axis=None):
    def check_implemented():
        matrix_norm = (x.ndim==2 and axis is None) or isinstance(axis, tuple)
        frobenius_norm = ord is None or ord == 'fro'
        diffable_pnorm = ord is None or ord > 1

        if matrix_norm and not frobenius_norm:
            raise NotImplementedError(
                'Gradient of matrix norm not implemented for ord={}'.format(ord))
        if not diffable_pnorm:
            raise NotImplementedError(
                'Gradient of norm not implemented for ord={}'.format(ord))

    expand = lambda a: a if axis is None else anp.expand_dims(a, axis=axis)

    def norm_grad(g):
        check_implemented()
        if ord is None or ord == 2 or ord is 'fro':
            return expand(g / ans) * x
        else:
            # see https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm
            return expand(g / ans**(ord-1)) * x * anp.abs(x)**(ord-2)
    return norm_grad
norm.defgrad(make_grad_norm)

def make_grad_eigh(ans, x, UPLO='L'):
    """Gradient for eigenvalues and vectors of a symmetric matrix."""
    N = x.shape[-1]
    w, v = ans              # Eigenvalues, eigenvectors.
    dot = anp.dot if x.ndim == 2 else partial(anp.einsum, '...ij,...jk->...ik')
    def eigh_grad(g):
        wg, vg = g          # Gradient w.r.t. eigenvalues, eigenvectors.
        w_repeated = anp.repeat(w[..., anp.newaxis], N, axis=-1)
        off_diag = anp.ones((N, N)) - anp.eye(N)
        F = off_diag / (T(w_repeated) - w_repeated + anp.eye(N))
        return dot(v * wg[..., anp.newaxis, :] + dot(v, F * dot(T(v), vg)), T(v))
    return eigh_grad
eigh.defgrad(make_grad_eigh)

def broadcasting_dsymv(alpha, A, x, lower=None):
    N = A.shape[-1]
    idxs = onp.arange(N)
    A_lower = onp.tril(A)
    A_sym = A_lower + T(A_lower)
    A_sym[..., idxs, idxs] *= 0.5
    return onp.einsum('...ij,...j->...i', A_sym, x)

def make_grad_cholesky(L, A):
    # based on choleskies_cython.pyx in SheffieldML/GPy and (Smith 1995)
    # TODO for higher-order differentiation, replace dsymv, get rid of inplace
    # ops, make cholesky grad primitive and defgrad? also ArrayNode assignment
    from scipy.linalg.blas import dsymv
    N = L.shape[-1]
    T = lambda x: onp.swapaxes(x, -1, -2)
    if A.ndim > 2:
        dsymv = broadcasting_dsymv
        dot = partial(onp.einsum, '...i,...i->...')
        diag = partial(onp.diagonal, axis1=-1, axis2=-2)
    else:
        dot = onp.dot
        diag = onp.diag

    @primitive
    def cholesky_grad(g):
        dL = onp.tril(g)
        dL[...,-1,-1] /= 2 * L[...,-1,-1]
        for k in range(N-2, -1, -1):
            dL[...,k+1:,k] -= dsymv(1., dL[...,k+1:,k+1:], L[...,k+1:,k], lower=True)
            dL[...,k+1:,k] -= diag(dL[...,k+1:,k+1:]) * L[...,k+1:,k]
            dL[...,k+1:,k] /= L[...,k:k+1,k]
            dL[...,k,k] -= dot(dL[...,k+1:,k], L[...,k+1:,k])
            dL[...,k,k] /= 2 * L[...,k,k]
        return (dL + T(dL))/2.
    return cholesky_grad
cholesky.defgrad(make_grad_cholesky)
<commit_msg>Minor tweak: Removed uncessary dot import<commit_after>from __future__ import absolute_import
from functools import partial
import numpy as onp
import numpy.linalg as npla
from .numpy_wrapper import wrap_namespace
from . import numpy_wrapper as anp
from ..core import primitive
from builtins import range

wrap_namespace(npla.__dict__, globals())

# Some formulas are from
# "An extended collection of matrix derivative results
#  for forward and reverse mode algorithmic differentiation"
# by Mike Giles
# https://people.maths.ox.ac.uk/gilesm/files/NA-08-01.pdf

# transpose by swapping last two dimensions
T = lambda x: anp.swapaxes(x, -1, -2)

# add two dimensions to the end of x
add2d = lambda x: anp.array(x)[...,None,None]

det.defgrad(lambda ans, x: lambda g: add2d(g) * add2d(ans) * T(inv(x)))
slogdet.defgrad(lambda ans, x: lambda g: add2d(g[1]) * T(inv(x)))

def make_grad_inv(ans, x):
    dot = anp.dot if ans.ndim == 2 else partial(anp.einsum, '...ij,...jk->...ik')
    return lambda g: -dot(dot(T(ans), g), T(ans))
inv.defgrad(make_grad_inv)

def make_grad_solve(argnum, ans, a, b):
    updim = lambda x: x if x.ndim == a.ndim else x[...,None]
    dot = anp.dot if a.ndim == 2 else partial(anp.einsum, '...ij,...jk->...ik')

    grad_arg0 = lambda g: -dot(updim(solve(T(a), g)), T(updim(ans)))
    grad_arg1 = lambda g: solve(T(a), g)

    return grad_arg0 if argnum == 0 else grad_arg1
solve.defgrads(make_grad_solve, [0, 1])

def make_grad_norm(ans, x, ord=None, axis=None):
    def check_implemented():
        matrix_norm = (x.ndim==2 and axis is None) or isinstance(axis, tuple)
        frobenius_norm = ord is None or ord == 'fro'
        diffable_pnorm = ord is None or ord > 1

        if matrix_norm and not frobenius_norm:
            raise NotImplementedError(
                'Gradient of matrix norm not implemented for ord={}'.format(ord))
        if not diffable_pnorm:
            raise NotImplementedError(
                'Gradient of norm not implemented for ord={}'.format(ord))

    expand = lambda a: a if axis is None else anp.expand_dims(a, axis=axis)

    def norm_grad(g):
        check_implemented()
        if ord is None or ord == 2 or ord is 'fro':
            return expand(g / ans) * x
        else:
            # see https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm
            return expand(g / ans**(ord-1)) * x * anp.abs(x)**(ord-2)
    return norm_grad
norm.defgrad(make_grad_norm)

def make_grad_eigh(ans, x, UPLO='L'):
    """Gradient for eigenvalues and vectors of a symmetric matrix."""
    N = x.shape[-1]
    w, v = ans              # Eigenvalues, eigenvectors.
    dot = anp.dot if x.ndim == 2 else partial(anp.einsum, '...ij,...jk->...ik')
    def eigh_grad(g):
        wg, vg = g          # Gradient w.r.t. eigenvalues, eigenvectors.
        w_repeated = anp.repeat(w[..., anp.newaxis], N, axis=-1)
        off_diag = anp.ones((N, N)) - anp.eye(N)
        F = off_diag / (T(w_repeated) - w_repeated + anp.eye(N))
        return dot(v * wg[..., anp.newaxis, :] + dot(v, F * dot(T(v), vg)), T(v))
    return eigh_grad
eigh.defgrad(make_grad_eigh)

def broadcasting_dsymv(alpha, A, x, lower=None):
    N = A.shape[-1]
    idxs = onp.arange(N)
    A_lower = onp.tril(A)
    A_sym = A_lower + T(A_lower)
    A_sym[..., idxs, idxs] *= 0.5
    return onp.einsum('...ij,...j->...i', A_sym, x)

def make_grad_cholesky(L, A):
    # based on choleskies_cython.pyx in SheffieldML/GPy and (Smith 1995)
    # TODO for higher-order differentiation, replace dsymv, get rid of inplace
    # ops, make cholesky grad primitive and defgrad? also ArrayNode assignment
    from scipy.linalg.blas import dsymv
    N = L.shape[-1]
    T = lambda x: onp.swapaxes(x, -1, -2)
    if A.ndim > 2:
        dsymv = broadcasting_dsymv
        dot = partial(onp.einsum, '...i,...i->...')
        diag = partial(onp.diagonal, axis1=-1, axis2=-2)
    else:
        dot = onp.dot
        diag = onp.diag

    @primitive
    def cholesky_grad(g):
        dL = onp.tril(g)
        dL[...,-1,-1] /= 2 * L[...,-1,-1]
        for k in range(N-2, -1, -1):
            dL[...,k+1:,k] -= dsymv(1., dL[...,k+1:,k+1:], L[...,k+1:,k], lower=True)
            dL[...,k+1:,k] -= diag(dL[...,k+1:,k+1:]) * L[...,k+1:,k]
            dL[...,k+1:,k] /= L[...,k:k+1,k]
            dL[...,k,k] -= dot(dL[...,k+1:,k], L[...,k+1:,k])
            dL[...,k,k] /= 2 * L[...,k,k]
        return (dL + T(dL))/2.
    return cholesky_grad
cholesky.defgrad(make_grad_cholesky)
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
import re
import json
import requests
import sys
try:
    import vim
except:
    pass

def clean(string):
    return string.encode("ascii", "ignore").decode("ascii")

def query_google(query, domain):
    url = "http://www.google.com/search?q=site:%s+%s" % (domain, query)
    urls = []
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html5lib")
    questions = soup.findAll("h3", { "class": "r" })
    for question in questions:
        urls += [re.search("\?q=(.*?)&", item["href"]).group(1) for item in question.find_all("a", href=True)]
    return urls

def get_answers(url):
    data = {}
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html5lib")

    all_posts = soup.findAll("div", { "class": "post-text" })
    if len(all_posts) == 0:
        return None

    data["title"] = clean(soup.find("a", { "class": "question-hyperlink" }).text)
    data["question"] = clean(all_posts[0].text)
    data["answers"] = []
    for x in range(1, len(all_posts)):
        data["answers"].append(clean(all_posts[x].text))
    return data

def fetch_answers(query):
    urls = query_google(query, "stackoverflow.com")
    if urls is None:
        return None
    answers = []
    for url in urls:
        responses = get_answers(url)
        if responses:
            answers.append(responses)

    return answers

# StackAnswers output ---------------------------------------------------------

def _goto_window_for_buffer(b):
    w = int(vim.eval('bufwinnr(%d)' % int(b)))
    vim.command('%dwincmd w' % w)

def _goto_window_for_buffer_name(bn):
    b = vim.eval('bufnr("%s")' % bn)
    return _goto_window_for_buffer(b)

def _output_preview_text(lines):
    _goto_window_for_buffer_name('__Answers__')
    vim.command('setlocal modifiable')
    lines = [clean(line).replace("\n", "\r") for line in lines]
    vim.current.buffer[:] = lines
    # Ensure that the global flag is off by default
    vim.command('setlocal nogdefault')
    # Replace 1-2 \r's with one.
    vim.command('silent %s/\r\{1,2\}/\r/ge')
    vim.command('setlocal nomodifiable')

def _generate_stack_answers_format(posts):
    response = []
    for post in posts:
        answers = len(post["answers"])

        question = "Q: " + post["title"]
        response.append(question)
        response.append(post["question"])
        response.append("%d Answer(s)" % answers)
        for answer in post["answers"]:
            response.append("=" * 80)
            response.append(answer + "\r")
        response.append("="*80)
        response.append("="*80)
    return response

def stackAnswersVim(query):
    query = vim.eval("a:2")
    data = fetch_answers(query)
    if data is None:
        text = ["Error fetching data..."]
        _output_preview_text(text)
    else:
        _output_preview_text(_generate_stack_answers_format(data))

def stackAnswersCI(query):
    data = fetch_answers(query)
    if data is None:
        text = ["Error fetching data..."]
        return text
    else:
        formatted = _generate_stack_answers_format(data)
        formatted = "\n".join(formatted)
        formatted = formatted.replace("\r", "\n")
        return formatted

if __name__ == "__main__":
    args = sys.argv
    if "--cli" in args:
        query = " ".join(args[2:])
        print stackAnswersCI(query)
<commit_msg>Use parentheses when calling print<commit_after>from bs4 import BeautifulSoup
import re
import json
import requests
import sys
try:
    import vim
except:
    pass

def clean(string):
    return string.encode("ascii", "ignore").decode("ascii")

def query_google(query, domain):
    url = "http://www.google.com/search?q=site:%s+%s" % (domain, query)
    urls = []
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html5lib")
    questions = soup.findAll("h3", { "class": "r" })
    for question in questions:
        urls += [re.search("\?q=(.*?)&", item["href"]).group(1) for item in question.find_all("a", href=True)]
    return urls

def get_answers(url):
    data = {}
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html5lib")

    all_posts = soup.findAll("div", { "class": "post-text" })
    if len(all_posts) == 0:
        return None

    data["title"] = clean(soup.find("a", { "class": "question-hyperlink" }).text)
    data["question"] = clean(all_posts[0].text)
    data["answers"] = []
    for x in range(1, len(all_posts)):
        data["answers"].append(clean(all_posts[x].text))
    return data

def fetch_answers(query):
    urls = query_google(query, "stackoverflow.com")
    if urls is None:
        return None
    answers = []
    for url in urls:
        responses = get_answers(url)
        if responses:
            answers.append(responses)

    return answers

# StackAnswers output ---------------------------------------------------------

def _goto_window_for_buffer(b):
    w = int(vim.eval('bufwinnr(%d)' % int(b)))
    vim.command('%dwincmd w' % w)

def _goto_window_for_buffer_name(bn):
    b = vim.eval('bufnr("%s")' % bn)
    return _goto_window_for_buffer(b)

def _output_preview_text(lines):
    _goto_window_for_buffer_name('__Answers__')
    vim.command('setlocal modifiable')
    lines = [clean(line).replace("\n", "\r") for line in lines]
    vim.current.buffer[:] = lines
    # Ensure that the global flag is off by default
    vim.command('setlocal nogdefault')
    # Replace 1-2 \r's with one.
    vim.command('silent %s/\r\{1,2\}/\r/ge')
    vim.command('setlocal nomodifiable')

def _generate_stack_answers_format(posts):
    response = []
    for post in posts:
        answers = len(post["answers"])

        question = "Q: " + post["title"]
        response.append(question)
        response.append(post["question"])
        response.append("%d Answer(s)" % answers)
        for answer in post["answers"]:
            response.append("=" * 80)
            response.append(answer + "\r")
        response.append("="*80)
        response.append("="*80)
    return response

def stackAnswersVim(query):
    query = vim.eval("a:2")
    data = fetch_answers(query)
    if data is None:
        text = ["Error fetching data..."]
        _output_preview_text(text)
    else:
        _output_preview_text(_generate_stack_answers_format(data))

def stackAnswersCI(query):
    data = fetch_answers(query)
    if data is None:
        text = ["Error fetching data..."]
        return text
    else:
        formatted = _generate_stack_answers_format(data)
        formatted = "\n".join(formatted)
        formatted = formatted.replace("\r", "\n")
        return formatted

if __name__ == "__main__":
    args = sys.argv
    if "--cli" in args:
        query = " ".join(args[2:])
        print(stackAnswersCI(query))
<|endoftext|>


<commit_before># Copyright 2017 Battelle Energy Alliance, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Factory for generating the instances of the  Models Module
"""

from EntityFactoryBase import EntityFactory
from .PostProcessorInterface import PostProcessorInterface
from .PostProcessorReadyInterface import PostProcessorReadyInterface
from .BasicStatistics import BasicStatistics
from .LimitSurface import LimitSurface
from .Metric import Metric
from .DataMining import DataMining
from .SafestPoint import SafestPoint
from .ValueDuration import ValueDuration
from .SampleSelector import SampleSelector
from .ImportanceRank import ImportanceRank
from .CrossValidation import CrossValidation
from .LimitSurfaceIntegral import LimitSurfaceIntegral
from .FastFourierTransform import FastFourierTransform
from .ExternalPostProcessor import ExternalPostProcessor
from .TopologicalDecomposition import TopologicalDecomposition
from .ComparisonStatisticsModule import ComparisonStatistics
from .RealizationAverager import RealizationAverager
from .ParetoFrontierPostProcessor import ParetoFrontier
from .EconomicRatio import EconomicRatio
from .Validation import Validation
from .validationAlgorithms.Probabilistic import Probabilistic
from .validationAlgorithms.Representativity import Representativity

### PostProcessorFunctions (orig: InterfacedPostProcessor)
from .HistorySetDelay import HistorySetDelay
from .HS2PS import HS2PS
from .HStoPSOperator import HStoPSOperator
from .HistorySetSampling import HistorySetSampling
from .HistorySetSnapShot import HistorySetSnapShot
from .HistorySetSync import HistorySetSync
from .TypicalHistoryFromHistorySet import TypicalHistoryFromHistorySet
from .dataObjectLabelFilter import dataObjectLabelFilter
## These utilize the optional prequisite library PySide, so don't error if they
## do not import appropriately.
try:
  from .TopologicalDecomposition import QTopologicalDecomposition
  from .DataMining import QDataMining
  renaming = {'QTopologicalDecomposition': 'TopologicalDecomposition',
              'QDataMining': 'DataMining'}
except ImportError:
  renaming = {}

factory = EntityFactory('PostProcessorInterface', needsRunInfo=True)
factory.registerAllSubtypes(PostProcessorInterface, alias=renaming)

factory.registerType('External', ExternalPostProcessor)
<commit_msg>Update Factory.py<commit_after># Copyright 2017 Battelle Energy Alliance, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Factory for generating the instances of the  Models Module
"""

from EntityFactoryBase import EntityFactory
from .PostProcessorInterface import PostProcessorInterface
from .PostProcessorReadyInterface import PostProcessorReadyInterface
from .BasicStatistics import BasicStatistics
from .LimitSurface import LimitSurface
from .Metric import Metric
from .DataMining import DataMining
from .SafestPoint import SafestPoint
from .ValueDuration import ValueDuration
from .SampleSelector import SampleSelector
from .ImportanceRank import ImportanceRank
from .CrossValidation import CrossValidation
from .LimitSurfaceIntegral import LimitSurfaceIntegral
from .FastFourierTransform import FastFourierTransform
from .ExternalPostProcessor import ExternalPostProcessor
from .TopologicalDecomposition import TopologicalDecomposition
from .ComparisonStatisticsModule import ComparisonStatistics
from .RealizationAverager import RealizationAverager
from .ParetoFrontierPostProcessor import ParetoFrontier
from .EconomicRatio import EconomicRatio
from .Validation import Validation
from .validationAlgorithms.Probabilistic import Probabilistic

### PostProcessorFunctions (orig: InterfacedPostProcessor)
from .HistorySetDelay import HistorySetDelay
from .HS2PS import HS2PS
from .HStoPSOperator import HStoPSOperator
from .HistorySetSampling import HistorySetSampling
from .HistorySetSnapShot import HistorySetSnapShot
from .HistorySetSync import HistorySetSync
from .TypicalHistoryFromHistorySet import TypicalHistoryFromHistorySet
from .dataObjectLabelFilter import dataObjectLabelFilter
## These utilize the optional prequisite library PySide, so don't error if they
## do not import appropriately.
try:
  from .TopologicalDecomposition import QTopologicalDecomposition
  from .DataMining import QDataMining
  renaming = {'QTopologicalDecomposition': 'TopologicalDecomposition',
              'QDataMining': 'DataMining'}
except ImportError:
  renaming = {}

factory = EntityFactory('PostProcessorInterface', needsRunInfo=True)
factory.registerAllSubtypes(PostProcessorInterface, alias=renaming)

factory.registerType('External', ExternalPostProcessor)
<|endoftext|>


<commit_before>import json
from collections import deque
from decimal import Decimal

import hyper.tls
from hyper import HTTP20Connection
from hyper.http20.exceptions import HTTP20Error

from autopush.exceptions import RouterException

SANDBOX = 'api.development.push.apple.com'
SERVER = 'api.push.apple.com'

APNS_MAX_CONNECTIONS = 20

# These values are defined by APNs as header values that should be sent.
# The hyper library requires that all header values be strings.
# These values should be considered "opaque" to APNs.
# see https://developer.apple.com/search/?q=%22apns-priority%22
APNS_PRIORITY_IMMEDIATE = '10'
APNS_PRIORITY_LOW = '5'

class ComplexEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Decimal):
            return int(obj.to_integral_value())
        # for most data types, this function isn't called.
        # the following is added for safety, but should not
        # be required.
        return json.JSONEncoder.default(self, obj)  # pragma nocover

class APNSException(Exception):
    pass

class APNSClient(object):
    def __init__(self, cert_file, key_file, topic,
                 alt=False, use_sandbox=False,
                 max_connections=APNS_MAX_CONNECTIONS,
                 logger=None, metrics=None,
                 load_connections=True,
                 max_retry=2):
        """Create the APNS client connector.

        The cert_file and key_file can be derived from the exported `.p12`
        **Apple Push Services: *bundleID* ** key contained in the **Keychain
        Access** application. To extract the proper PEM formatted data, you
        can use the following commands:

        ```
        openssl pkcs12 -in file.p12 -out apns_cert.pem -clcerts -nokeys
        openssl pkcs12 -in file.p12 -out apns_key.pem -nocerts -nodes
        ```

        The *topic* is the Bundle ID of the bridge recipient iOS application.
        Since the cert needs to be tied directly to an application, the topic
        is usually similar to "com.example.MyApplication".

        :param cert_file: Path to the PEM formatted APNs certification file.
        :type cert_file: str
        :param key_file: Path to the PEM formatted APNs key file.
        :type key_file: str
        :param topic: The *Bundle ID* that identifies the assoc. iOS app.
        :type topic: str
        :param alt: Use the alternate APNs publication port (if 443 is blocked)
        :type alt: bool
        :param use_sandbox: Use the development sandbox
        :type use_sandbox: bool
        :param max_connections: Max number of pooled connections to use
        :type max_connections: int
        :param logger: Status logger
        :type logger: logger
        :param metrics: Metric recorder
        :type metrics: autopush.metrics.IMetric
        :param load_connections: used for testing
        :type load_connections: bool
        :param max_retry: Number of HTTP2 transmit attempts
        :type max_retry: int

        """
        self.server = SANDBOX if use_sandbox else SERVER
        self.port = 2197 if alt else 443
        self.log = logger
        self.metrics = metrics
        self.topic = topic
        self._max_connections = max_connections
        self._max_retry = max_retry
        self.connections = deque(maxlen=max_connections)
        if load_connections:
            self.ssl_context = hyper.tls.init_context(cert=(cert_file,
                                                            key_file))
            self.connections.extendleft((HTTP20Connection(
                self.server,
                self.port,
                ssl_context=self.ssl_context,
                force_proto='h2') for x in range(0, max_connections)))
        if self.log:
            self.log.debug("Starting APNS connection")

    def send(self, router_token, payload, apns_id,
             priority=True, topic=None, exp=None):
        """Send the dict of values to the remote bridge

        This sends the raw data to the remote bridge application using the
        APNS2 HTTP2 API.

        :param router_token: APNs provided hex token identifying recipient
        :type router_token: str
        :param payload: Data to send to recipient
        :type payload: dict
        :param priority: True is high priority, false is low priority
        :type priority: bool
        :param topic: BundleID for the recipient application (overides default)
        :type topic: str
        :param exp: Message expiration timestamp
        :type exp: timestamp

        """
        body = json.dumps(payload, cls=ComplexEncoder)
        priority = APNS_PRIORITY_IMMEDIATE if priority else APNS_PRIORITY_LOW
        # NOTE: Hyper requires that all header values be strings. 'Priority'
        # is a integer string, which may be "simplified" and cause an error.
        # The added str() function safeguards against that.
        headers = {
            'apns-id': apns_id,
            'apns-priority': str(priority),
            'apns-topic': topic or self.topic,
        }
        if exp:
            headers['apns-expiration'] = str(exp)
        url = '/3/device/' + router_token
        attempt = 0
        while True:
            try:
                connection = self._get_connection()
                # request auto-opens closed connections, so if a connection
                # has timed out or failed for other reasons, it's automatically
                # re-established.
                stream_id = connection.request(
                    'POST', url=url, body=body, headers=headers)
                # get_response() may return an AttributeError. Not really sure
                # how it happens, but the connected socket may get set to None.
                # We'll treat that as a premature socket closure.
                response = connection.get_response(stream_id)
                if response.status != 200:
                    reason = json.loads(
                            response.read().decode('utf-8'))['reason']
                    raise RouterException(
                        "APNS Transmit Error {}:{}".format(response.status,
                                                           reason),
                        status_code=response.status,
                        response_body="APNS could not process "
                                      "your message {}".format(reason),
                        log_exception=False,
                        reason=reason
                    )
                break
            except (HTTP20Error, IOError):
                connection.close()
                attempt += 1
                if attempt < self._max_retry:
                    continue
                raise
            finally:
                # Returning a closed connection to the pool is ok.
                # hyper will reconnect on .request()
                self._return_connection(connection)

    def _get_connection(self):
        try:
            connection = self.connections.pop()
            return connection
        except IndexError:
            raise RouterException(
                "Too many APNS requests, increase pool from {}".format(
                    self._max_connections
                ),
                status_code=503,
                response_body="APNS busy, please retry")

    def _return_connection(self, connection):
        self.connections.appendleft(connection)
<commit_msg>fix: Re-enable APNS error logging (#1460)<commit_after>import json
from collections import deque
from decimal import Decimal

import hyper.tls
from hyper import HTTP20Connection
from hyper.http20.exceptions import HTTP20Error

from autopush.exceptions import RouterException

SANDBOX = 'api.development.push.apple.com'
SERVER = 'api.push.apple.com'

APNS_MAX_CONNECTIONS = 20

# These values are defined by APNs as header values that should be sent.
# The hyper library requires that all header values be strings.
# These values should be considered "opaque" to APNs.
# see https://developer.apple.com/search/?q=%22apns-priority%22
APNS_PRIORITY_IMMEDIATE = '10'
APNS_PRIORITY_LOW = '5'

class ComplexEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Decimal):
            return int(obj.to_integral_value())
        # for most data types, this function isn't called.
        # the following is added for safety, but should not
        # be required.
        return json.JSONEncoder.default(self, obj)  # pragma nocover

class APNSException(Exception):
    pass

class APNSClient(object):
    def __init__(self, cert_file, key_file, topic,
                 alt=False, use_sandbox=False,
                 max_connections=APNS_MAX_CONNECTIONS,
                 logger=None, metrics=None,
                 load_connections=True,
                 max_retry=2):
        """Create the APNS client connector.

        The cert_file and key_file can be derived from the exported `.p12`
        **Apple Push Services: *bundleID* ** key contained in the **Keychain
        Access** application. To extract the proper PEM formatted data, you
        can use the following commands:

        ```
        openssl pkcs12 -in file.p12 -out apns_cert.pem -clcerts -nokeys
        openssl pkcs12 -in file.p12 -out apns_key.pem -nocerts -nodes
        ```

        The *topic* is the Bundle ID of the bridge recipient iOS application.
        Since the cert needs to be tied directly to an application, the topic
        is usually similar to "com.example.MyApplication".

        :param cert_file: Path to the PEM formatted APNs certification file.
        :type cert_file: str
        :param key_file: Path to the PEM formatted APNs key file.
        :type key_file: str
        :param topic: The *Bundle ID* that identifies the assoc. iOS app.
        :type topic: str
        :param alt: Use the alternate APNs publication port (if 443 is blocked)
        :type alt: bool
        :param use_sandbox: Use the development sandbox
        :type use_sandbox: bool
        :param max_connections: Max number of pooled connections to use
        :type max_connections: int
        :param logger: Status logger
        :type logger: logger
        :param metrics: Metric recorder
        :type metrics: autopush.metrics.IMetric
        :param load_connections: used for testing
        :type load_connections: bool
        :param max_retry: Number of HTTP2 transmit attempts
        :type max_retry: int

        """
        self.server = SANDBOX if use_sandbox else SERVER
        self.port = 2197 if alt else 443
        self.log = logger
        self.metrics = metrics
        self.topic = topic
        self._max_connections = max_connections
        self._max_retry = max_retry
        self.connections = deque(maxlen=max_connections)
        if load_connections:
            self.ssl_context = hyper.tls.init_context(cert=(cert_file,
                                                            key_file))
            self.connections.extendleft((HTTP20Connection(
                self.server,
                self.port,
                ssl_context=self.ssl_context,
                force_proto='h2') for x in range(0, max_connections)))
        if self.log:
            self.log.debug("Starting APNS connection")

    def send(self, router_token, payload, apns_id,
             priority=True, topic=None, exp=None):
        """Send the dict of values to the remote bridge

        This sends the raw data to the remote bridge application using the
        APNS2 HTTP2 API.

        :param router_token: APNs provided hex token identifying recipient
        :type router_token: str
        :param payload: Data to send to recipient
        :type payload: dict
        :param priority: True is high priority, false is low priority
        :type priority: bool
        :param topic: BundleID for the recipient application (overides default)
        :type topic: str
        :param exp: Message expiration timestamp
        :type exp: timestamp

        """
        body = json.dumps(payload, cls=ComplexEncoder)
        priority = APNS_PRIORITY_IMMEDIATE if priority else APNS_PRIORITY_LOW
        # NOTE: Hyper requires that all header values be strings. 'Priority'
        # is a integer string, which may be "simplified" and cause an error.
        # The added str() function safeguards against that.
        headers = {
            'apns-id': apns_id,
            'apns-priority': str(priority),
            'apns-topic': topic or self.topic,
        }
        if exp:
            headers['apns-expiration'] = str(exp)
        url = '/3/device/' + router_token
        attempt = 0
        while True:
            try:
                connection = self._get_connection()
                # request auto-opens closed connections, so if a connection
                # has timed out or failed for other reasons, it's automatically
                # re-established.
                stream_id = connection.request(
                    'POST', url=url, body=body, headers=headers)
                # get_response() may return an AttributeError. Not really sure
                # how it happens, but the connected socket may get set to None.
                # We'll treat that as a premature socket closure.
                response = connection.get_response(stream_id)
                if response.status != 200:
                    reason = json.loads(
                            response.read().decode('utf-8'))['reason']
                    raise RouterException(
                        "APNS Transmit Error {}:{}".format(response.status,
                                                           reason),
                        status_code=response.status,
                        response_body="APNS could not process "
                                      "your message {}".format(reason),
                        log_exception=True,
                        reason=reason
                    )
                break
            except (HTTP20Error, IOError):
                connection.close()
                attempt += 1
                if attempt < self._max_retry:
                    continue
                raise
            finally:
                # Returning a closed connection to the pool is ok.
                # hyper will reconnect on .request()
                self._return_connection(connection)

    def _get_connection(self):
        try:
            connection = self.connections.pop()
            return connection
        except IndexError:
            raise RouterException(
                "Too many APNS requests, increase pool from {}".format(
                    self._max_connections
                ),
                status_code=503,
                response_body="APNS busy, please retry")

    def _return_connection(self, connection):
        self.connections.appendleft(connection)
<|endoftext|>


<commit_before>import discord
import os
import logging
from discord.ext import commands
from .utils.dataIO import fileIO
from .utils import checks

log = logging.getLogger(__name__)

class Autostatus:

    """ Automatically set the bot's status on startup """

    def __init__(self, bot):
        self.bot = bot
        try:
            self.autostatus = fileIO("data/autostatus/autostatus.json", "load")
            self.bot.loop.create_task(self.on_load())
        except:
            log.debug("Exception when loading autostatus!")

    @commands.command(pass_context=False, no_pm=False)
    @checks.mod_or_permissions(manage_messages=True)
    async def status(self, newstatus=None):
        if newstatus:
            self.autostatus['startup'] = str(newstatus).strip()
            fileIO("data/autostatus/autostatus.json", "save", self.autostatus)
            await self.bot.change_presence(discord.Game(name=newstatus))
        else:
            response = 'Usage: !status "add the status you want here"'
            if self.autostatus:
                log.debug("current autostatus: " + str(self.autostatus))
            await self.bot.say(response)

    async def on_ready(self):
        await self.bot.wait_until_ready()
        if self.autostatus['startup']:
            log.debug("attempting to set the status to {}".format(
                   self.autostatus['startup']))
            await self.bot.change_presence(discord.Game(
                  name=self.autostatus['startup']))
        else:
            return

def build_folders():
    folders = ("data", "data/autostatus/")
    for folder in folders:
        if not os.path.exists(folder):
            log.debug("Creating " + folder + " folder...")
            os.makedirs(folder)
    if not os.path.isfile("data/autostatus/autostatus.json"):
        log.debug("creating default autostatus.json...")
        fileIO("data/autostatus/autostatus.json", "save", {})

def setup(bot):
    build_folders()
    bot.add_cog(Autostatus(bot))
<commit_msg>fix autostatus<commit_after>import discord
import os
import logging
from discord.ext import commands
from .utils.dataIO import fileIO
from .utils import checks

log = logging.getLogger(__name__)

class Autostatus:

    """ Automatically set the bot's status on startup """

    def __init__(self, bot):
        self.bot = bot
        try:
            self.autostatus = fileIO("data/autostatus/autostatus.json", "load")
            self.bot.loop.create_task(self.on_load())
        except:
            log.debug("Exception when loading autostatus!")

    @commands.command(pass_context=False, no_pm=False)
    @checks.mod_or_permissions(manage_messages=True)
    async def status(self, newstatus=None):
        if newstatus:
            self.autostatus['startup'] = str(newstatus).strip()
            fileIO("data/autostatus/autostatus.json", "save", self.autostatus)
            await self.bot.change_presence(game=None, status=newstatus, afk=False)
        else:
            response = 'Usage: !status "add the status you want here"'
            if self.autostatus:
                log.debug("current autostatus: " + str(self.autostatus))
            await self.bot.say(response)

    async def on_ready(self):
        await self.bot.wait_until_ready()
        if self.autostatus['startup']:
            log.debug("attempting to set the status to {}".format(
                   self.autostatus['startup']))
            await self.bot.change_presence(game=None, status=self.autostatus['startup'], afk=False)
        else:
            return

def build_folders():
    folders = ("data", "data/autostatus/")
    for folder in folders:
        if not os.path.exists(folder):
            log.debug("Creating " + folder + " folder...")
            os.makedirs(folder)
    if not os.path.isfile("data/autostatus/autostatus.json"):
        log.debug("creating default autostatus.json...")
        fileIO("data/autostatus/autostatus.json", "save", {})

def setup(bot):
    build_folders()
    bot.add_cog(Autostatus(bot))
<|endoftext|>