<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2017 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import mimetypes
import os
import tempfile
import warnings

from girder.models.model_base import ModelImporter, GirderException
from girder.constants import SettingKey

warnings.warn(
    'setup_database.py is only meant for test fixtures, not for provisioning.'
    '\nSee Girder\'s ansible client for a deployment solution: '
    'https://github.com/girder/girder/tree/master/devops/ansible/roles/girder/library'
)

#: A prefix for all relative file paths
prefix = '.'

def resolvePath(path):
    """Resolve a path to an absolute path string."""
    global prefix
    if os.path.isabs(path):
        return path
    return os.path.abspath(os.path.join(prefix, path))

def loadModel(kind):
    """Load a model class from its name."""
    return ModelImporter().model(kind)

def setAssetstore(name=None):
    """Set the assetstore that will be used for all files.

    The default behavior of this function is to assert that
    an assetstore exists, if not it will create a new filesystem
    assetstore with the name ``test``.  Optionally, a name
    can be provided to pick (or create) a specific assetstore.

    :param name: The name of the assetstore
    :type name: str
    """
    assetstore = None
    assetstoreModel = loadModel('assetstore')
    if not name:
        try:
            assetstore = assetstoreModel.getCurrent()
        except GirderException:
            pass
    else:
        for a in assetstoreModel.list():
            if name == a['name']:
                assetstore = a

    if assetstore is None:
        dir = tempfile.mkdtemp()
        assetstore = assetstoreModel.createFilesystemAssetstore(
            name or 'test', dir)

    assetstore['current'] = True
    assetstoreModel.save(assetstore)

def addCreator(spec, parent=None):
    """Inject a creator into document spec.

    This function mutates the input spec setting the ``creator``
    key to a valid user document.  If the key is set to a
    name, then the document will be queried from the global
    ``users`` cache, otherwise it will use the creator of
    the parent document.

    :param spec: The document to mutate
    :type spec: dict
    :param parent: The parent of current document
    :type parent: dict
    """
    userModel = loadModel('user')
    if 'creator' in spec:
        spec['creator'] = userModel.findOne({'login': spec['creator'].lower()}, force=True)
    elif parent is not None:

        if parent.get('_modelType') == 'user':
            # if the parent is a user, then use that as the creator
            # (users don't have creators)
            spec['creator'] = parent
        elif parent.get('creatorId'):
            # otherwise, use the creator of the parent
            spec['creator'] = userModel.load(parent.get('creatorId'), force=True)
    else:
        # if all else fails, just use any user for the creator
        spec['creator'] = userModel.findOne({}, force=True)

def createUser(defaultFolders=False, **args):
    """Create a user document from a user spec.

    As a side effect of this function, the created user will be
    cached in a global dictionary to be used by :py:func:`addCreator`.
    Keyword arguments are passed directly to the ``createUser``
    function.

    The default behavior is to **not** create the default public and
    private folders.  Automatically created folders cannot be used
    by this module as a parent for new folders and items.

    :param defaultFolders: Create default public and private folders
    :type defaultFolders: bool
    :returns: The generated user document
    """
    settingModel = loadModel('setting')
    userModel = loadModel('user')
    if not defaultFolders:
        settingModel.set(SettingKey.USER_DEFAULT_FOLDERS, 'none')
    user = userModel.createUser(**args)
    if not defaultFolders:
        settingModel.unset(SettingKey.USER_DEFAULT_FOLDERS)
    return user

def createCollection(**args):
    """Create a collection document from a collection spec.

    Keyword arguments are passed directly to the ``createCollection``
    function.

    :returns: The generated collection document
    """
    addCreator(args)
    collectionModel = loadModel('collection')
    return collectionModel.createCollection(**args)

def createFolder(parent, **args):
    """Create a folder document from a folder spec.

    The parent type can be any of ``collection``, ``folder``, or
    ``user``.  The parent is automatically injected into
    the keyword arguments.

    :param parent: The parent document
    :type parent: dict
    :returns: The generated folder document
    """
    addCreator(args, parent)
    folderModel = loadModel('folder')
    args['parentType'] = parent['_modelType']
    return folderModel.createFolder(parent, **args)

def createItem(parent, **args):
    """Create an item document from an item spec.

    The parent folder document must be provided and is injected
    automatically into the keyword arguments.

    :param parent: The parent folder
    :type parent: dict
    :returns: The generated item document
    """
    addCreator(args, parent)
    itemModel = loadModel('item')
    args['folder'] = parent
    return itemModel.createItem(**args)

def createFile(parent, path, **args):
    """Create a file document from a file path.

    The provided path should resolve to a valid file.  The
    ``size`` and ``mimeType`` of the file will be automatically
    resolved from this path.

    :param parent: The parent item
    :type parent: dict
    :param path: The local path to a file
    :type path: str
    :returns: The generated file document
    """
    addCreator(args, parent)
    uploadModel = loadModel('upload')

    path = resolvePath(path)
    args['parentType'] = 'item'
    args['parent'] = parent
    args['user'] = args.pop('creator')
    args['size'] = os.path.getsize(path)
    args.setdefault('reference', 'setup_database')

    if 'name' not in args:
        args['name'] = os.path.basename(path)

    if not args.get('mimeType'):
        args['mimeType'] = mimetypes.guess_type(path)[0]

    with open(path, 'rb') as f:
        return uploadModel.uploadFromFile(f, **args)

dispatch = {
    'user': createUser,
    'collection': createCollection,
    'folder': createFolder,
    'item': createItem,
    'file': createFile
}

def createDocument(type, node):
    """Create a generic document type from a spec.

    :param type: The type of document to create
    :type type: str
    :param node: The document spec
    :type node: dict
    :returns: An objected describing the document's children
    """
    # automatically generate children specs from a local path
    path = node.pop('import', None)
    if path:
        importRecursive(type, node, path)

    # create the return object with all possible children
    children = {
        'folder': node.pop('folders', []),
        'item': node.pop('items', []),
        'file': node.pop('files', [])
    }

    # remove invalid child types
    if type in ['user', 'collection']:
        del children['item']
        del children['file']
    elif type == 'folder':
        del children['file']
    elif type == 'item':
        del children['folder']
        del children['item']

    # generate the document
    doc = dispatch[type](**node)

    # set the type for certain models (collection) that don't do it
    # on their own
    doc['_modelType'] = type

    # inject this document as the parent for all child specs
    for childType in children:
        children[childType] = [
            dict(parent=doc, **child) for child in children[childType]
        ]

    return children

def createRecursive(type, node):
    """Generate all documents from a given node recursively.

    :param type: The type of the root node
    :type type: str
    :param node: The root node spec
    :type node: dict
    """
    children = createDocument(type, node)
    for childType in children:
        for childNode in children[childType]:
            createRecursive(childType, childNode)

def createUsers(users):
    """Create a list of users.

    This method is special because we need to generate all users
    before any other type in order to inject the ``creator``
    argument correctly.

    :param users: A list of user specs
    :type users: list
    :returns: A list of all child documents for all users
    """
    folders = []
    for user in users:
        for folder in folders:
            # By default set the creator under a user to that user
            folder.setdefault('creator', user['login'])

        folders.extend(createDocument('user', user)['folder'])
    return folders

def importRecursive(type, parent, root):
    """Generate a document tree from a local path.

    The parent object will be mutated to append the folders and
    directories contained in the provided local path.

    :param type: The parent node type
    :type type: str
    :param parent: The parent document
    :type parent: dict
    :param root: The local path where the import will begin
    :type root: str
    """
    root = resolvePath(root)
    folders = {root: parent}

    for root, dirs, files in os.walk(root, followlinks=True):
        parent = folders[root]
        parent.setdefault('folders', [])
        parent.setdefault('items', [])

        for dir in dirs:
            path = os.path.join(root, dir)
            folder = {
                'name': dir
            }
            folders[path] = folder
            parent['folders'].append(folder)

        for file in files:
            path = os.path.join(root, file)

            parent['items'].append({
                'name': file,
                'files': [{
                    'name': file,
                    'path': path
                }]
            })

def generate(spec):
    """Generate documents from a nested object."""
    global prefix
    prefix = spec.get('prefix', prefix)

    setAssetstore(spec.get('assetstore'))

    # users have to generated first
    spec['folders'] = createUsers(spec.get('users', []))

    for collection in spec.get('collections', []):
        createRecursive('collection', collection)

    for folder in spec.get('folders', []):
        createRecursive('folder', folder)

def main(file):
    """Load a file into memory and generate the documents it describes."""
    prefix = os.path.dirname(file)
    with open(file) as f:
        import yaml
        spec = yaml.safe_load(f)
        spec.setdefault('prefix', prefix)
        generate(spec)

# For standalone usage, this file can be called as a script to generate
# a testing environment.  You should provide an empty database as an
# environment variable.  For example:
#
#   GIRDER_MONGO_URI='mongodb://127.0.0.1:27017/import_test' python setup_database.py spec.yml
#   GIRDER_MONGO_URI='mongodb://127.0.0.1:27017/import_test' girder-server
if __name__ == '__main__':
    import sys
    main(sys.argv[1])
<commit_msg>Throw errors instead of initializing models with no creator<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2017 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import mimetypes
import os
import tempfile
import warnings

from girder.models.model_base import ModelImporter, GirderException
from girder.constants import SettingKey

warnings.warn(
    'setup_database.py is only meant for test fixtures, not for provisioning.'
    '\nSee Girder\'s ansible client for a deployment solution: '
    'https://github.com/girder/girder/tree/master/devops/ansible/roles/girder/library'
)

#: A prefix for all relative file paths
prefix = '.'

def resolvePath(path):
    """Resolve a path to an absolute path string."""
    global prefix
    if os.path.isabs(path):
        return path
    return os.path.abspath(os.path.join(prefix, path))

def loadModel(kind):
    """Load a model class from its name."""
    return ModelImporter().model(kind)

def setAssetstore(name=None):
    """Set the assetstore that will be used for all files.

    The default behavior of this function is to assert that
    an assetstore exists, if not it will create a new filesystem
    assetstore with the name ``test``.  Optionally, a name
    can be provided to pick (or create) a specific assetstore.

    :param name: The name of the assetstore
    :type name: str
    """
    assetstore = None
    assetstoreModel = loadModel('assetstore')
    if not name:
        try:
            assetstore = assetstoreModel.getCurrent()
        except GirderException:
            pass
    else:
        for a in assetstoreModel.list():
            if name == a['name']:
                assetstore = a

    if assetstore is None:
        dir = tempfile.mkdtemp()
        assetstore = assetstoreModel.createFilesystemAssetstore(
            name or 'test', dir)

    assetstore['current'] = True
    assetstoreModel.save(assetstore)

def addCreator(spec, parent=None):
    """Inject a creator into document spec.

    This function mutates the input spec setting the ``creator``
    key to a valid user document.  If the key is set to a
    name, then the document will be queried from the global
    ``users`` cache, otherwise it will use the creator of
    the parent document.

    :param spec: The document to mutate
    :type spec: dict
    :param parent: The parent of current document
    :type parent: dict
    """
    userModel = loadModel('user')
    if 'creator' in spec:
        spec['creator'] = userModel.findOne({'login': spec['creator'].lower()}, force=True)
    elif parent is not None:

        if parent.get('_modelType') == 'user':
            # if the parent is a user, then use that as the creator
            # (users don't have creators)
            spec['creator'] = parent
        elif parent.get('creatorId'):
            # otherwise, use the creator of the parent
            spec['creator'] = userModel.load(parent.get('creatorId'), force=True)
    else:
        # if all else fails, just use any user for the creator
        spec['creator'] = userModel.findOne({}, force=True)

        if spec['creator'] is None:
            raise Exception('At least one user must be provided in the spec')

    if spec.get('creator') is None:
        raise Exception('Could not find the requested creator')

def createUser(defaultFolders=False, **args):
    """Create a user document from a user spec.

    As a side effect of this function, the created user will be
    cached in a global dictionary to be used by :py:func:`addCreator`.
    Keyword arguments are passed directly to the ``createUser``
    function.

    The default behavior is to **not** create the default public and
    private folders.  Automatically created folders cannot be used
    by this module as a parent for new folders and items.

    :param defaultFolders: Create default public and private folders
    :type defaultFolders: bool
    :returns: The generated user document
    """
    settingModel = loadModel('setting')
    userModel = loadModel('user')
    if not defaultFolders:
        settingModel.set(SettingKey.USER_DEFAULT_FOLDERS, 'none')
    user = userModel.createUser(**args)
    if not defaultFolders:
        settingModel.unset(SettingKey.USER_DEFAULT_FOLDERS)
    return user

def createCollection(**args):
    """Create a collection document from a collection spec.

    Keyword arguments are passed directly to the ``createCollection``
    function.

    :returns: The generated collection document
    """
    addCreator(args)
    collectionModel = loadModel('collection')
    return collectionModel.createCollection(**args)

def createFolder(parent, **args):
    """Create a folder document from a folder spec.

    The parent type can be any of ``collection``, ``folder``, or
    ``user``.  The parent is automatically injected into
    the keyword arguments.

    :param parent: The parent document
    :type parent: dict
    :returns: The generated folder document
    """
    addCreator(args, parent)
    folderModel = loadModel('folder')
    args['parentType'] = parent['_modelType']
    return folderModel.createFolder(parent, **args)

def createItem(parent, **args):
    """Create an item document from an item spec.

    The parent folder document must be provided and is injected
    automatically into the keyword arguments.

    :param parent: The parent folder
    :type parent: dict
    :returns: The generated item document
    """
    addCreator(args, parent)
    itemModel = loadModel('item')
    args['folder'] = parent
    return itemModel.createItem(**args)

def createFile(parent, path, **args):
    """Create a file document from a file path.

    The provided path should resolve to a valid file.  The
    ``size`` and ``mimeType`` of the file will be automatically
    resolved from this path.

    :param parent: The parent item
    :type parent: dict
    :param path: The local path to a file
    :type path: str
    :returns: The generated file document
    """
    addCreator(args, parent)
    uploadModel = loadModel('upload')

    path = resolvePath(path)
    args['parentType'] = 'item'
    args['parent'] = parent
    args['user'] = args.pop('creator')
    args['size'] = os.path.getsize(path)
    args.setdefault('reference', 'setup_database')

    if 'name' not in args:
        args['name'] = os.path.basename(path)

    if not args.get('mimeType'):
        args['mimeType'] = mimetypes.guess_type(path)[0]

    with open(path, 'rb') as f:
        return uploadModel.uploadFromFile(f, **args)

dispatch = {
    'user': createUser,
    'collection': createCollection,
    'folder': createFolder,
    'item': createItem,
    'file': createFile
}

def createDocument(type, node):
    """Create a generic document type from a spec.

    :param type: The type of document to create
    :type type: str
    :param node: The document spec
    :type node: dict
    :returns: An objected describing the document's children
    """
    # automatically generate children specs from a local path
    path = node.pop('import', None)
    if path:
        importRecursive(type, node, path)

    # create the return object with all possible children
    children = {
        'folder': node.pop('folders', []),
        'item': node.pop('items', []),
        'file': node.pop('files', [])
    }

    # remove invalid child types
    if type in ['user', 'collection']:
        del children['item']
        del children['file']
    elif type == 'folder':
        del children['file']
    elif type == 'item':
        del children['folder']
        del children['item']

    # generate the document
    doc = dispatch[type](**node)

    # set the type for certain models (collection) that don't do it
    # on their own
    doc['_modelType'] = type

    # inject this document as the parent for all child specs
    for childType in children:
        children[childType] = [
            dict(parent=doc, **child) for child in children[childType]
        ]

    return children

def createRecursive(type, node):
    """Generate all documents from a given node recursively.

    :param type: The type of the root node
    :type type: str
    :param node: The root node spec
    :type node: dict
    """
    children = createDocument(type, node)
    for childType in children:
        for childNode in children[childType]:
            createRecursive(childType, childNode)

def createUsers(users):
    """Create a list of users.

    This method is special because we need to generate all users
    before any other type in order to inject the ``creator``
    argument correctly.

    :param users: A list of user specs
    :type users: list
    :returns: A list of all child documents for all users
    """
    folders = []
    for user in users:
        for folder in folders:
            # By default set the creator under a user to that user
            folder.setdefault('creator', user['login'])

        folders.extend(createDocument('user', user)['folder'])
    return folders

def importRecursive(type, parent, root):
    """Generate a document tree from a local path.

    The parent object will be mutated to append the folders and
    directories contained in the provided local path.

    :param type: The parent node type
    :type type: str
    :param parent: The parent document
    :type parent: dict
    :param root: The local path where the import will begin
    :type root: str
    """
    root = resolvePath(root)
    folders = {root: parent}

    for root, dirs, files in os.walk(root, followlinks=True):
        parent = folders[root]
        parent.setdefault('folders', [])
        parent.setdefault('items', [])

        for dir in dirs:
            path = os.path.join(root, dir)
            folder = {
                'name': dir
            }
            folders[path] = folder
            parent['folders'].append(folder)

        for file in files:
            path = os.path.join(root, file)

            parent['items'].append({
                'name': file,
                'files': [{
                    'name': file,
                    'path': path
                }]
            })

def generate(spec):
    """Generate documents from a nested object."""
    global prefix
    prefix = spec.get('prefix', prefix)

    setAssetstore(spec.get('assetstore'))

    # users have to generated first
    spec['folders'] = createUsers(spec.get('users', []))

    for collection in spec.get('collections', []):
        createRecursive('collection', collection)

    for folder in spec.get('folders', []):
        createRecursive('folder', folder)

def main(file):
    """Load a file into memory and generate the documents it describes."""
    prefix = os.path.dirname(file)
    with open(file) as f:
        import yaml
        spec = yaml.safe_load(f)
        spec.setdefault('prefix', prefix)
        generate(spec)

# For standalone usage, this file can be called as a script to generate
# a testing environment.  You should provide an empty database as an
# environment variable.  For example:
#
#   GIRDER_MONGO_URI='mongodb://127.0.0.1:27017/import_test' python setup_database.py spec.yml
#   GIRDER_MONGO_URI='mongodb://127.0.0.1:27017/import_test' girder-server
if __name__ == '__main__':
    import sys
    main(sys.argv[1])
<|endoftext|>


<commit_before>from __future__ import unicode_literals, division, absolute_import
import urllib2
import time
import logging
from datetime import timedelta, datetime
from urlparse import urlparse

import requests
# Allow some request objects to be imported from here instead of requests
from requests import RequestException, HTTPError

from flexget import __version__ as version
from flexget.utils.tools import parse_timedelta, TimedDict

log = logging.getLogger('requests')

# Don't emit info level urllib3 log messages or below
logging.getLogger('requests.packages.urllib3').setLevel(logging.WARNING)

# Time to wait before trying an unresponsive site again
WAIT_TIME = timedelta(seconds=60)
# Remembers sites that have timed out
unresponsive_hosts = TimedDict(WAIT_TIME)

def is_unresponsive(url):
    """
    Checks if host of given url has timed out within WAIT_TIME

    :param url: The url to check
    :return: True if the host has timed out within WAIT_TIME
    :rtype: bool
    """
    host = urlparse(url).hostname
    return host in unresponsive_hosts

def set_unresponsive(url):
    """
    Marks the host of a given url as unresponsive

    :param url: The url that timed out
    """
    host = urlparse(url).hostname
    if host in unresponsive_hosts:
        # If somehow this is called again before previous timer clears, don't refresh
        return
    unresponsive_hosts[host] = True

def wait_for_domain(url, delay_dict):
    for domain, domain_dict in delay_dict.iteritems():
        if domain in url:
            next_req = domain_dict.get('next_req')
            if next_req and datetime.now() < next_req:
                wait_time = next_req - datetime.now()
                seconds = wait_time.seconds + (wait_time.microseconds / 1000000.0)
                log.debug('Waiting %.2f seconds until next request to %s' % (seconds, domain))
                # Sleep until it is time for the next request
                time.sleep(seconds)
            # Record the next allowable request time for this domain
            domain_dict['next_req'] = datetime.now() + domain_dict['delay']
            break

def _wrap_urlopen(url, timeout=None):
    """
    Handles alternate schemes using urllib, wraps the response in a requests.Response

    This is not installed as an adapter in requests, since urls without network locations
    (e.g. file:///somewhere) will cause errors

    """
    try:
        raw = urllib2.urlopen(url, timeout=timeout)
    except IOError as e:
        msg = 'Error getting %s: %s' % (url, e)
        log.error(msg)
        raise RequestException(msg)
    resp = requests.Response()
    resp.raw = raw
    # requests passes the `decode_content` kwarg to read
    orig_read = raw.read
    resp.raw.read = lambda size, **kwargs: orig_read(size)
    resp.status_code = raw.code or 200
    resp.headers = requests.structures.CaseInsensitiveDict(raw.headers)
    return resp

class Session(requests.Session):
    """
    Subclass of requests Session class which defines some of our own defaults, records unresponsive sites,
    and raises errors by default.

    """

    def __init__(self, timeout=30, max_retries=1):
        """Set some defaults for our session if not explicitly defined."""
        requests.Session.__init__(self)
        self.timeout = timeout
        self.stream = True
        self.adapters['http://'].max_retries = max_retries
        # Stores min intervals between requests for certain sites
        self.domain_delay = {}
        self.headers.update({'User-Agent': 'FlexGet/%s (www.flexget.com)' % version})

    def add_cookiejar(self, cookiejar):
        """
        Merges cookies from `cookiejar` into cookiejar for this session.

        :param cookiejar: CookieJar instance to add to the session.
        """
        for cookie in cookiejar:
            self.cookies.set_cookie(cookie)

    def set_domain_delay(self, domain, delay):
        """
        Registers a minimum interval between requests to `domain`

        :param domain: The domain to set the interval on
        :param delay: The amount of time between requests, can be a timedelta or string like '3 seconds'
        """
        self.domain_delay[domain] = {'delay': parse_timedelta(delay)}

    def request(self, method, url, *args, **kwargs):
        """
        Does a request, but raises Timeout immediately if site is known to timeout, and records sites that timeout.
        Also raises errors getting the content by default.
        """

        # Raise Timeout right away if site is known to timeout
        if is_unresponsive(url):
            raise requests.Timeout('Requests to this site have timed out recently. Waiting before trying again.')

        # Delay, if needed, before another request to this site
        wait_for_domain(url, self.domain_delay)

        kwargs.setdefault('timeout', self.timeout)
        raise_status = kwargs.pop('raise_status', True)

        # If we do not have an adapter for this url, pass it off to urllib
        if not any(url.startswith(adapter) for adapter in self.adapters):
            return _wrap_urlopen(url, timeout=kwargs['timeout'])

        try:
            result = requests.Session.request(self, method, url, *args, **kwargs)
        except (requests.Timeout, requests.ConnectionError):
            # Mark this site in known unresponsive list
            set_unresponsive(url)
            raise

        if raise_status:
            result.raise_for_status()

        return result

# Define some module level functions that use our Session, so this module can be used like main requests module
def request(method, url, **kwargs):
    s = kwargs.pop('session', Session())
    return s.request(method=method, url=url, **kwargs)

def get(url, **kwargs):
    """Sends a GET request. Returns :class:`Response` object.

    :param url: URL for the new :class:`Request` object.
    :param kwargs: Optional arguments that ``request`` takes.
    """
    kwargs.setdefault('allow_redirects', True)
    return request('get', url, **kwargs)

def post(url, data=None, **kwargs):
    """Sends a POST request. Returns :class:`Response` object.

    :param url: URL for the new :class:`Request` object.
    :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
    :param kwargs: Optional arguments that ``request`` takes.
    """
    return request('post', url, data=data, **kwargs)
<commit_msg>apply warn log filter to proper logger name to exclude INFO log messages when opening url's (i.e. Starting new HTTP connection (1): ... )<commit_after>from __future__ import unicode_literals, division, absolute_import
import urllib2
import time
import logging
from datetime import timedelta, datetime
from urlparse import urlparse

import requests
# Allow some request objects to be imported from here instead of requests
from requests import RequestException, HTTPError

from flexget import __version__ as version
from flexget.utils.tools import parse_timedelta, TimedDict

log = logging.getLogger('requests')

# Don't emit info level urllib3 log messages or below
logging.getLogger('urllib3').setLevel(logging.WARNING)

# Time to wait before trying an unresponsive site again
WAIT_TIME = timedelta(seconds=60)
# Remembers sites that have timed out
unresponsive_hosts = TimedDict(WAIT_TIME)

def is_unresponsive(url):
    """
    Checks if host of given url has timed out within WAIT_TIME

    :param url: The url to check
    :return: True if the host has timed out within WAIT_TIME
    :rtype: bool
    """
    host = urlparse(url).hostname
    return host in unresponsive_hosts

def set_unresponsive(url):
    """
    Marks the host of a given url as unresponsive

    :param url: The url that timed out
    """
    host = urlparse(url).hostname
    if host in unresponsive_hosts:
        # If somehow this is called again before previous timer clears, don't refresh
        return
    unresponsive_hosts[host] = True

def wait_for_domain(url, delay_dict):
    for domain, domain_dict in delay_dict.iteritems():
        if domain in url:
            next_req = domain_dict.get('next_req')
            if next_req and datetime.now() < next_req:
                wait_time = next_req - datetime.now()
                seconds = wait_time.seconds + (wait_time.microseconds / 1000000.0)
                log.debug('Waiting %.2f seconds until next request to %s' % (seconds, domain))
                # Sleep until it is time for the next request
                time.sleep(seconds)
            # Record the next allowable request time for this domain
            domain_dict['next_req'] = datetime.now() + domain_dict['delay']
            break

def _wrap_urlopen(url, timeout=None):
    """
    Handles alternate schemes using urllib, wraps the response in a requests.Response

    This is not installed as an adapter in requests, since urls without network locations
    (e.g. file:///somewhere) will cause errors

    """
    try:
        raw = urllib2.urlopen(url, timeout=timeout)
    except IOError as e:
        msg = 'Error getting %s: %s' % (url, e)
        log.error(msg)
        raise RequestException(msg)
    resp = requests.Response()
    resp.raw = raw
    # requests passes the `decode_content` kwarg to read
    orig_read = raw.read
    resp.raw.read = lambda size, **kwargs: orig_read(size)
    resp.status_code = raw.code or 200
    resp.headers = requests.structures.CaseInsensitiveDict(raw.headers)
    return resp

class Session(requests.Session):
    """
    Subclass of requests Session class which defines some of our own defaults, records unresponsive sites,
    and raises errors by default.

    """

    def __init__(self, timeout=30, max_retries=1):
        """Set some defaults for our session if not explicitly defined."""
        requests.Session.__init__(self)
        self.timeout = timeout
        self.stream = True
        self.adapters['http://'].max_retries = max_retries
        # Stores min intervals between requests for certain sites
        self.domain_delay = {}
        self.headers.update({'User-Agent': 'FlexGet/%s (www.flexget.com)' % version})

    def add_cookiejar(self, cookiejar):
        """
        Merges cookies from `cookiejar` into cookiejar for this session.

        :param cookiejar: CookieJar instance to add to the session.
        """
        for cookie in cookiejar:
            self.cookies.set_cookie(cookie)

    def set_domain_delay(self, domain, delay):
        """
        Registers a minimum interval between requests to `domain`

        :param domain: The domain to set the interval on
        :param delay: The amount of time between requests, can be a timedelta or string like '3 seconds'
        """
        self.domain_delay[domain] = {'delay': parse_timedelta(delay)}

    def request(self, method, url, *args, **kwargs):
        """
        Does a request, but raises Timeout immediately if site is known to timeout, and records sites that timeout.
        Also raises errors getting the content by default.
        """

        # Raise Timeout right away if site is known to timeout
        if is_unresponsive(url):
            raise requests.Timeout('Requests to this site have timed out recently. Waiting before trying again.')

        # Delay, if needed, before another request to this site
        wait_for_domain(url, self.domain_delay)

        kwargs.setdefault('timeout', self.timeout)
        raise_status = kwargs.pop('raise_status', True)

        # If we do not have an adapter for this url, pass it off to urllib
        if not any(url.startswith(adapter) for adapter in self.adapters):
            return _wrap_urlopen(url, timeout=kwargs['timeout'])

        try:
            result = requests.Session.request(self, method, url, *args, **kwargs)
        except (requests.Timeout, requests.ConnectionError):
            # Mark this site in known unresponsive list
            set_unresponsive(url)
            raise

        if raise_status:
            result.raise_for_status()

        return result

# Define some module level functions that use our Session, so this module can be used like main requests module
def request(method, url, **kwargs):
    s = kwargs.pop('session', Session())
    return s.request(method=method, url=url, **kwargs)

def get(url, **kwargs):
    """Sends a GET request. Returns :class:`Response` object.

    :param url: URL for the new :class:`Request` object.
    :param kwargs: Optional arguments that ``request`` takes.
    """
    kwargs.setdefault('allow_redirects', True)
    return request('get', url, **kwargs)

def post(url, data=None, **kwargs):
    """Sends a POST request. Returns :class:`Response` object.

    :param url: URL for the new :class:`Request` object.
    :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
    :param kwargs: Optional arguments that ``request`` takes.
    """
    return request('post', url, data=data, **kwargs)
<|endoftext|>


<commit_before>"""The IPython ZMQ-based parallel computing interface.

Authors:

* MinRK
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2011 The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import os
import warnings

import zmq

from IPython.config.configurable import MultipleInstanceError
from IPython.zmq import check_for_zmq

if os.name == 'nt':
    min_pyzmq = '2.1.7'
else:
    min_pyzmq = '2.1.4'

check_for_zmq(min_pyzmq, 'IPython.parallel')

from IPython.utils.pickleutil import Reference

from .client.asyncresult import *
from .client.client import Client
from .client.remotefunction import *
from .client.view import *
from .util import interactive
from .controller.dependency import *

#-----------------------------------------------------------------------------
# Functions
#-----------------------------------------------------------------------------

def bind_kernel(**kwargs):
    """Bind an Engine's Kernel to be used as a full IPython kernel.

    This allows a running Engine to be used simultaneously as a full IPython kernel
    with the QtConsole or other frontends.

    This function returns immediately.
    """
    from IPython.zmq.ipkernel import IPKernelApp
    from IPython.parallel.apps.ipengineapp import IPEngineApp

    # first check for IPKernelApp, in which case this should be a no-op
    # because there is already a bound kernel
    if IPKernelApp.initialized() and isinstance(IPKernelApp._instance, IPKernelApp):
        return

    if IPEngineApp.initialized():
        try:
            app = IPEngineApp.instance()
        except MultipleInstanceError:
            pass
        else:
            return app.bind_kernel(**kwargs)

    raise RuntimeError("bind_kernel be called from an IPEngineApp instance")

<commit_msg>include errors in IPython.parallel namespace<commit_after>"""The IPython ZMQ-based parallel computing interface.

Authors:

* MinRK
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2011 The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import os
import warnings

import zmq

from IPython.config.configurable import MultipleInstanceError
from IPython.zmq import check_for_zmq

if os.name == 'nt':
    min_pyzmq = '2.1.7'
else:
    min_pyzmq = '2.1.4'

check_for_zmq(min_pyzmq, 'IPython.parallel')

from IPython.utils.pickleutil import Reference

from .client.asyncresult import *
from .client.client import Client
from .client.remotefunction import *
from .client.view import *
from .controller.dependency import *
from .error import *
from .util import interactive

#-----------------------------------------------------------------------------
# Functions
#-----------------------------------------------------------------------------

def bind_kernel(**kwargs):
    """Bind an Engine's Kernel to be used as a full IPython kernel.

    This allows a running Engine to be used simultaneously as a full IPython kernel
    with the QtConsole or other frontends.

    This function returns immediately.
    """
    from IPython.zmq.ipkernel import IPKernelApp
    from IPython.parallel.apps.ipengineapp import IPEngineApp

    # first check for IPKernelApp, in which case this should be a no-op
    # because there is already a bound kernel
    if IPKernelApp.initialized() and isinstance(IPKernelApp._instance, IPKernelApp):
        return

    if IPEngineApp.initialized():
        try:
            app = IPEngineApp.instance()
        except MultipleInstanceError:
            pass
        else:
            return app.bind_kernel(**kwargs)

    raise RuntimeError("bind_kernel be called from an IPEngineApp instance")

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_webanalytics
# Purpose:      SpiderFoot plug-in for scanning retrieved content by other
#               modules (such as sfp_spider) and identifying web analytics IDs.
#
# Author:      Brendan Coles <bcoles@gmail.com>
#
# Created:     2018-10-28
# Copyright:   (c) Brendan Coles 2018
# Licence:     GPL
# -------------------------------------------------------------------------------

import re
from hashlib import sha256
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_webanalytics(SpiderFootPlugin):
    """Web Analytics:Footprint,Investigate:Content Analysis::Identify web analytics IDs in scraped webpages."""

    opts = {}
    optdescs = {}

    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["TARGET_WEB_CONTENT"]

    # What events this module produces
    def producedEvents(self):
        return ["WEB_ANALYTICS_ID"]

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data
        sourceData = self.sf.hashstring(eventData)

        if sourceData in self.results:
            return None
        else:
            self.results[sourceData] = True

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        matches = re.findall("ua\-\d{4,10}\-\d{1,4}", eventData, re.IGNORECASE)
        for m in matches:
            self.sf.debug("Google Analytics match: " + m)
            evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Google Analytics: " + m, self.__name__, event)
            self.notifyListeners(evt)

        return None

# End of sfp_webanalytics class
<commit_msg>Add Google AdSense to sfp_webanalytics<commit_after># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_webanalytics
# Purpose:      SpiderFoot plug-in for scanning retrieved content by other
#               modules (such as sfp_spider) and identifying web analytics IDs.
#
# Author:      Brendan Coles <bcoles@gmail.com>
#
# Created:     2018-10-28
# Copyright:   (c) Brendan Coles 2018
# Licence:     GPL
# -------------------------------------------------------------------------------

import re
from hashlib import sha256
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_webanalytics(SpiderFootPlugin):
    """Web Analytics:Footprint,Investigate:Content Analysis::Identify web analytics IDs in scraped webpages."""

    opts = {}
    optdescs = {}

    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["TARGET_WEB_CONTENT"]

    # What events this module produces
    def producedEvents(self):
        return ["WEB_ANALYTICS_ID"]

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data
        sourceData = self.sf.hashstring(eventData)

        if sourceData in self.results:
            return None

        self.results[sourceData] = True

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        # Google Analytics
        matches = re.findall(r"\bua\-\d{4,10}\-\d{1,4}\b", eventData, re.IGNORECASE)
        for m in matches:
            if m.lower().startswith('ua-000000-'):
                continue
            if m.lower().startswith('ua-123456-'):
                continue
            if m.lower().startswith('ua-123456789'):
                continue

            self.sf.debug("Google Analytics match: " + m)
            evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Google Analytics: " + m, self.__name__, event)
            self.notifyListeners(evt)

        # Google AdSense
        matches = re.findall(r"\b(pub-\d{10,20})\b", eventData, re.IGNORECASE)
        for m in matches:
            if m.lower().startswith('pub-123456789'):
                continue

            self.sf.debug("Google AdSense match: " + m)
            evt = SpiderFootEvent("WEB_ANALYTICS_ID", "Google AdSense: " + m, self.__name__, event)
            self.notifyListeners(evt)

        return None

# End of sfp_webanalytics class
<|endoftext|>


<commit_before># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Script for starting control service server.
"""

from twisted.python.usage import Options
from twisted.internet.endpoints import TCP4ServerEndpoint

from .httpapi import create_api_service
from ..common.script import (
    flocker_standard_options, FlockerScriptRunner, main_for_service)

@flocker_standard_options
class ControlOptions(Options):
    """
    Command line options for ``flocker-control`` cluster management process.
    """
    optParameters = [
        # XXX also choose where configuration will be stored
        ["port", "p", 4523, "The port to listen on.", int],
        ]

class ControlScript(object):
    """
    A command to start a long-running process to control a Flocker
    cluster.
    """
    def main(self, reactor, options):
        # XXX also start persistence service
        api_service = create_api_service(
            TCP4ServerEndpoint(reactor, options["port"]))
        return main_for_service(reactor, api_service)

def flocker_control_main():
    return FlockerScriptRunner(
        script=ControlScript(),
        options=ControlOptions()
    ).main()
<commit_msg>Better explanation.<commit_after># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Script for starting control service server.
"""

from twisted.python.usage import Options
from twisted.internet.endpoints import TCP4ServerEndpoint

from .httpapi import create_api_service
from ..common.script import (
    flocker_standard_options, FlockerScriptRunner, main_for_service)

@flocker_standard_options
class ControlOptions(Options):
    """
    Command line options for ``flocker-control`` cluster management process.
    """
    optParameters = [
        # XXX also choose where persistent will be stored
        ["port", "p", 4523, "The port to listen on.", int],
        ]

class ControlScript(object):
    """
    A command to start a long-running process to control a Flocker
    cluster.
    """
    def main(self, reactor, options):
        # XXX also start persistence service
        api_service = create_api_service(
            TCP4ServerEndpoint(reactor, options["port"]))
        return main_for_service(reactor, api_service)

def flocker_control_main():
    return FlockerScriptRunner(
        script=ControlScript(),
        options=ControlOptions()
    ).main()
<|endoftext|>


<commit_before># Copyright Hybrid Logic Ltd.  See LICENSE file for details.
# -*- test-case-name: flocker.volume.test.test_service -*-

"""Volume manager service, the main entry point that manages volumes."""

from __future__ import absolute_import

import os
import json
import stat
from uuid import UUID, uuid4

from characteristic import attributes

from twisted.python.filepath import FilePath
from twisted.application.service import Service
from twisted.internet.endpoints import ProcessEndpoint, connectProtocol
from twisted.internet import reactor
from twisted.internet.defer import fail
from twisted.internet.task import LoopingCall

# We might want to make these utilities shared, rather than in zfs
# module... but in this case the usage is temporary and should go away as
# part of https://github.com/ClusterHQ/flocker/issues/64
from .filesystems.zfs import _AccumulatingProtocol, CommandFailed

DEFAULT_CONFIG_PATH = FilePath(b"/etc/flocker/volume.json")

WAIT_FOR_VOLUME_INTERVAL = 0.1

class CreateConfigurationError(Exception):
    """Create the configuration file failed."""

class VolumeService(Service):
    """Main service for volume management.

    :ivar unicode uuid: A unique identifier for this particular node's
        volume manager. Only available once the service has started.
    """

    def __init__(self, config_path, pool, reactor):
        """
        :param FilePath config_path: Path to the volume manager config file.
        :param pool: An object that is both a
            ``flocker.volume.filesystems.interface.IStoragePool`` provider
            and a ``twisted.application.service.IService`` provider.
        :param reactor: A ``twisted.internet.interface.IReactorTime`` provider.
        """
        self._config_path = config_path
        self._pool = pool
        self._reactor = reactor

    def startService(self):
        parent = self._config_path.parent()
        try:
            if not parent.exists():
                parent.makedirs()
            if not self._config_path.exists():
                uuid = unicode(uuid4())
                self._config_path.setContent(json.dumps({u"uuid": uuid,
                                                         u"version": 1}))
        except OSError as e:
            raise CreateConfigurationError(e.args[1])
        config = json.loads(self._config_path.getContent())
        self.uuid = config[u"uuid"]
        #return self._pool.startService()

    def create(self, name):
        """Create a new volume.

        :param unicode name: The name of the volume.

        :return: A ``Deferred`` that fires with a :class:`Volume`.
        """
        volume = Volume(uuid=self.uuid, name=name, _pool=self._pool)
        d = self._pool.create(volume)

        def created(filesystem):
            filesystem.get_path().chmod(
                # 0o777 the long way:
                stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
            return volume
        d.addCallback(created)
        return d

    def get(self, name, uuid=None):
        """
        Return a locally-owned ``Volume`` with the given name.

        Whether or not this volume actually exists is not checked in any
        way.

        :param unicode name: The name of the volume.

        :param uuid: Either ``None``, in which case the local UUID will be
            used, or a UUID to use for the volume.

        :return: A ``Volume``.
        """
        #if uuid is None:
        #    uuid=self.uuid
        return Volume(uuid=self.uuid, name=name, _pool=self._pool)

    def wait_for_volume(self, name):
        """
        Wait for a volume by the given name, owned by thus service, to exist.

        Polls the storage pool for the specified volume to appear.

        :param unicode name: The name of the volume.

        :return: A ``Deferred`` that fires with a :class:`Volume`.
        """
        volume = Volume(uuid=self.uuid, name=name, _pool=self._pool)

        def check_for_volume(volumes):
            if volume in volumes:
                call.stop()

        def loop():
            d = self.enumerate()
            d.addCallback(check_for_volume)
            return d

        call = LoopingCall(loop)
        call.clock = self._reactor
        d = call.start(WAIT_FOR_VOLUME_INTERVAL)
        d.addCallback(lambda _: volume)
        return d

    def enumerate(self):
        """Get a listing of all volumes managed by this service.

        :return: A ``Deferred`` that fires with an iterator of :class:`Volume`.
        """
        enumerating = self._pool.enumerate()

        def enumerated(filesystems):
            for filesystem in filesystems:
                # XXX It so happens that this works but it's kind of a
                # fragile way to recover the information:
                #    https://github.com/ClusterHQ/flocker/issues/78
                basename = filesystem.get_path().basename()
                try:
                    uuid, name = basename.split(b".", 1)
                    uuid = UUID(uuid)
                except ValueError:
                    # If we can't split on `.` and get two parts then it's not
                    # a filesystem Flocker is managing.  Likewise if we can't
                    # interpret the bit before the `.` as a UUID.  Perhaps a
                    # user created it, who knows.  Just ignore it.
                    continue

                # Probably shouldn't yield this volume if the uuid doesn't
                # match this service's uuid.
                yield Volume(
                    uuid=unicode(uuid),
                    name=name.decode('utf8'),
                    _pool=self._pool)
        enumerating.addCallback(enumerated)
        return enumerating

    def push(self, volume, destination, config_path=DEFAULT_CONFIG_PATH):
        """
        Push the latest data in the volume to a remote destination.

        This is a blocking API for now.

        Only locally owned volumes (i.e. volumes whose ``uuid`` matches
        this service's) can be pushed.

        :param Volume volume: The volume to push.

        :param IRemoteVolumeManager destination: The remote volume manager
            to push to.

        :param FilePath config_path: Path to configuration file for the
            remote ``flocker-volume``.

        :raises ValueError: If the uuid of the volume is different than
            our own; only locally-owned volumes can be pushed.
        """
        if volume.uuid != self.uuid:
            raise ValueError()
        fs = volume.get_filesystem()
        with destination.receive(volume) as receiver:
            with fs.reader() as contents:
                for chunk in iter(lambda: contents.read(1024 * 1024), b""):
                    receiver.write(chunk)

    def receive(self, volume_uuid, volume_name, input_file):
        """
        Process a volume's data that can be read from a file-like object.

        This is a blocking API for now.

        Only remotely owned volumes (i.e. volumes whose ``uuid`` do not match
        this service's) can be received.

        :param unicode volume_uuid: The volume's UUID.
        :param unicode volume_name: The volume's name.
        :param input_file: A file-like object, typically ``sys.stdin``, from
            which to read the data.

        :raises ValueError: If the uuid of the volume matches our own;
            remote nodes can't overwrite locally-owned volumes.
        """
        if volume_uuid == self.uuid:
            raise ValueError()
        volume = Volume(uuid=volume_uuid, name=volume_name, _pool=self._pool)
        with volume.get_filesystem().writer() as writer:
            for chunk in iter(lambda: input_file.read(1024 * 1024), b""):
                writer.write(chunk)

    def acquire(self, volume_uuid, volume_name):
        """
        Take ownership of a volume.

        This is a blocking API for now.

        Only remotely owned volumes (i.e. volumes whose ``uuid`` do not match
        this service's) can be acquired.

        :param unicode volume_uuid: The volume owner's UUID.
        :param unicode volume_name: The volume's name.

        :return: ``Deferred`` that fires on success, or errbacks with
            ``ValueError`` If the uuid of the volume matches our own.
        """
        if volume_uuid == self.uuid:
            return fail(ValueError("Can't acquire already-owned volume"))
        volume = Volume(uuid=volume_uuid, name=volume_name, _pool=self._pool)
        return volume.change_owner(self.uuid)

    def handoff(self, volume, destination):
        """
        Handoff a locally owned volume to a remote destination.

        The remote destination will be the new owner of the volume.

        This is a blocking API for now (but it does return a ``Deferred``
        for success/failure).

        :param Volume volume: The volume to handoff.
        :param IRemoteVolumeManager destination: The remote volume manager
            to handoff to.

        :return: ``Deferred`` that fires when the handoff has finished, or
            errbacks on error (specifcally with a ``ValueError`` if the
            volume is not locally owned).
        """
        try:
            self.push(volume, destination)
        except ValueError:
            return fail()
        remote_uuid = destination.acquire(volume)
        return volume.change_owner(remote_uuid)

# Communication with Docker should be done via its API, not with this
# approach, but that depends on unreleased Twisted 14.1:
# https://github.com/ClusterHQ/flocker/issues/64
def _docker_command(reactor, arguments):
    """Run the ``docker`` command-line tool with the given arguments.

    :param reactor: A ``IReactorProcess`` provider.

    :param arguments: A ``list`` of ``bytes``, command-line arguments to
    ``docker``.

    :return: A :class:`Deferred` firing with the bytes of the result (on
        exit code 0), or errbacking with :class:`CommandFailed` or
        :class:`BadArguments` depending on the exit code (1 or 2).
    """
    endpoint = ProcessEndpoint(reactor, b"docker", [b"docker"] + arguments,
                               os.environ)
    d = connectProtocol(endpoint, _AccumulatingProtocol())
    d.addCallback(lambda protocol: protocol._result)
    return d

@attributes(["uuid", "name", "service"])
class Volume(object):
    """A data volume's identifier.

    :ivar unicode uuid: The UUID of the volume manager that owns this volume.
    :ivar unicode name: The name of the volume. Since volume names must
        match Docker container names, the characters used should be limited to
        those that Docker allows for container names.
    :ivar service: The ``VolumeService`` in whose pool this volume is stored.
    """
    # def is_locally_owned(self):
    #     """
    #     Return whether this volume is locally owned.
    #     """
    #     return self.uuid == self.service.uuid

    def change_owner(self, new_owner_uuid):
        """
        Change which volume manager owns this volume.

        :param unicode new_owner_uuid: The UUID of the new owner.

        :return: ``Deferred`` that fires with a new :class:`Volume`
            instance once the ownership has been changed.
        """
        new_volume = Volume(uuid=new_owner_uuid, name=self.name,
                            _pool=self._pool)
        d = self._pool.change_owner(self, new_volume)

        def filesystem_changed(_):
            return new_volume
        d.addCallback(filesystem_changed)
        return d

    def get_filesystem(self):
        """Return the volume's filesystem.

        :return: The ``IFilesystem`` provider for the volume.
        """
        return self._pool.get(self)

    @property
    def _container_name(self):
        """Return the corresponding Docker container name.

        :return: Container name as ``bytes``.
        """
        return b"%s-data" % (self.name.encode("ascii"),)

    def expose_to_docker(self, mount_path):
        """
        Create a container that will expose the volume to Docker at the given
        mount path.

        Can be called multiple times. Mount paths from previous calls will
        be overridden.

        :param FilePath mount_path: The path at which to mount the volume
            within the container.

        :return: ``Deferred`` firing when the operation is done.
        """
        local_path = self.get_filesystem().get_path().path
        mount_path = mount_path.path
        d = self.remove_from_docker()
        d.addCallback(
            lambda _:
                _docker_command(reactor,
                                [b"run", b"--name", self._container_name,
                                 b"--volume=%s:%s:rw" % (local_path,
                                                         mount_path),
                                 b"busybox", b"/bin/true"]))
        return d

    def remove_from_docker(self):
        """
        Remove the Docker container created for the volume.

        If no container exists this will silently do nothing.

        :return: ``Deferred`` firing with ``None`` when the operation is
           done.
        """
        d = _docker_command(reactor, [b"rm", self._container_name])
        d.addErrback(lambda failure: failure.trap(CommandFailed))
        d.addCallback(lambda _: None)
        return d
<commit_msg>Revert most of the changes that were moved into 584.<commit_after># Copyright Hybrid Logic Ltd.  See LICENSE file for details.
# -*- test-case-name: flocker.volume.test.test_service -*-

"""Volume manager service, the main entry point that manages volumes."""

from __future__ import absolute_import

import os
import json
import stat
from uuid import UUID, uuid4

from characteristic import attributes

from twisted.python.filepath import FilePath
from twisted.application.service import Service
from twisted.internet.endpoints import ProcessEndpoint, connectProtocol
from twisted.internet import reactor
from twisted.internet.defer import fail
from twisted.internet.task import LoopingCall

# We might want to make these utilities shared, rather than in zfs
# module... but in this case the usage is temporary and should go away as
# part of https://github.com/ClusterHQ/flocker/issues/64
from .filesystems.zfs import _AccumulatingProtocol, CommandFailed

DEFAULT_CONFIG_PATH = FilePath(b"/etc/flocker/volume.json")

WAIT_FOR_VOLUME_INTERVAL = 0.1

class CreateConfigurationError(Exception):
    """Create the configuration file failed."""

class VolumeService(Service):
    """Main service for volume management.

    :ivar unicode uuid: A unique identifier for this particular node's
        volume manager. Only available once the service has started.
    """

    def __init__(self, config_path, pool, reactor):
        """
        :param FilePath config_path: Path to the volume manager config file.
        :param pool: An object that is both a
            ``flocker.volume.filesystems.interface.IStoragePool`` provider
            and a ``twisted.application.service.IService`` provider.
        :param reactor: A ``twisted.internet.interface.IReactorTime`` provider.
        """
        self._config_path = config_path
        self._pool = pool
        self._reactor = reactor

    def startService(self):
        parent = self._config_path.parent()
        try:
            if not parent.exists():
                parent.makedirs()
            if not self._config_path.exists():
                uuid = unicode(uuid4())
                self._config_path.setContent(json.dumps({u"uuid": uuid,
                                                         u"version": 1}))
        except OSError as e:
            raise CreateConfigurationError(e.args[1])
        config = json.loads(self._config_path.getContent())
        self.uuid = config[u"uuid"]
        #return self._pool.startService()

    def create(self, name):
        """Create a new volume.

        :param unicode name: The name of the volume.

        :return: A ``Deferred`` that fires with a :class:`Volume`.
        """
        volume = Volume(uuid=self.uuid, name=name, _pool=self._pool)
        d = self._pool.create(volume)

        def created(filesystem):
            filesystem.get_path().chmod(
                # 0o777 the long way:
                stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
            return volume
        d.addCallback(created)
        return d

    def get(self, name):
        """
        Return a locally-owned ``Volume`` with the given name.

        Whether or not this volume actually exists is not checked in any
        way.

        :param unicode name: The name of the volume.

        :param uuid: Either ``None``, in which case the local UUID will be
            used, or a UUID to use for the volume.

        :return: A ``Volume``.
        """
        return Volume(uuid=self.uuid, name=name, _pool=self._pool)

    def wait_for_volume(self, name):
        """
        Wait for a volume by the given name, owned by thus service, to exist.

        Polls the storage pool for the specified volume to appear.

        :param unicode name: The name of the volume.

        :return: A ``Deferred`` that fires with a :class:`Volume`.
        """
        volume = Volume(uuid=self.uuid, name=name, _pool=self._pool)

        def check_for_volume(volumes):
            if volume in volumes:
                call.stop()

        def loop():
            d = self.enumerate()
            d.addCallback(check_for_volume)
            return d

        call = LoopingCall(loop)
        call.clock = self._reactor
        d = call.start(WAIT_FOR_VOLUME_INTERVAL)
        d.addCallback(lambda _: volume)
        return d

    def enumerate(self):
        """Get a listing of all volumes managed by this service.

        :return: A ``Deferred`` that fires with an iterator of :class:`Volume`.
        """
        enumerating = self._pool.enumerate()

        def enumerated(filesystems):
            for filesystem in filesystems:
                # XXX It so happens that this works but it's kind of a
                # fragile way to recover the information:
                #    https://github.com/ClusterHQ/flocker/issues/78
                basename = filesystem.get_path().basename()
                try:
                    uuid, name = basename.split(b".", 1)
                    uuid = UUID(uuid)
                except ValueError:
                    # If we can't split on `.` and get two parts then it's not
                    # a filesystem Flocker is managing.  Likewise if we can't
                    # interpret the bit before the `.` as a UUID.  Perhaps a
                    # user created it, who knows.  Just ignore it.
                    continue

                # Probably shouldn't yield this volume if the uuid doesn't
                # match this service's uuid.
                yield Volume(
                    uuid=unicode(uuid),
                    name=name.decode('utf8'),
                    _pool=self._pool)
        enumerating.addCallback(enumerated)
        return enumerating

    def push(self, volume, destination, config_path=DEFAULT_CONFIG_PATH):
        """
        Push the latest data in the volume to a remote destination.

        This is a blocking API for now.

        Only locally owned volumes (i.e. volumes whose ``uuid`` matches
        this service's) can be pushed.

        :param Volume volume: The volume to push.

        :param IRemoteVolumeManager destination: The remote volume manager
            to push to.

        :param FilePath config_path: Path to configuration file for the
            remote ``flocker-volume``.

        :raises ValueError: If the uuid of the volume is different than
            our own; only locally-owned volumes can be pushed.
        """
        if volume.uuid != self.uuid:
            raise ValueError()
        fs = volume.get_filesystem()
        with destination.receive(volume) as receiver:
            with fs.reader() as contents:
                for chunk in iter(lambda: contents.read(1024 * 1024), b""):
                    receiver.write(chunk)

    def receive(self, volume_uuid, volume_name, input_file):
        """
        Process a volume's data that can be read from a file-like object.

        This is a blocking API for now.

        Only remotely owned volumes (i.e. volumes whose ``uuid`` do not match
        this service's) can be received.

        :param unicode volume_uuid: The volume's UUID.
        :param unicode volume_name: The volume's name.
        :param input_file: A file-like object, typically ``sys.stdin``, from
            which to read the data.

        :raises ValueError: If the uuid of the volume matches our own;
            remote nodes can't overwrite locally-owned volumes.
        """
        if volume_uuid == self.uuid:
            raise ValueError()
        volume = Volume(uuid=volume_uuid, name=volume_name, _pool=self._pool)
        with volume.get_filesystem().writer() as writer:
            for chunk in iter(lambda: input_file.read(1024 * 1024), b""):
                writer.write(chunk)

    def acquire(self, volume_uuid, volume_name):
        """
        Take ownership of a volume.

        This is a blocking API for now.

        Only remotely owned volumes (i.e. volumes whose ``uuid`` do not match
        this service's) can be acquired.

        :param unicode volume_uuid: The volume owner's UUID.
        :param unicode volume_name: The volume's name.

        :return: ``Deferred`` that fires on success, or errbacks with
            ``ValueError`` If the uuid of the volume matches our own.
        """
        if volume_uuid == self.uuid:
            return fail(ValueError("Can't acquire already-owned volume"))
        volume = Volume(uuid=volume_uuid, name=volume_name, _pool=self._pool)
        return volume.change_owner(self.uuid)

    def handoff(self, volume, destination):
        """
        Handoff a locally owned volume to a remote destination.

        The remote destination will be the new owner of the volume.

        This is a blocking API for now (but it does return a ``Deferred``
        for success/failure).

        :param Volume volume: The volume to handoff.
        :param IRemoteVolumeManager destination: The remote volume manager
            to handoff to.

        :return: ``Deferred`` that fires when the handoff has finished, or
            errbacks on error (specifcally with a ``ValueError`` if the
            volume is not locally owned).
        """
        try:
            self.push(volume, destination)
        except ValueError:
            return fail()
        remote_uuid = destination.acquire(volume)
        return volume.change_owner(remote_uuid)

# Communication with Docker should be done via its API, not with this
# approach, but that depends on unreleased Twisted 14.1:
# https://github.com/ClusterHQ/flocker/issues/64
def _docker_command(reactor, arguments):
    """Run the ``docker`` command-line tool with the given arguments.

    :param reactor: A ``IReactorProcess`` provider.

    :param arguments: A ``list`` of ``bytes``, command-line arguments to
    ``docker``.

    :return: A :class:`Deferred` firing with the bytes of the result (on
        exit code 0), or errbacking with :class:`CommandFailed` or
        :class:`BadArguments` depending on the exit code (1 or 2).
    """
    endpoint = ProcessEndpoint(reactor, b"docker", [b"docker"] + arguments,
                               os.environ)
    d = connectProtocol(endpoint, _AccumulatingProtocol())
    d.addCallback(lambda protocol: protocol._result)
    return d

@attributes(["uuid", "name", "service"])
class Volume(object):
    """A data volume's identifier.

    :ivar unicode uuid: The UUID of the volume manager that owns this volume.
    :ivar unicode name: The name of the volume. Since volume names must
        match Docker container names, the characters used should be limited to
        those that Docker allows for container names.
    :ivar service: The ``VolumeService`` in whose pool this volume is stored.
    """
    def change_owner(self, new_owner_uuid):
        """
        Change which volume manager owns this volume.

        :param unicode new_owner_uuid: The UUID of the new owner.

        :return: ``Deferred`` that fires with a new :class:`Volume`
            instance once the ownership has been changed.
        """
        new_volume = Volume(uuid=new_owner_uuid, name=self.name,
                            _pool=self._pool)
        d = self._pool.change_owner(self, new_volume)

        def filesystem_changed(_):
            return new_volume
        d.addCallback(filesystem_changed)
        return d

    def get_filesystem(self):
        """Return the volume's filesystem.

        :return: The ``IFilesystem`` provider for the volume.
        """
        return self._pool.get(self)

    @property
    def _container_name(self):
        """Return the corresponding Docker container name.

        :return: Container name as ``bytes``.
        """
        return b"%s-data" % (self.name.encode("ascii"),)

    def expose_to_docker(self, mount_path):
        """
        Create a container that will expose the volume to Docker at the given
        mount path.

        Can be called multiple times. Mount paths from previous calls will
        be overridden.

        :param FilePath mount_path: The path at which to mount the volume
            within the container.

        :return: ``Deferred`` firing when the operation is done.
        """
        local_path = self.get_filesystem().get_path().path
        mount_path = mount_path.path
        d = self.remove_from_docker()
        d.addCallback(
            lambda _:
                _docker_command(reactor,
                                [b"run", b"--name", self._container_name,
                                 b"--volume=%s:%s:rw" % (local_path,
                                                         mount_path),
                                 b"busybox", b"/bin/true"]))
        return d

    def remove_from_docker(self):
        """
        Remove the Docker container created for the volume.

        If no container exists this will silently do nothing.

        :return: ``Deferred`` firing with ``None`` when the operation is
           done.
        """
        d = _docker_command(reactor, [b"rm", self._container_name])
        d.addErrback(lambda failure: failure.trap(CommandFailed))
        d.addCallback(lambda _: None)
        return d
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from flask import Flask
from flask_flatpages import FlatPages
from flask_mail import Mail
from flask_migrate import Migrate
from flask_redis import FlaskRedis
from flask_rq2 import RQ
import itsdangerous

from baseframe import Bundle, Version, assets, baseframe
import coaster.app

from ._version import __version__

# Three apps
app = Flask(__name__, instance_relative_config=True)
funnelapp = Flask(__name__, instance_relative_config=True, subdomain_matching=True)
lastuserapp = Flask(__name__, instance_relative_config=True)

mail = Mail()
pages = FlatPages()
redis_store = FlaskRedis(decode_responses=True)
rq = RQ()

# --- Assets ------------------------------------------------------------------

version = Version(__version__)
assets['funnel.css'][version] = 'css/app.css'
assets['funnel.js'][version] = 'js/scripts.js'
assets['spectrum.js'][version] = 'js/libs/spectrum.js'
assets['spectrum.css'][version] = 'css/spectrum.css'
assets['screens.css'][version] = 'css/screens.css'
assets['schedules.js'][version] = 'js/schedules.js'
assets['schedule-print.css'][version] = 'css/schedule-print.css'

# --- Import rest of the app --------------------------------------------------

from . import models, signals, loginproviders, forms, views  # NOQA  # isort:skip
from .models import db  # isort:skip

# --- Configuration------------------------------------------------------------
coaster.app.init_app(app)
coaster.app.init_app(funnelapp)
coaster.app.init_app(lastuserapp)

# These are app specific confguration files that must exist
# inside the `instance/` directory. Sample config files are
# provided as example.
coaster.app.load_config_from_file(app, 'hasgeekapp.py')
coaster.app.load_config_from_file(funnelapp, 'funnelapp.py')
coaster.app.load_config_from_file(lastuserapp, 'lastuserapp.py')

app.config['LEGACY'] = False
funnelapp.config['LEGACY'] = True
lastuserapp.config['LEGACY'] = True

app.cookie_serializer = itsdangerous.JSONWebSignatureSerializer(
    app.config.get('LASTUSER_SECRET_KEY') or app.config['SECRET_KEY']
)
funnelapp.cookie_serializer = itsdangerous.JSONWebSignatureSerializer(
    funnelapp.config.get('LASTUSER_SECRET_KEY') or funnelapp.config['SECRET_KEY']
)
lastuserapp.cookie_serializer = itsdangerous.JSONWebSignatureSerializer(
    lastuserapp.config.get('LASTUSER_SECRET_KEY') or lastuserapp.config['SECRET_KEY']
)

# Talkfunnel login support
app.login_serializer = itsdangerous.URLSafeTimedSerializer(
    app.config.get('LASTUSER_SECRET_KEY') or app.config['SECRET_KEY']
)

# TODO: Replace this with something cleaner. The `login_manager` attr expectation is
# from coaster.auth. It attempts to call `current_app.login_manager._load_user`
app.login_manager = views.helpers.LoginManager()
funnelapp.login_manager = lastuserapp.login_manager = app.login_manager

db.init_app(app)
db.init_app(funnelapp)
db.init_app(lastuserapp)
db.app = app

migrate = Migrate(app, db)

mail.init_app(app)
mail.init_app(funnelapp)
mail.init_app(lastuserapp)

app.config['FLATPAGES_MARKDOWN_EXTENSIONS'] = ['markdown.extensions.nl2br']
pages.init_app(app)

redis_store.init_app(app)

rq.init_app(app)
rq.init_app(funnelapp)
rq.init_app(lastuserapp)

baseframe.init_app(
    app,
    requires=['funnel'],
    ext_requires=['pygments', 'toastr', 'baseframe-mui', 'jquery.cookie', 'timezone'],
    theme='mui',
    asset_modules=('baseframe_private_assets',),
)
baseframe.init_app(
    funnelapp,
    requires=['funnel'],
    ext_requires=['pygments', 'toastr', 'baseframe-mui'],
    theme='mui',
    asset_modules=('baseframe_private_assets',),
)
baseframe.init_app(
    lastuserapp,
    requires=['funnel'],
    ext_requires=['pygments', 'toastr', 'baseframe-mui'],
    theme='mui',
    asset_modules=('baseframe_private_assets',),
)

loginproviders.init_app(app)
loginproviders.init_app(lastuserapp)

# Register JS and CSS assets on both apps
app.assets.register(
    'js_fullcalendar',
    Bundle(
        assets.require(
            '!jquery.js',
            'jquery.fullcalendar.js',
            'moment.js',
            'moment-timezone-data.js',
            'spectrum.js',
            'jquery.ui.sortable.touch.js',
        ),
        output='js/fullcalendar.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_fullcalendar',
    Bundle(
        assets.require('jquery.fullcalendar.css', 'spectrum.css'),
        output='css/fullcalendar.packed.css',
        filters='cssmin',
    ),
)
app.assets.register(
    'js_schedules',
    Bundle(
        assets.require('schedules.js'),
        output='js/schedules.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'js_codemirrormarkdown',
    Bundle(
        assets.require('codemirror-markdown.js'),
        output='js/codemirror-markdown.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_codemirrormarkdown',
    Bundle(
        assets.require('codemirror-markdown.css'),
        output='css/codemirror-markdown.packed.css',
        filters='cssmin',
    ),
)
app.assets.register(
    'css_screens',
    Bundle(
        assets.require('screens.css'), output='css/screens.packed.css', filters='cssmin'
    ),
)
app.assets.register(
    'js_jquerytruncate',
    Bundle(
        assets.require('!jquery.js', 'jquery.truncate8.js'),
        output='js/jquerytruncate.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'js_jqueryeasytabs',
    Bundle(
        assets.require('!jquery.js', 'jquery-easytabs.js'),
        output='js/jqueryeasytabs.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'js_leaflet',
    Bundle(
        assets.require('leaflet.js', 'leaflet-search.js'),
        output='js/leaflet.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_leaflet',
    Bundle(
        assets.require('leaflet.css', 'leaflet-search.css'),
        output='css/leaflet.packed.css',
        filters='cssmin',
    ),
)
app.assets.register(
    'js_emojionearea',
    Bundle(
        assets.require('!jquery.js', 'emojionearea-material.js'),
        output='js/emojionearea.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_emojionearea',
    Bundle(
        assets.require('emojionearea-material.css'),
        output='css/emojionearea.packed.css',
        filters='cssmin',
    ),
)
app.assets.register(
    'js_sortable',
    Bundle(
        assets.require('!jquery.js', 'jquery.ui.js', 'jquery.ui.sortable.touch.js'),
        output='js/sortable.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_schedule_print',
    Bundle(
        assets.require('schedule-print.css'),
        output='css/schedule-print.packed.css',
        filters='cssmin',
    ),
)

funnelapp.assets.register(
    'js_fullcalendar',
    Bundle(
        assets.require(
            '!jquery.js',
            'jquery.fullcalendar.js',
            'moment.js',
            'moment-timezone-data.js',
            'spectrum.js',
            'jquery.ui.sortable.touch.js',
        ),
        output='js/fullcalendar.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_fullcalendar',
    Bundle(
        assets.require('jquery.fullcalendar.css', 'spectrum.css'),
        output='css/fullcalendar.packed.css',
        filters='cssmin',
    ),
)
funnelapp.assets.register(
    'js_schedules',
    Bundle(
        assets.require('schedules.js'),
        output='js/schedules.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'js_codemirrormarkdown',
    Bundle(
        assets.require('codemirror-markdown.js'),
        output='js/codemirror-markdown.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_codemirrormarkdown',
    Bundle(
        assets.require('codemirror-markdown.css'),
        output='css/codemirror-markdown.packed.css',
        filters='cssmin',
    ),
)
funnelapp.assets.register(
    'css_screens',
    Bundle(
        assets.require('screens.css'), output='css/screens.packed.css', filters='cssmin'
    ),
)
funnelapp.assets.register(
    'js_jqueryeasytabs',
    Bundle(
        assets.require('!jquery.js', 'jquery-easytabs.js'),
        output='js/jqueryeasytabs.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'js_jquerytruncate',
    Bundle(
        assets.require('!jquery.js', 'jquery.truncate8.js'),
        output='js/jquerytruncate.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'js_leaflet',
    Bundle(
        assets.require('leaflet.js', 'leaflet-search.js'),
        output='js/leaflet.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_leaflet',
    Bundle(
        assets.require('leaflet.css', 'leaflet-search.css'),
        output='css/leaflet.packed.css',
        filters='cssmin',
    ),
)
funnelapp.assets.register(
    'js_emojionearea',
    Bundle(
        assets.require('!jquery.js', 'emojionearea-material.js'),
        output='js/emojionearea.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_emojionearea',
    Bundle(
        assets.require('emojionearea-material.css'),
        output='css/emojionearea.packed.css',
        filters='cssmin',
    ),
)
funnelapp.assets.register(
    'js_sortable',
    Bundle(
        assets.require('!jquery.js', 'jquery.ui.js', 'jquery.ui.sortable.touch.js'),
        output='js/sortable.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_schedule_print',
    Bundle(
        assets.require('schedule-print.css'),
        output='css/schedule-print.packed.css',
        filters='cssmin',
    ),
)

# FIXME: Hack for external build system generating relative /static URLs.
# Fix this by generating absolute URLs to the static subdomain during build.
funnelapp.add_url_rule(
    '/static/<path:filename>',
    endpoint='static',
    view_func=funnelapp.send_static_file,
    subdomain=None,
)
funnelapp.add_url_rule(
    '/static/<path:filename>',
    endpoint='static',
    view_func=funnelapp.send_static_file,
    subdomain='<subdomain>',
)

# Database model loading (from Funnel or extensions) is complete.
# Configure database mappers now, before the process is forked for workers.
db.configure_mappers()
<commit_msg>Use code mirror material css.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from flask import Flask
from flask_flatpages import FlatPages
from flask_mail import Mail
from flask_migrate import Migrate
from flask_redis import FlaskRedis
from flask_rq2 import RQ
import itsdangerous

from baseframe import Bundle, Version, assets, baseframe
import coaster.app

from ._version import __version__

# Three apps
app = Flask(__name__, instance_relative_config=True)
funnelapp = Flask(__name__, instance_relative_config=True, subdomain_matching=True)
lastuserapp = Flask(__name__, instance_relative_config=True)

mail = Mail()
pages = FlatPages()
redis_store = FlaskRedis(decode_responses=True)
rq = RQ()

# --- Assets ------------------------------------------------------------------

version = Version(__version__)
assets['funnel.css'][version] = 'css/app.css'
assets['funnel.js'][version] = 'js/scripts.js'
assets['spectrum.js'][version] = 'js/libs/spectrum.js'
assets['spectrum.css'][version] = 'css/spectrum.css'
assets['screens.css'][version] = 'css/screens.css'
assets['schedules.js'][version] = 'js/schedules.js'
assets['schedule-print.css'][version] = 'css/schedule-print.css'

# --- Import rest of the app --------------------------------------------------

from . import models, signals, loginproviders, forms, views  # NOQA  # isort:skip
from .models import db  # isort:skip

# --- Configuration------------------------------------------------------------
coaster.app.init_app(app)
coaster.app.init_app(funnelapp)
coaster.app.init_app(lastuserapp)

# These are app specific confguration files that must exist
# inside the `instance/` directory. Sample config files are
# provided as example.
coaster.app.load_config_from_file(app, 'hasgeekapp.py')
coaster.app.load_config_from_file(funnelapp, 'funnelapp.py')
coaster.app.load_config_from_file(lastuserapp, 'lastuserapp.py')

app.config['LEGACY'] = False
funnelapp.config['LEGACY'] = True
lastuserapp.config['LEGACY'] = True

app.cookie_serializer = itsdangerous.JSONWebSignatureSerializer(
    app.config.get('LASTUSER_SECRET_KEY') or app.config['SECRET_KEY']
)
funnelapp.cookie_serializer = itsdangerous.JSONWebSignatureSerializer(
    funnelapp.config.get('LASTUSER_SECRET_KEY') or funnelapp.config['SECRET_KEY']
)
lastuserapp.cookie_serializer = itsdangerous.JSONWebSignatureSerializer(
    lastuserapp.config.get('LASTUSER_SECRET_KEY') or lastuserapp.config['SECRET_KEY']
)

# Talkfunnel login support
app.login_serializer = itsdangerous.URLSafeTimedSerializer(
    app.config.get('LASTUSER_SECRET_KEY') or app.config['SECRET_KEY']
)

# TODO: Replace this with something cleaner. The `login_manager` attr expectation is
# from coaster.auth. It attempts to call `current_app.login_manager._load_user`
app.login_manager = views.helpers.LoginManager()
funnelapp.login_manager = lastuserapp.login_manager = app.login_manager

db.init_app(app)
db.init_app(funnelapp)
db.init_app(lastuserapp)
db.app = app

migrate = Migrate(app, db)

mail.init_app(app)
mail.init_app(funnelapp)
mail.init_app(lastuserapp)

app.config['FLATPAGES_MARKDOWN_EXTENSIONS'] = ['markdown.extensions.nl2br']
pages.init_app(app)

redis_store.init_app(app)

rq.init_app(app)
rq.init_app(funnelapp)
rq.init_app(lastuserapp)

baseframe.init_app(
    app,
    requires=['funnel'],
    ext_requires=['pygments', 'toastr', 'baseframe-mui', 'jquery.cookie', 'timezone'],
    theme='mui',
    asset_modules=('baseframe_private_assets',),
)
baseframe.init_app(
    funnelapp,
    requires=['funnel'],
    ext_requires=['pygments', 'toastr', 'baseframe-mui'],
    theme='mui',
    asset_modules=('baseframe_private_assets',),
)
baseframe.init_app(
    lastuserapp,
    requires=['funnel'],
    ext_requires=['pygments', 'toastr', 'baseframe-mui'],
    theme='mui',
    asset_modules=('baseframe_private_assets',),
)

loginproviders.init_app(app)
loginproviders.init_app(lastuserapp)

# Register JS and CSS assets on both apps
app.assets.register(
    'js_fullcalendar',
    Bundle(
        assets.require(
            '!jquery.js',
            'jquery.fullcalendar.js',
            'moment.js',
            'moment-timezone-data.js',
            'spectrum.js',
            'jquery.ui.sortable.touch.js',
        ),
        output='js/fullcalendar.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_fullcalendar',
    Bundle(
        assets.require('jquery.fullcalendar.css', 'spectrum.css'),
        output='css/fullcalendar.packed.css',
        filters='cssmin',
    ),
)
app.assets.register(
    'js_schedules',
    Bundle(
        assets.require('schedules.js'),
        output='js/schedules.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'js_codemirrormarkdown',
    Bundle(
        assets.require('codemirror-markdown.js'),
        output='js/codemirror-markdown.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_codemirrormarkdown',
    Bundle(
        assets.require('codemirror-markdown-material.css'),
        output='css/codemirror-markdown.packed.css',
        filters='cssmin',
    ),
)
app.assets.register(
    'css_screens',
    Bundle(
        assets.require('screens.css'), output='css/screens.packed.css', filters='cssmin'
    ),
)
app.assets.register(
    'js_jquerytruncate',
    Bundle(
        assets.require('!jquery.js', 'jquery.truncate8.js'),
        output='js/jquerytruncate.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'js_jqueryeasytabs',
    Bundle(
        assets.require('!jquery.js', 'jquery-easytabs.js'),
        output='js/jqueryeasytabs.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'js_leaflet',
    Bundle(
        assets.require('leaflet.js', 'leaflet-search.js'),
        output='js/leaflet.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_leaflet',
    Bundle(
        assets.require('leaflet.css', 'leaflet-search.css'),
        output='css/leaflet.packed.css',
        filters='cssmin',
    ),
)
app.assets.register(
    'js_emojionearea',
    Bundle(
        assets.require('!jquery.js', 'emojionearea-material.js'),
        output='js/emojionearea.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_emojionearea',
    Bundle(
        assets.require('emojionearea-material.css'),
        output='css/emojionearea.packed.css',
        filters='cssmin',
    ),
)
app.assets.register(
    'js_sortable',
    Bundle(
        assets.require('!jquery.js', 'jquery.ui.js', 'jquery.ui.sortable.touch.js'),
        output='js/sortable.packed.js',
        filters='uglipyjs',
    ),
)
app.assets.register(
    'css_schedule_print',
    Bundle(
        assets.require('schedule-print.css'),
        output='css/schedule-print.packed.css',
        filters='cssmin',
    ),
)

funnelapp.assets.register(
    'js_fullcalendar',
    Bundle(
        assets.require(
            '!jquery.js',
            'jquery.fullcalendar.js',
            'moment.js',
            'moment-timezone-data.js',
            'spectrum.js',
            'jquery.ui.sortable.touch.js',
        ),
        output='js/fullcalendar.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_fullcalendar',
    Bundle(
        assets.require('jquery.fullcalendar.css', 'spectrum.css'),
        output='css/fullcalendar.packed.css',
        filters='cssmin',
    ),
)
funnelapp.assets.register(
    'js_schedules',
    Bundle(
        assets.require('schedules.js'),
        output='js/schedules.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'js_codemirrormarkdown',
    Bundle(
        assets.require('codemirror-markdown.js'),
        output='js/codemirror-markdown.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_codemirrormarkdown',
    Bundle(
        assets.require('codemirror-markdown.css'),
        output='css/codemirror-markdown.packed.css',
        filters='cssmin',
    ),
)
funnelapp.assets.register(
    'css_screens',
    Bundle(
        assets.require('screens.css'), output='css/screens.packed.css', filters='cssmin'
    ),
)
funnelapp.assets.register(
    'js_jqueryeasytabs',
    Bundle(
        assets.require('!jquery.js', 'jquery-easytabs.js'),
        output='js/jqueryeasytabs.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'js_jquerytruncate',
    Bundle(
        assets.require('!jquery.js', 'jquery.truncate8.js'),
        output='js/jquerytruncate.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'js_leaflet',
    Bundle(
        assets.require('leaflet.js', 'leaflet-search.js'),
        output='js/leaflet.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_leaflet',
    Bundle(
        assets.require('leaflet.css', 'leaflet-search.css'),
        output='css/leaflet.packed.css',
        filters='cssmin',
    ),
)
funnelapp.assets.register(
    'js_emojionearea',
    Bundle(
        assets.require('!jquery.js', 'emojionearea-material.js'),
        output='js/emojionearea.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_emojionearea',
    Bundle(
        assets.require('emojionearea-material.css'),
        output='css/emojionearea.packed.css',
        filters='cssmin',
    ),
)
funnelapp.assets.register(
    'js_sortable',
    Bundle(
        assets.require('!jquery.js', 'jquery.ui.js', 'jquery.ui.sortable.touch.js'),
        output='js/sortable.packed.js',
        filters='uglipyjs',
    ),
)
funnelapp.assets.register(
    'css_schedule_print',
    Bundle(
        assets.require('schedule-print.css'),
        output='css/schedule-print.packed.css',
        filters='cssmin',
    ),
)

# FIXME: Hack for external build system generating relative /static URLs.
# Fix this by generating absolute URLs to the static subdomain during build.
funnelapp.add_url_rule(
    '/static/<path:filename>',
    endpoint='static',
    view_func=funnelapp.send_static_file,
    subdomain=None,
)
funnelapp.add_url_rule(
    '/static/<path:filename>',
    endpoint='static',
    view_func=funnelapp.send_static_file,
    subdomain='<subdomain>',
)

# Database model loading (from Funnel or extensions) is complete.
# Configure database mappers now, before the process is forked for workers.
db.configure_mappers()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function, absolute_import, division

import os
import psutil
import shutil
import subprocess
import tempfile
import time
import unittest2

class ShutdownTimeoutTests(unittest2.TestCase):
    def setUp(self):
        self.temp_dir = tempfile.mkdtemp(prefix="succubus-test")
        self.pid_file = os.path.join(self.temp_dir, 'succubus.pid')
        os.environ['PID_FILE'] = self.pid_file

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_reliable_kill(self):
        """reliable_kill must send SIGKILL after the configured timeout"""
        daemon = "./src/integrationtest/python/shutdown_timeout_daemon.py"
        subprocess.check_call([daemon, "start"])

        time.sleep(0.3)
        subprocess.check_call([daemon, "status"])
        with open(self.pid_file) as pid_file:
            daemon_pid = int(pid_file.read().strip())
        self.assertTrue(psutil.pid_exists(daemon_pid))

        start = time.time()
        subprocess.check_call([daemon, "stop"])
        stop = time.time()

        shutdown_time = stop - start
        # reliable_kill() must give the process 1.4 seconds to shut down...
        self.assertGreater(shutdown_time, 1.4)
        # ...but not much more than that.
        self.assertLess(shutdown_time, 1.7)

        # Daemon must be stopped from init script point of view.
        self.assertRaises(Exception, subprocess.check_call, [daemon, "status"])
        # And the process must really be gone.
        self.assertFalse(psutil.pid_exists(daemon_pid))

if __name__ == "__main__":
    unittest2.main()
<commit_msg>be even more patient, TravisCI is no speed daemon<commit_after>#!/usr/bin/env python
from __future__ import print_function, absolute_import, division

import os
import psutil
import shutil
import subprocess
import tempfile
import time
import unittest2

class ShutdownTimeoutTests(unittest2.TestCase):
    def setUp(self):
        self.temp_dir = tempfile.mkdtemp(prefix="succubus-test")
        self.pid_file = os.path.join(self.temp_dir, 'succubus.pid')
        os.environ['PID_FILE'] = self.pid_file

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_reliable_kill(self):
        """reliable_kill must send SIGKILL after the configured timeout"""
        daemon = "./src/integrationtest/python/shutdown_timeout_daemon.py"
        subprocess.check_call([daemon, "start"])

        time.sleep(0.3)
        subprocess.check_call([daemon, "status"])
        with open(self.pid_file) as pid_file:
            daemon_pid = int(pid_file.read().strip())
        self.assertTrue(psutil.pid_exists(daemon_pid))

        start = time.time()
        subprocess.check_call([daemon, "stop"])
        stop = time.time()

        shutdown_time = stop - start
        # reliable_kill() must give the process 1.4 seconds to shut down...
        self.assertGreater(shutdown_time, 1.4)
        # ...but not much more than that.
        self.assertLess(shutdown_time, 1.9)

        # Daemon must be stopped from init script point of view.
        self.assertRaises(Exception, subprocess.check_call, [daemon, "status"])
        # And the process must really be gone.
        self.assertFalse(psutil.pid_exists(daemon_pid))

if __name__ == "__main__":
    unittest2.main()
<|endoftext|>