<commit_before>"""
Copyright (c) 2013, katonori All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are
permitted provided that the following conditions are met:

  1. Redistributions of source code must retain the above copyright notice, this list
     of conditions and the following disclaimer.
  2. Redistributions in binary form must reproduce the above copyright notice, this
     list of conditions and the following disclaimer in the documentation and/or other
     materials provided with the distribution.
  3. Neither the name of the katonori nor the names of its contributors may be used to
     endorse or promote products derived from this software without specific prior
     written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
DAMAGE.
"""
import sys
import getopt
import shutil
import commands

argvs = sys.argv
argc = len(argvs)

fileList = []
tmpFileList = ['', '']

# parse args
try:
    opts, args = getopt.getopt(sys.argv[1:], 'L:0:1:')
except getopt.GetoptError:
    usage()
    sys.exit(1)
output = None
for o, a in opts:
    if o == '-L':
        fileList.append(a)
    elif o == '-0':
        tmpFileList[0] = a;
    elif o == '-1':
        tmpFileList[1] = a;

if len(args) != 2:
    print "ERROR: arg num is not correct"
    sys.exit(1)

shutil.copy(args[0], tmpFileList[0])
shutil.copy(args[1], tmpFileList[1])

if fileList[0]:
    print fileList[0],
print '|',
if fileList[1]:
    print fileList[1],
<commit_msg>added chmod for temporary files.<commit_after>"""
Copyright (c) 2013, katonori All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are
permitted provided that the following conditions are met:

  1. Redistributions of source code must retain the above copyright notice, this list
     of conditions and the following disclaimer.
  2. Redistributions in binary form must reproduce the above copyright notice, this
     list of conditions and the following disclaimer in the documentation and/or other
     materials provided with the distribution.
  3. Neither the name of the katonori nor the names of its contributors may be used to
     endorse or promote products derived from this software without specific prior
     written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
DAMAGE.
"""
import sys
import getopt
import shutil
import commands
import os

argvs = sys.argv
argc = len(argvs)

fileList = []
tmpFileList = ['', '']

# parse args
try:
    opts, args = getopt.getopt(sys.argv[1:], 'L:0:1:')
except getopt.GetoptError:
    usage()
    sys.exit(1)
output = None
for o, a in opts:
    if o == '-L':
        fileList.append(a)
    elif o == '-0':
        tmpFileList[0] = a;
    elif o == '-1':
        tmpFileList[1] = a;

if len(args) != 2:
    print "ERROR: arg num is not correct"
    sys.exit(1)

shutil.copy(args[0], tmpFileList[0])
os.chmod(tmpFileList[0], 0644)
shutil.copy(args[1], tmpFileList[1])
os.chmod(tmpFileList[1], 0644)

if fileList[0]:
    print fileList[0],
print '|',
if fileList[1]:
    print fileList[1],
<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import time
import json

import httplib2
from cloudify import ctx
from cloudify.exceptions import NonRecoverableError
from googleapiclient.discovery import build
from oauth2client.client import SignedJwtAssertionCredentials

def create_instance(compute, project, zone, instance_name, agent_image):
    ctx.logger.info('Create instance')
    machine_type = 'zones/{0}/machineTypes/n1-standard-1'.format(zone)

    body = {
        'name': instance_name,
        'machineType': machine_type,

        'disks': [
            {
                'boot': True,
                'autoDelete': True,
                'initializeParams': {
                    'sourceImage': agent_image
                }
            }
        ],
        'networkInterfaces': [{
            'network': 'global/networks/default',
            'accessConfigs': [
                {'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT'}
            ]
        }],
        'serviceAccounts': [{
            'email': 'default',
            'scopes': [
                'https://www.googleapis.com/auth/devstorage.read_write',
                'https://www.googleapis.com/auth/logging.write'
            ]
        }],
        'metadata': {
            'items': [{
                'key': 'bucket',
                'value': project
            }]
        }
    }

    return compute.instances().insert(
        project=project,
        zone=zone,
        body=body).execute()

def delete_instance(compute, project, zone, instance_name):
    ctx.logger.info('Delete instance')
    return compute.instances().delete(
        project=project,
        zone=zone,
        instance=instance_name).execute()

def list_instances(compute, project, zone):
    ctx.logger.info("List instances")
    return compute.instances().list(project=project,
                                    zone=zone).execute()

def wait_for_operation(compute,
                       project,
                       zone,
                       operation,
                       global_operation=False):
    ctx.logger.info('Wait for operation: {0}.'.format(operation))
    while True:
        if global_operation:
            result = compute.globalOperations().get(
                project=project,
                operation=operation).execute()
        else:
            result = compute.zoneOperations().get(
                project=project,
                zone=zone,
                operation=operation).execute()
        if result['status'] == 'DONE':
            if 'error' in result:
                raise NonRecoverableError(result['error'])
            ctx.logger.info('Done')
            return result
        else:
            time.sleep(1)

def compute(service_account, scope):
    with open(service_account) as f:
        account_data = json.load(f)
    credentials = SignedJwtAssertionCredentials(account_data['client_email'],
                                                account_data['private_key'],
                                                scope=scope)
    http = httplib2.Http()
    credentials.authorize(http)
    return build('compute', 'v1', http=http)

def set_ip(compute, config):
    instances = list_instances(compute, config['project'], config['zone'])
    item = _get_item_from_gcp_response(ctx.node.name, instances)
    ctx.instance.runtime_properties['ip'] = \
        item['networkInterfaces'][0]['networkIP']
    # only with one default network interface

def _get_item_from_gcp_response(name, items):
    for item in items.get('items'):
        if item.get('name') == name:
            return item
    return None

def create_network(compute, project, network):
    ctx.logger.info('Create network')
    body = {
        "description": "Cloudify generated network",
        "name": network
    }
    return compute.networks().insert(project=project,
                                     body=body).execute()

def delete_network(compute, project, network):
    ctx.logger.info('Delete network')
    return compute.networks().delete(project=project,
                                     network=network).execute()

def list_networks(compute, project):
    ctx.logger.info('List networks')
    return compute.networks().list(project=project).execute()

def create_firewall_rule(compute, project, network, firewall):
    ctx.logger.info('Create firewall rule')
    firewall['network'] = \
        'global/networks/{0}'.format(network)
    firewall['name'] = '{0}-{1}'.format(network, firewall['name'])
    # should be changed in node runtime properties
    return compute.firewalls().insert(project=project,
                                      body=firewall).execute()

def delete_firewall_rule(compute, project, firewall_name):
    ctx.logger.info('Delete firewall rule')
    return compute.firewalls().delete(
        project=project,
        firewall=firewall_name).execute()

def list_firewall_rules(compute, project):
    ctx.logger.info('List firewall rules in project')
    return compute.firewalls().list(project=project).execute()
<commit_msg>Make create instance function more generic<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import time
import json

import httplib2
from cloudify import ctx
from cloudify.exceptions import NonRecoverableError
from googleapiclient.discovery import build
from oauth2client.client import SignedJwtAssertionCredentials

def create_instance(compute,
                    project,
                    zone,
                    instance_name,
                    agent_image,
                    machine_type='n1-standard-1',
                    network='default'):
    ctx.logger.info('Create instance')
    machine_type = 'zones/{0}/machineTypes/{1}'.format(zone, machine_type)

    body = {
        'name': instance_name,
        'machineType': machine_type,

        'disks': [
            {
                'boot': True,
                'autoDelete': True,
                'initializeParams': {
                    'sourceImage': agent_image
                }
            }
        ],
        'networkInterfaces': [{
            'network': 'global/networks/{0}'.format(network),
            'accessConfigs': [
                {'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT'}
            ]
        }],
        'serviceAccounts': [{
            'email': 'default',
            'scopes': [
                'https://www.googleapis.com/auth/devstorage.read_write',
                'https://www.googleapis.com/auth/logging.write'
            ]
        }],
        'metadata': {
            'items': [{
                'key': 'bucket',
                'value': project
            }]
        }
    }

    return compute.instances().insert(
        project=project,
        zone=zone,
        body=body).execute()

def delete_instance(compute, project, zone, instance_name):
    ctx.logger.info('Delete instance')
    return compute.instances().delete(
        project=project,
        zone=zone,
        instance=instance_name).execute()

def list_instances(compute, project, zone):
    ctx.logger.info("List instances")
    return compute.instances().list(project=project,
                                    zone=zone).execute()

def wait_for_operation(compute,
                       project,
                       zone,
                       operation,
                       global_operation=False):
    ctx.logger.info('Wait for operation: {0}.'.format(operation))
    while True:
        if global_operation:
            result = compute.globalOperations().get(
                project=project,
                operation=operation).execute()
        else:
            result = compute.zoneOperations().get(
                project=project,
                zone=zone,
                operation=operation).execute()
        if result['status'] == 'DONE':
            if 'error' in result:
                raise NonRecoverableError(result['error'])
            ctx.logger.info('Done')
            return result
        else:
            time.sleep(1)

def compute(service_account, scope):
    with open(service_account) as f:
        account_data = json.load(f)
    credentials = SignedJwtAssertionCredentials(account_data['client_email'],
                                                account_data['private_key'],
                                                scope=scope)
    http = httplib2.Http()
    credentials.authorize(http)
    return build('compute', 'v1', http=http)

def set_ip(compute, config):
    instances = list_instances(compute, config['project'], config['zone'])
    item = _get_item_from_gcp_response(ctx.node.name, instances)
    ctx.instance.runtime_properties['ip'] = \
        item['networkInterfaces'][0]['networkIP']
    # only with one default network interface

def _get_item_from_gcp_response(name, items):
    for item in items.get('items'):
        if item.get('name') == name:
            return item
    return None

def create_network(compute, project, network):
    ctx.logger.info('Create network')
    body = {
        "description": "Cloudify generated network",
        "name": network
    }
    return compute.networks().insert(project=project,
                                     body=body).execute()

def delete_network(compute, project, network):
    ctx.logger.info('Delete network')
    return compute.networks().delete(project=project,
                                     network=network).execute()

def list_networks(compute, project):
    ctx.logger.info('List networks')
    return compute.networks().list(project=project).execute()

def create_firewall_rule(compute, project, network, firewall):
    ctx.logger.info('Create firewall rule')
    firewall['network'] = \
        'global/networks/{0}'.format(network)
    firewall['name'] = '{0}-{1}'.format(network, firewall['name'])
    # should be changed in node runtime properties
    return compute.firewalls().insert(project=project,
                                      body=firewall).execute()

def delete_firewall_rule(compute, project, firewall_name):
    ctx.logger.info('Delete firewall rule')
    return compute.firewalls().delete(
        project=project,
        firewall=firewall_name).execute()

def list_firewall_rules(compute, project):
    ctx.logger.info('List firewall rules in project')
    return compute.firewalls().list(project=project).execute()
<|endoftext|>


<commit_before>###
# Copyright (c) 2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

import re
import sgmllib
import htmlentitydefs

import supybot.conf as conf
import supybot.utils as utils
from supybot.commands import *
import supybot.plugins as plugins
import supybot.ircutils as ircutils
import supybot.callbacks as callbacks

class Title(sgmllib.SGMLParser):
    entitydefs = htmlentitydefs.entitydefs.copy()
    entitydefs['nbsp'] = ' '
    def __init__(self):
        self.inTitle = False
        self.title = ''
        sgmllib.SGMLParser.__init__(self)

    def start_title(self, attrs):
        self.inTitle = True

    def end_title(self):
        self.inTitle = False

    def unknown_entityref(self, name):
        if self.inTitle:
            self.title += ' '

    def unknown_charref(self, name):
        if self.inTitle:
            self.title += ' '

    def handle_data(self, data):
        if self.inTitle:
            self.title += data

class Web(callbacks.PluginRegexp):
    """Add the help for "@help Web" here."""
    threaded = True
    regexps = ['titleSnarfer']
    def callCommand(self, command, irc, msg, *args, **kwargs):
        try:
            super(Web, self).callCommand(command, irc, msg, *args, **kwargs)
        except utils.web.Error, e:
            irc.reply(str(e))

    def titleSnarfer(self, irc, msg, match):
        r"https?://[^\])>\s]+"
        channel = msg.args[0]
        if not irc.isChannel(channel):
            return
        if callbacks.addressed(irc.nick, msg):
            return
        if self.registryValue('titleSnarfer', channel):
            url = match.group(0)
            r = self.registryValue('nonSnarfingRegexp', channel)
            if r and r.search(url):
                self.log.debug('Not titleSnarfing %q.', url)
                return
            try:
                size = conf.supybot.protocols.http.peekSize()
                text = utils.web.getUrl(url, size=size)
            except utils.web.Error, e:
                self.log.info('Couldn\'t snarf title of %u: %s.', url, e)
                return
            parser = Title()
            try:
                parser.feed(text)
            except HTMLParser.HTMLParseError:
                self.log.debug('Encountered a problem parsing %u.  Title may '
                               'already be set, though', url)
            if parser.title:
                domain = utils.web.getDomain(url)
                title = utils.web.htmlToText(parser.title.strip())
                s = format('Title: %s (at %s)', title, domain)
                irc.reply(s, prefixNick=False)
    titleSnarfer = urlSnarfer(titleSnarfer)

    def headers(self, irc, msg, args, url):
        """<url>

        Returns the HTTP headers of <url>.  Only HTTP urls are valid, of
        course.
        """
        fd = utils.web.getUrlFd(url)
        try:
            s = ', '.join([format('%s: %s', k, v)
                           for (k, v) in fd.headers.items()])
            irc.reply(s)
        finally:
            fd.close()
    headers = wrap(headers, ['httpUrl'])

    _doctypeRe = re.compile(r'(<!DOCTYPE[^>]+>)', re.M)
    def doctype(self, irc, msg, args, url):
        """<url>

        Returns the DOCTYPE string of <url>.  Only HTTP urls are valid, of
        course.
        """
        size = conf.supybot.protocols.http.peekSize()
        s = utils.web.getUrl(url, size=size)
        m = self._doctypeRe.search(s)
        if m:
            s = utils.str.normalizeWhitespace(m.group(0))
            irc.reply(s)
        else:
            irc.reply('That URL has no specified doctype.')
    doctype = wrap(doctype, ['httpUrl'])

    def size(self, irc, msg, args, url):
        """<url>

        Returns the Content-Length header of <url>.  Only HTTP urls are valid,
        of course.
        """
        fd = utils.web.getUrlFd(url)
        try:
            try:
                size = fd.headers['Content-Length']
                irc.reply(format('%u is %i bytes long.', url, size))
            except KeyError:
                size = conf.supybot.protocols.http.peekSize()
                s = fd.read(size)
                if len(s) != size:
                    irc.reply(format('%u is %i bytes long.', url, len(s)))
                else:
                    irc.reply(format('The server didn\'t tell me how long %u '
                                     'is but it\'s longer than %i bytes.',
                                     url, size))
        finally:
            fd.close()
    size = wrap(size, ['httpUrl'])

    def title(self, irc, msg, args, url):
        """<url>

        Returns the HTML <title>...</title> of a URL.
        """
        size = conf.supybot.protocols.http.peekSize()
        text = utils.web.getUrl(url, size=size)
        parser = Title()
        try:
            parser.feed(text)
        except HTMLParser.HTMLParseError:
            self.log.debug('Encountered a problem parsing %u.  Title may '
                           'already be set, though', url)
        if parser.title:
            irc.reply(utils.web.htmlToText(parser.title.strip()))
        else:
            irc.reply(format('That URL appears to have no HTML title '
                             'within the first %i bytes.', size))
    title = wrap(title, ['httpUrl'])

    _netcraftre = re.compile(r'td align="left">\s+<a[^>]+>(.*?)<a href',
                             re.S | re.I)
    def netcraft(self, irc, msg, args, hostname):
        """<hostname|ip>

        Returns Netcraft.com's determination of what operating system and
        webserver is running on the host given.
        """
        url = 'http://uptime.netcraft.com/up/graph/?host=' + hostname
        html = utils.web.getUrl(url)
        m = self._netcraftre.search(html)
        if m:
            html = m.group(1)
            s = utils.web.htmlToText(html, tagReplace='').strip()
            s = s.rstrip('-').strip()
            irc.reply(s) # Snip off "the site"
        elif 'We could not get any results' in html:
            irc.reply('No results found for %s.' % hostname)
        else:
            irc.error('The format of page the was odd.')
    netcraft = wrap(netcraft, ['text'])

    def urlquote(self, irc, msg, args, text):
        """<text>

        Returns the URL quoted form of the text.
        """
        irc.reply(utils.web.urlquote(text))
    urlquote = wrap(urlquote, ['text'])

    def urlunquote(self, irc, msg, args, text):
        """<text>

        Returns the text un-URL quoted.
        """
        s = utils.web.urlunquote(text)
        irc.reply(s)
    urlunquote = wrap(urlunquote, ['text'])

    def fetch(self, irc, msg, args, url):
        """<url>

        Returns the contents of <url>, or as much as is configured in
        supybot.plugins.Web.fetch.maximum.  If that configuration variable is
        set to 0, this command will be effectively disabled.
        """
        max = self.registryValue('fetch.maximum')
        if not max:
            irc.error('This command is disabled '
                      '(supybot.plugins.Web.fetch.maximum is set to 0).',
                      Raise=True)
        fd = utils.web.getUrlFd(url)
        irc.reply(fd.read(max))
    fetch = wrap(fetch, ['url'])

Class = Web

# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79:
<commit_msg>plugins/Web: Update the exception handling for the change in parsers.<commit_after>###
# Copyright (c) 2005, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

import re
import sgmllib
import htmlentitydefs

import supybot.conf as conf
import supybot.utils as utils
from supybot.commands import *
import supybot.plugins as plugins
import supybot.ircutils as ircutils
import supybot.callbacks as callbacks

class Title(sgmllib.SGMLParser):
    entitydefs = htmlentitydefs.entitydefs.copy()
    entitydefs['nbsp'] = ' '
    def __init__(self):
        self.inTitle = False
        self.title = ''
        sgmllib.SGMLParser.__init__(self)

    def start_title(self, attrs):
        self.inTitle = True

    def end_title(self):
        self.inTitle = False

    def unknown_entityref(self, name):
        if self.inTitle:
            self.title += ' '

    def unknown_charref(self, name):
        if self.inTitle:
            self.title += ' '

    def handle_data(self, data):
        if self.inTitle:
            self.title += data

class Web(callbacks.PluginRegexp):
    """Add the help for "@help Web" here."""
    threaded = True
    regexps = ['titleSnarfer']
    def callCommand(self, command, irc, msg, *args, **kwargs):
        try:
            super(Web, self).callCommand(command, irc, msg, *args, **kwargs)
        except utils.web.Error, e:
            irc.reply(str(e))

    def titleSnarfer(self, irc, msg, match):
        r"https?://[^\])>\s]+"
        channel = msg.args[0]
        if not irc.isChannel(channel):
            return
        if callbacks.addressed(irc.nick, msg):
            return
        if self.registryValue('titleSnarfer', channel):
            url = match.group(0)
            r = self.registryValue('nonSnarfingRegexp', channel)
            if r and r.search(url):
                self.log.debug('Not titleSnarfing %q.', url)
                return
            try:
                size = conf.supybot.protocols.http.peekSize()
                text = utils.web.getUrl(url, size=size)
            except utils.web.Error, e:
                self.log.info('Couldn\'t snarf title of %u: %s.', url, e)
                return
            parser = Title()
            try:
                parser.feed(text)
            except sgmllib.SGMLParseError:
                self.log.debug('Encountered a problem parsing %u.  Title may '
                               'already be set, though', url)
            if parser.title:
                domain = utils.web.getDomain(url)
                title = utils.web.htmlToText(parser.title.strip())
                s = format('Title: %s (at %s)', title, domain)
                irc.reply(s, prefixNick=False)
    titleSnarfer = urlSnarfer(titleSnarfer)

    def headers(self, irc, msg, args, url):
        """<url>

        Returns the HTTP headers of <url>.  Only HTTP urls are valid, of
        course.
        """
        fd = utils.web.getUrlFd(url)
        try:
            s = ', '.join([format('%s: %s', k, v)
                           for (k, v) in fd.headers.items()])
            irc.reply(s)
        finally:
            fd.close()
    headers = wrap(headers, ['httpUrl'])

    _doctypeRe = re.compile(r'(<!DOCTYPE[^>]+>)', re.M)
    def doctype(self, irc, msg, args, url):
        """<url>

        Returns the DOCTYPE string of <url>.  Only HTTP urls are valid, of
        course.
        """
        size = conf.supybot.protocols.http.peekSize()
        s = utils.web.getUrl(url, size=size)
        m = self._doctypeRe.search(s)
        if m:
            s = utils.str.normalizeWhitespace(m.group(0))
            irc.reply(s)
        else:
            irc.reply('That URL has no specified doctype.')
    doctype = wrap(doctype, ['httpUrl'])

    def size(self, irc, msg, args, url):
        """<url>

        Returns the Content-Length header of <url>.  Only HTTP urls are valid,
        of course.
        """
        fd = utils.web.getUrlFd(url)
        try:
            try:
                size = fd.headers['Content-Length']
                irc.reply(format('%u is %i bytes long.', url, size))
            except KeyError:
                size = conf.supybot.protocols.http.peekSize()
                s = fd.read(size)
                if len(s) != size:
                    irc.reply(format('%u is %i bytes long.', url, len(s)))
                else:
                    irc.reply(format('The server didn\'t tell me how long %u '
                                     'is but it\'s longer than %i bytes.',
                                     url, size))
        finally:
            fd.close()
    size = wrap(size, ['httpUrl'])

    def title(self, irc, msg, args, url):
        """<url>

        Returns the HTML <title>...</title> of a URL.
        """
        size = conf.supybot.protocols.http.peekSize()
        text = utils.web.getUrl(url, size=size)
        parser = Title()
        try:
            parser.feed(text)
        except sgmllib.SGMLParseError:
            self.log.debug('Encountered a problem parsing %u.  Title may '
                           'already be set, though', url)
        if parser.title:
            irc.reply(utils.web.htmlToText(parser.title.strip()))
        else:
            irc.reply(format('That URL appears to have no HTML title '
                             'within the first %i bytes.', size))
    title = wrap(title, ['httpUrl'])

    _netcraftre = re.compile(r'td align="left">\s+<a[^>]+>(.*?)<a href',
                             re.S | re.I)
    def netcraft(self, irc, msg, args, hostname):
        """<hostname|ip>

        Returns Netcraft.com's determination of what operating system and
        webserver is running on the host given.
        """
        url = 'http://uptime.netcraft.com/up/graph/?host=' + hostname
        html = utils.web.getUrl(url)
        m = self._netcraftre.search(html)
        if m:
            html = m.group(1)
            s = utils.web.htmlToText(html, tagReplace='').strip()
            s = s.rstrip('-').strip()
            irc.reply(s) # Snip off "the site"
        elif 'We could not get any results' in html:
            irc.reply('No results found for %s.' % hostname)
        else:
            irc.error('The format of page the was odd.')
    netcraft = wrap(netcraft, ['text'])

    def urlquote(self, irc, msg, args, text):
        """<text>

        Returns the URL quoted form of the text.
        """
        irc.reply(utils.web.urlquote(text))
    urlquote = wrap(urlquote, ['text'])

    def urlunquote(self, irc, msg, args, text):
        """<text>

        Returns the text un-URL quoted.
        """
        s = utils.web.urlunquote(text)
        irc.reply(s)
    urlunquote = wrap(urlunquote, ['text'])

    def fetch(self, irc, msg, args, url):
        """<url>

        Returns the contents of <url>, or as much as is configured in
        supybot.plugins.Web.fetch.maximum.  If that configuration variable is
        set to 0, this command will be effectively disabled.
        """
        max = self.registryValue('fetch.maximum')
        if not max:
            irc.error('This command is disabled '
                      '(supybot.plugins.Web.fetch.maximum is set to 0).',
                      Raise=True)
        fd = utils.web.getUrlFd(url)
        irc.reply(fd.read(max))
    fetch = wrap(fetch, ['url'])

Class = Web

# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79:
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
##
## This file is part of CDS Indico.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
``fossilize`` allows us to "serialize" complex python objects into dictionaries
and lists. Such operation is very useful for generating JSON data structures
from business objects. It works as a wrapper around ``zope.interface``.

Some of the features are:
 * Different "fossil" types for the same source class;
 * Built-in inheritance support;
"""

import inspect
import re
import zope.interface
from types import NoneType

def fossilizes(*classList):
    """
    Simple wrapper around 'implements'
    """
    zope.interface.declarations._implements("fossilizes",
                                            classList,
                                            zope.interface.classImplements)

def addFossil(klazz, fossils):
    """
    Declares fossils for a class

    :param klazz: a class object
    :type klass: class object
    :param fossils: a fossil class (or a list of fossil classes)
    """
    if not type(fossils) is list:
        fossils = [fossils]

    for fossil in fossils:
        zope.interface.classImplements(klazz, fossil)

def clearCache():
    """
    Shortcut for Fossilizable.clearCache()
    """
    Fossilizable.clearCache()

class NonFossilizableException(Exception):
    """
    Object is not fossilizable (doesn't implement Fossilizable)
    """

class WrongFossilTypeException(Exception):
    """
    Fossil type doesn't apply to target object
    """

class InvalidFossilException(Exception):
    """
    The fossil name doesn't follow the convention I(\w+)Fossil
    or has an invalid method name and did not declare a .name tag for it
    """

class IFossil(zope.interface.Interface):
    """
    Fossil base interface. All fossil classes should derive from this one.
    """

class Fossilizable(object):
    """
    Base class for all the objects that can be fossilized
    """

    __fossilNameRE = re.compile('^I(\w+)Fossil$')
    __methodNameRE = re.compile('^get(\w+)|(has\w+)|(is\w+)$')
    __methodNameCache = {}
    __fossilNameCache = {}
    __fossilAttrsCache = {} # Attribute Cache for Fossils with
                            # fields that are repeated

    @classmethod
    def __extractName(cls, name):
        """
        'De-camelcase' the name
        """

        if name in cls.__methodNameCache:
            return cls.__methodNameCache[name]
        else:
            nmatch = cls.__methodNameRE.match(name)

            if not nmatch:
                raise InvalidFossilException("method name '%s' is not valid! "
                                             "has to start by 'get', 'has', 'is' "
                                             "or use 'name' tag" % name)
            else:
                group = nmatch.group(1) or nmatch.group(2) or nmatch.group(3)
                extractedName = group[0:1].lower() + group[1:]
                cls.__methodNameCache[name] = extractedName
                return extractedName

    @classmethod
    def __extractFossilName(cls, name):
        """
        Extracts the fossil name from a I(.*)Fossil
        class name.
        IMyObjectBasicFossil -> myObjectBasic
        """

        if name in cls.__fossilNameCache:
            fossilName = cls.__fossilNameCache[name]
        else:
            fossilNameMatch = Fossilizable.__fossilNameRE.match(name)
            if fossilNameMatch is None:
                raise InvalidFossilException("Invalid fossil name: %s."
                                             " A fossil name should follow the"
                                             " pattern: I(\w+)Fossil." % name)
            else:
                fossilName = fossilNameMatch.group(1)[0].lower() + \
                fossilNameMatch.group(1)[1:]

                cls.__fossilNameCache[name] = fossilName
        return fossilName

    def __obtainInterface(self, interfaceArg):
        """
        Obtains the appropriate interface for this object.

        :param interfaceArg: the target fossile type
        :type interfaceArg: IFossil, NoneType, or dict

        * If IFossil, we will use it.
        * If None, we will take the default fossil
        (the first one of this class's 'fossilizes' list)
        * If a dict, we will use the objects class, class name, or full class name
        as key.

        Also verifies that the interface obtained through these 3 methods is
        effectively provided by the object.
        """

        if interfaceArg is None:
            # we try to take the 1st interface declared with fossilizes
            implementedInterfaces = list(
                i for i in zope.interface.implementedBy(self.__class__) \
                if i.extends(IFossil) )

            if not implementedInterfaces:
                raise NonFossilizableException(
                    "Object %s of class %s cannot be fossilized,"
                    "no fossils were declared for it" %
                    (str(self), self.__class__.__name__))
            else:
                interface = implementedInterfaces[0]

        elif type(interfaceArg) is dict:

            className = self.__class__.__module__ + '.' + \
                        self.__class__.__name__

            # interfaceArg is a dictionary of class:Fossil pairs
            if className in interfaceArg:
                interface = interfaceArg[className]
            else:
                raise NonFossilizableException(
                    "Object %s of class %s cannot be fossilized; "
                    "its class was not a key in the provided fossils dictionary" %
                    (str(self), self.__class__.__name__))
        else:
            interface = interfaceArg

        if not interface.providedBy(self):

            raise WrongFossilTypeException("Interface '%s' not provided"
                                           " by '%s'" %
                                           (interface.__name__,
                                            self.__class__.__name__))

        return interface

    @classmethod
    def clearCache(cls):
        """
        Clears the fossil attribute cache
        """
        cls.__fossilAttrsCache = {}

    @classmethod
    def fossilizeIterable(cls, target, interface, useAttrCache = False, **kwargs):
        """
        Fossilizes an object, be it a 'direct' fossilizable
        object, or an iterable (dict, list, set);
        """

        if isinstance(target, Fossilizable):
            return target.fossilize(interface, useAttrCache, **kwargs)
        else:
            ttype = type(target)
            if ttype in [int, str, float, NoneType]:
                return target
            elif ttype is dict:
                container = {}
                for key, value in target.iteritems():
                    container[key] = fossilize(value, interface, useAttrCache,
                                               **kwargs)
                return container
            elif hasattr(target, '__iter__'):
                # we turn sets and tuples into lists since JSON does not
                # have sets / tuples
                return list(fossilize(elem,
                                      interface,
                                      useAttrCache,
                                      **kwargs) for elem in target)
            # If the object is a wrapper for an iterable, by default we fossilize
            # the iterable the object is wrapping. This behaviour is included in
            # order to let objects like PersistentList to be fossilized
            elif hasattr(target, '__dict__') and len(target.__dict__) == 1 and \
                     hasattr(target.__dict__.values()[0], '__iter__'):
                return list(fossilize(elem,
                                      interface,
                                      useAttrCache,
                                      **kwargs) for elem in target.__dict__.values()[0])
            else:
                raise NonFossilizableException("Type %s is not fossilizable!" %
                                               ttype)

            return fossilize(target, interface, useAttrCache)

    def fossilize(self, interfaceArg = None, useAttrCache = False, **kwargs):
        """
        Fossilizes the object, using the fossil provided by `interface`.

        :param interfaceArg: the target fossile type
        :type interfaceArg: IFossil, NoneType, or dict
        :param useAttrCache: use caching of attributes if same fields are
            repeated for a fossil
        :type useAttrCache: boolean
        """

        interface = self.__obtainInterface(interfaceArg)

        name = interface.getName()
        fossilName = self.__extractFossilName(name)

        result = {}

        for method in interface:

            tags = interface[method].getTaggedValueTags()

            # In some cases it is better to use the attribute cache to
            # speed up the fossilization
            cacheUsed = False
            if useAttrCache:
                try:
                    methodResult = self.__fossilAttrsCache[self._p_oid][method]
                    cacheUsed = True
                except KeyError:
                    pass
            if not cacheUsed:
	            # Please use 'produce' as little as possible;
    	        # there is almost always a more elegant and modular solution!
                if 'produce' in tags:
                    methodResult = interface[method].getTaggedValue('produce')(self)
                else:
                    methodResult = getattr(self, method)()

                if hasattr(self, "_p_oid"):
                    try:
                        self.__fossilAttrsCache[self._p_oid]
                    except KeyError:
                        self.__fossilAttrsCache[self._p_oid] = {}
                    self.__fossilAttrsCache[self._p_oid][method] = methodResult

            # Result conversion
            if 'result' in tags:
                targetInterface = interface[method].getTaggedValue('result')
                #targetInterface = globals()[targetInterfaceName]

                methodResult = Fossilizable.fossilizeIterable(
                    methodResult, targetInterface, **kwargs)

            # Conversion function
            if 'convert' in tags:
                convertFunction = interface[method].getTaggedValue('convert')
                converterArgNames = inspect.getargspec(convertFunction)[0]
                converterArgs = dict((name, kwargs[name])
                                     for name in converterArgNames
                                     if name in kwargs)
                methodResult = convertFunction(methodResult, **converterArgs)

            # Re-name the attribute produced by the method
            if 'name' in tags:
                attrName = interface[method].getTaggedValue('name')
            else:
                attrName = self.__extractName(method)

            # In case the name contains dots, each of the 'domains' but the
            # last one are translated into nested dictionnaries. For example,
            # if we want to re-name an attribute into "foo.bar.tofu", the
            # corresponding fossilized attribute will be of the form:
            # {"foo":{"bar":{"tofu": res,...},...},...}
            # instead of:
            # {"foo.bar.tofu": res, ...}

            current = result
            attrList = attrName.split('.')

            while len(attrList) > 1:
                attr = attrList.pop(0)
                try:
                    current = current[attr]
                except KeyError:
                    current[attr] = {}
                    current = current[attr]

            # For the last attribute level
            current[attrList[0]] = methodResult

        if "_type" in result or "_fossil" in result:
            raise InvalidFossilException('"_type" or "_fossil"'
                                         ' cannot be a fossil attribute  name')
        else:
            result["_type"] = self.__class__.__name__
            if fossilName: #we check that it's not an empty string
                result["_fossil"] = fossilName
            else:
                result["_fossil"] = ""

        return result

def fossilize(target, interfaceArg = None, useAttrCache = False, **kwargs):
    """
    Method that allows the "fossilization" process to
    be called on data structures (lists, dictionaries
    and sets) as well as normal `Fossilizable` objects.

    :param target: target object to be fossilized
    :type target: Fossilizable
    :param interfaceArg: target fossil type
    :type interfaceArg: IFossil, NoneType, or dict
    :param useAttrCache: use the attribute caching
    :type useAttrCache: boolean
    """
    return Fossilizable.fossilizeIterable(target, interfaceArg, useAttrCache,
                                           **kwargs)
<commit_msg>[FIX] Fossilize: Added `bool` type<commit_after># -*- coding: utf-8 -*-
##
##
## This file is part of CDS Indico.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
``fossilize`` allows us to "serialize" complex python objects into dictionaries
and lists. Such operation is very useful for generating JSON data structures
from business objects. It works as a wrapper around ``zope.interface``.

Some of the features are:
 * Different "fossil" types for the same source class;
 * Built-in inheritance support;
"""

import inspect
import re
import zope.interface
from types import NoneType

def fossilizes(*classList):
    """
    Simple wrapper around 'implements'
    """
    zope.interface.declarations._implements("fossilizes",
                                            classList,
                                            zope.interface.classImplements)

def addFossil(klazz, fossils):
    """
    Declares fossils for a class

    :param klazz: a class object
    :type klass: class object
    :param fossils: a fossil class (or a list of fossil classes)
    """
    if not type(fossils) is list:
        fossils = [fossils]

    for fossil in fossils:
        zope.interface.classImplements(klazz, fossil)

def clearCache():
    """
    Shortcut for Fossilizable.clearCache()
    """
    Fossilizable.clearCache()

class NonFossilizableException(Exception):
    """
    Object is not fossilizable (doesn't implement Fossilizable)
    """

class WrongFossilTypeException(Exception):
    """
    Fossil type doesn't apply to target object
    """

class InvalidFossilException(Exception):
    """
    The fossil name doesn't follow the convention I(\w+)Fossil
    or has an invalid method name and did not declare a .name tag for it
    """

class IFossil(zope.interface.Interface):
    """
    Fossil base interface. All fossil classes should derive from this one.
    """

class Fossilizable(object):
    """
    Base class for all the objects that can be fossilized
    """

    __fossilNameRE = re.compile('^I(\w+)Fossil$')
    __methodNameRE = re.compile('^get(\w+)|(has\w+)|(is\w+)$')
    __methodNameCache = {}
    __fossilNameCache = {}
    __fossilAttrsCache = {} # Attribute Cache for Fossils with
                            # fields that are repeated

    @classmethod
    def __extractName(cls, name):
        """
        'De-camelcase' the name
        """

        if name in cls.__methodNameCache:
            return cls.__methodNameCache[name]
        else:
            nmatch = cls.__methodNameRE.match(name)

            if not nmatch:
                raise InvalidFossilException("method name '%s' is not valid! "
                                             "has to start by 'get', 'has', 'is' "
                                             "or use 'name' tag" % name)
            else:
                group = nmatch.group(1) or nmatch.group(2) or nmatch.group(3)
                extractedName = group[0:1].lower() + group[1:]
                cls.__methodNameCache[name] = extractedName
                return extractedName

    @classmethod
    def __extractFossilName(cls, name):
        """
        Extracts the fossil name from a I(.*)Fossil
        class name.
        IMyObjectBasicFossil -> myObjectBasic
        """

        if name in cls.__fossilNameCache:
            fossilName = cls.__fossilNameCache[name]
        else:
            fossilNameMatch = Fossilizable.__fossilNameRE.match(name)
            if fossilNameMatch is None:
                raise InvalidFossilException("Invalid fossil name: %s."
                                             " A fossil name should follow the"
                                             " pattern: I(\w+)Fossil." % name)
            else:
                fossilName = fossilNameMatch.group(1)[0].lower() + \
                fossilNameMatch.group(1)[1:]

                cls.__fossilNameCache[name] = fossilName
        return fossilName

    def __obtainInterface(self, interfaceArg):
        """
        Obtains the appropriate interface for this object.

        :param interfaceArg: the target fossile type
        :type interfaceArg: IFossil, NoneType, or dict

        * If IFossil, we will use it.
        * If None, we will take the default fossil
        (the first one of this class's 'fossilizes' list)
        * If a dict, we will use the objects class, class name, or full class name
        as key.

        Also verifies that the interface obtained through these 3 methods is
        effectively provided by the object.
        """

        if interfaceArg is None:
            # we try to take the 1st interface declared with fossilizes
            implementedInterfaces = list(
                i for i in zope.interface.implementedBy(self.__class__) \
                if i.extends(IFossil) )

            if not implementedInterfaces:
                raise NonFossilizableException(
                    "Object %s of class %s cannot be fossilized,"
                    "no fossils were declared for it" %
                    (str(self), self.__class__.__name__))
            else:
                interface = implementedInterfaces[0]

        elif type(interfaceArg) is dict:

            className = self.__class__.__module__ + '.' + \
                        self.__class__.__name__

            # interfaceArg is a dictionary of class:Fossil pairs
            if className in interfaceArg:
                interface = interfaceArg[className]
            else:
                raise NonFossilizableException(
                    "Object %s of class %s cannot be fossilized; "
                    "its class was not a key in the provided fossils dictionary" %
                    (str(self), self.__class__.__name__))
        else:
            interface = interfaceArg

        if not interface.providedBy(self):

            raise WrongFossilTypeException("Interface '%s' not provided"
                                           " by '%s'" %
                                           (interface.__name__,
                                            self.__class__.__name__))

        return interface

    @classmethod
    def clearCache(cls):
        """
        Clears the fossil attribute cache
        """
        cls.__fossilAttrsCache = {}

    @classmethod
    def fossilizeIterable(cls, target, interface, useAttrCache = False, **kwargs):
        """
        Fossilizes an object, be it a 'direct' fossilizable
        object, or an iterable (dict, list, set);
        """

        if isinstance(target, Fossilizable):
            return target.fossilize(interface, useAttrCache, **kwargs)
        else:
            ttype = type(target)
            if ttype in [int, str, float, bool, NoneType]:
                return target
            elif ttype is dict:
                container = {}
                for key, value in target.iteritems():
                    container[key] = fossilize(value, interface, useAttrCache,
                                               **kwargs)
                return container
            elif hasattr(target, '__iter__'):
                # we turn sets and tuples into lists since JSON does not
                # have sets / tuples
                return list(fossilize(elem,
                                      interface,
                                      useAttrCache,
                                      **kwargs) for elem in target)
            # If the object is a wrapper for an iterable, by default we fossilize
            # the iterable the object is wrapping. This behaviour is included in
            # order to let objects like PersistentList to be fossilized
            elif hasattr(target, '__dict__') and len(target.__dict__) == 1 and \
                     hasattr(target.__dict__.values()[0], '__iter__'):
                return list(fossilize(elem,
                                      interface,
                                      useAttrCache,
                                      **kwargs) for elem in target.__dict__.values()[0])
            else:
                raise NonFossilizableException("Type %s is not fossilizable!" %
                                               ttype)

            return fossilize(target, interface, useAttrCache)

    def fossilize(self, interfaceArg = None, useAttrCache = False, **kwargs):
        """
        Fossilizes the object, using the fossil provided by `interface`.

        :param interfaceArg: the target fossile type
        :type interfaceArg: IFossil, NoneType, or dict
        :param useAttrCache: use caching of attributes if same fields are
            repeated for a fossil
        :type useAttrCache: boolean
        """

        interface = self.__obtainInterface(interfaceArg)

        name = interface.getName()
        fossilName = self.__extractFossilName(name)

        result = {}

        for method in interface:

            tags = interface[method].getTaggedValueTags()

            # In some cases it is better to use the attribute cache to
            # speed up the fossilization
            cacheUsed = False
            if useAttrCache:
                try:
                    methodResult = self.__fossilAttrsCache[self._p_oid][method]
                    cacheUsed = True
                except KeyError:
                    pass
            if not cacheUsed:
	            # Please use 'produce' as little as possible;
    	        # there is almost always a more elegant and modular solution!
                if 'produce' in tags:
                    methodResult = interface[method].getTaggedValue('produce')(self)
                else:
                    methodResult = getattr(self, method)()

                if hasattr(self, "_p_oid"):
                    try:
                        self.__fossilAttrsCache[self._p_oid]
                    except KeyError:
                        self.__fossilAttrsCache[self._p_oid] = {}
                    self.__fossilAttrsCache[self._p_oid][method] = methodResult

            # Result conversion
            if 'result' in tags:
                targetInterface = interface[method].getTaggedValue('result')
                #targetInterface = globals()[targetInterfaceName]

                methodResult = Fossilizable.fossilizeIterable(
                    methodResult, targetInterface, **kwargs)

            # Conversion function
            if 'convert' in tags:
                convertFunction = interface[method].getTaggedValue('convert')
                converterArgNames = inspect.getargspec(convertFunction)[0]
                converterArgs = dict((name, kwargs[name])
                                     for name in converterArgNames
                                     if name in kwargs)
                methodResult = convertFunction(methodResult, **converterArgs)

            # Re-name the attribute produced by the method
            if 'name' in tags:
                attrName = interface[method].getTaggedValue('name')
            else:
                attrName = self.__extractName(method)

            # In case the name contains dots, each of the 'domains' but the
            # last one are translated into nested dictionnaries. For example,
            # if we want to re-name an attribute into "foo.bar.tofu", the
            # corresponding fossilized attribute will be of the form:
            # {"foo":{"bar":{"tofu": res,...},...},...}
            # instead of:
            # {"foo.bar.tofu": res, ...}

            current = result
            attrList = attrName.split('.')

            while len(attrList) > 1:
                attr = attrList.pop(0)
                try:
                    current = current[attr]
                except KeyError:
                    current[attr] = {}
                    current = current[attr]

            # For the last attribute level
            current[attrList[0]] = methodResult

        if "_type" in result or "_fossil" in result:
            raise InvalidFossilException('"_type" or "_fossil"'
                                         ' cannot be a fossil attribute  name')
        else:
            result["_type"] = self.__class__.__name__
            if fossilName: #we check that it's not an empty string
                result["_fossil"] = fossilName
            else:
                result["_fossil"] = ""

        return result

def fossilize(target, interfaceArg = None, useAttrCache = False, **kwargs):
    """
    Method that allows the "fossilization" process to
    be called on data structures (lists, dictionaries
    and sets) as well as normal `Fossilizable` objects.

    :param target: target object to be fossilized
    :type target: Fossilizable
    :param interfaceArg: target fossil type
    :type interfaceArg: IFossil, NoneType, or dict
    :param useAttrCache: use the attribute caching
    :type useAttrCache: boolean
    """
    return Fossilizable.fossilizeIterable(target, interfaceArg, useAttrCache,
                                           **kwargs)
<|endoftext|>


<commit_before>import logging
import requests
from functools import lru_cache

pubchem_url = 'https://pubchem.ncbi.nlm.nih.gov/rest/pug'

logger = logging.getLogger(__name__)

@lru_cache(maxsize=5000)
def get_inchi_key(pubchem_cid):
    """Return the InChIKey for a given PubChem CID.

    Parameters
    ----------
    pubchem_cid : str
        The PubChem CID whose InChIKey should be returned.

    Returns
    -------
    str
        The InChIKey corresponding to the PubChem CID.
    """
    url = '%s/compound/cid/%s/property/InChIKey/TXT' % \
        (pubchem_url, pubchem_cid)
    res = requests.get(url)
    if res.status_code != 200:
        logger.error('Could not retrieve InChIKey for %s' % pubchem_cid)
        return None
    return res.text.strip()

@lru_cache(maxsize=5000)
def get_json_record(pubchem_cid):
    """Return the JSON record of a given PubChem CID.

    Parameters
    ----------
    pubchem_cid : str
        The PubChem CID whose record should be returned.

    Returns
    -------
    dict
        The record deserialized from JSON.
    """
    url = (pubchem_url + '_view/data/compound/%s/JSON/') % pubchem_cid
    res = requests.get(url)
    return res.json()

def get_preferred_compound_ids(pubchem_cid):
    """Return a list of preferred CIDs for a given PubChem CID.

    Parameters
    ----------
    pubchem_cid : str
        The PubChem CID whose preferred CIDs should be returned.

    Returns
    -------
    list of str
        The list of preferred CIDs for the given CID. If there are no
        preferred CIDs for the given CID then an empty list is returned.
    """
    record = get_json_record(pubchem_cid)
    sections = record['Record']['Section']
    pref_ids = set()
    for section in sections:
        if section['TOCHeading'] == 'Preferred Compound':
            for pref_cpd in section['Information']:
                pref_ids |= set(pref_cpd['Value']['Number'])
    pref_ids = sorted([str(pid) for pid in pref_ids])
    return pref_ids

def get_pmids(pubchem_cid):
    """Return xref PMIDs for a given PubChem CID.

    Parameters
    ----------
    pubchem_cid : str
        The PubChem CID whose xref PubMedID's will be returned.

    Returns
    -------
    list
        PubMedIDs corresponding to the given PubChem CID. If none present,
        an empty list is returned.
    """
    url = '%s/compound/cid/%s/xrefs/PubMedID/JSON' % \
          (pubchem_url, pubchem_cid)
    res = requests.get(url)
    res_json = res.json()
    pmids_list = res_json['InformationList']['Information'][0]['PubMedID']
    return pmids_list
<commit_msg>fix bad request<commit_after>import logging
import requests
from functools import lru_cache

pubchem_url = 'https://pubchem.ncbi.nlm.nih.gov/rest/pug'

logger = logging.getLogger(__name__)

@lru_cache(maxsize=5000)
def get_inchi_key(pubchem_cid):
    """Return the InChIKey for a given PubChem CID.

    Parameters
    ----------
    pubchem_cid : str
        The PubChem CID whose InChIKey should be returned.

    Returns
    -------
    str
        The InChIKey corresponding to the PubChem CID.
    """
    url = '%s/compound/cid/%s/property/InChIKey/TXT' % \
        (pubchem_url, pubchem_cid)
    res = requests.get(url)
    if res.status_code != 200:
        logger.error('Could not retrieve InChIKey for %s' % pubchem_cid)
        return None
    return res.text.strip()

@lru_cache(maxsize=5000)
def get_json_record(pubchem_cid):
    """Return the JSON record of a given PubChem CID.

    Parameters
    ----------
    pubchem_cid : str
        The PubChem CID whose record should be returned.

    Returns
    -------
    dict
        The record deserialized from JSON.
    """
    url = (pubchem_url + '_view/data/compound/%s/JSON/') % pubchem_cid
    res = requests.get(url)
    return res.json()

def get_preferred_compound_ids(pubchem_cid):
    """Return a list of preferred CIDs for a given PubChem CID.

    Parameters
    ----------
    pubchem_cid : str
        The PubChem CID whose preferred CIDs should be returned.

    Returns
    -------
    list of str
        The list of preferred CIDs for the given CID. If there are no
        preferred CIDs for the given CID then an empty list is returned.
    """
    record = get_json_record(pubchem_cid)
    sections = record['Record']['Section']
    pref_ids = set()
    for section in sections:
        if section['TOCHeading'] == 'Preferred Compound':
            for pref_cpd in section['Information']:
                pref_ids |= set(pref_cpd['Value']['Number'])
    pref_ids = sorted([str(pid) for pid in pref_ids])
    return pref_ids

def get_pmids(pubchem_cid):
    """Return xref PMIDs for a given PubChem CID.

    Parameters
    ----------
    pubchem_cid : str
        The PubChem CID whose xref PubMedID's will be returned.

    Returns
    -------
    list
        PubMedIDs corresponding to the given PubChem CID. If none present,
        an empty list is returned.
    """
    url = '%s/compound/cid/%s/xrefs/PubMedID/JSON' % \
          (pubchem_url, pubchem_cid)
    res = requests.get(url)
    if res.status_code != 200:
        logger.error('Could not retrieve PMIDs for %s' % pubchem_cid)
        return None
    res_json = res.json()
    pmids_list = res_json['InformationList']['Information'][0]['PubMedID']
    return pmids_list
<|endoftext|>


<commit_before>"""
    Created on 1 May 2014

    @author: Max Demian
"""

import re
import sys

from PyQt4 import QtGui, QtCore, uic

class FileBrowser(QtGui.QMainWindow):

    def __init__(self, parent=None):

        super(FileBrowser, self).__init__(parent)
        uic.loadUi("filebrowser.ui", self)
        # Main Window
#         self.setGeometry(300, 200, 760, 600)
#         self.setWindowTitle("FileBrowser")
#         self.setWindowIcon(QtGui.QIcon("data/icon.png"))
#
#         QtGui.QToolTip.setFont(QtGui.QFont("SansSerif", 10))
#
#         model = QtGui.QFileSystemModel()
#         model.setRootPath("")
#         model.setFilter(QtCore.QDir.AllDirs | QtCore.QDir.NoDotAndDotDot |
#                         QtCore.QDir.Hidden)
#         view = QtGui.QTreeView()
#         view.setModel(model)
#         view.setColumnHidden(1, True)
#         view.setColumnHidden(2, True)
#         view.setColumnHidden(3, True)
#         view.header().hide()
#          self.setCentralWidget(view)

def main():
    "Main entry point."

    app = QtGui.QApplication(sys.argv)
    browser = FileBrowser()
    browser.show()
    sys.exit(app.exec_())

if __name__ == "__main__":
    main()
<commit_msg>retroactive continuity<commit_after>"""
    Created on 1 May 2014

    @author: Max Demian
"""

import re
import sys

from PyQt4 import QtGui, QtCore, uic

class FileBrowser(QtGui.QMainWindow):

    def __init__(self, parent=None):

        super(FileBrowser, self).__init__(parent)
        uic.loadUi("filebrowser.ui", self)
        # Main Window
#         self.setGeometry(300, 200, 760, 600)
#         self.setWindowTitle("FileBrowser")
#         self.setWindowIcon(QtGui.QIcon("data/icon.png"))
#
#         QtGui.QToolTip.setFont(QtGui.QFont("SansSerif", 10))
#
#         model = QtGui.QFileSystemModel()
#         model.setRootPath("")
#         model.setFilter(QtCore.QDir.AllDirs | QtCore.QDir.NoDotAndDotDot |
#                         QtCore.QDir.Hidden)
#         view = QtGui.QTreeView()
#         view.setModel(model)
#         view.setColumnHidden(1, True)
#         view.setColumnHidden(2, True)
#         view.setColumnHidden(3, True)
#         view.header().hide()
#         self.setCentralWidget(view)

def main():
    "Main entry point."

    app = QtGui.QApplication(sys.argv)
    browser = FileBrowser()
    browser.show()
    sys.exit(app.exec_())

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import contextlib
import os

from cupy.cuda import compiler  # NOQA
from cupy.cuda import device  # NOQA
from cupy.cuda import driver  # NOQA
from cupy.cuda import function  # NOQA
from cupy.cuda import memory  # NOQA
from cupy.cuda import memory_hook  # NOQA
from cupy.cuda import memory_hooks  # NOQA
from cupy.cuda import pinned_memory  # NOQA
from cupy.cuda import profiler  # NOQA
from cupy.cuda import runtime  # NOQA
from cupy.cuda import stream  # NOQA
from cupy.cuda import texture  # NOQA

_available = None
_cuda_path = None
_cub_disabled = None

from cupy.cuda import cusolver  # NOQA
cusolver_enabled = True

try:
    from cupy.cuda import nvtx  # NOQA
    nvtx_enabled = True
except ImportError:
    nvtx_enabled = False

try:
    from cupy.cuda import thrust  # NOQA
    thrust_enabled = True
except ImportError:
    thrust_enabled = False

cub_enabled = False
if int(os.getenv('CUB_DISABLED', 0)) == 0:
    try:
        from cupy.cuda import cub  # NOQA
        cub_enabled = True
    except ImportError:
        pass

try:
    from cupy.cuda import nccl  # NOQA
    nccl_enabled = True
except ImportError:
    nccl_enabled = False

try:
    from cupy.cuda import cutensor  # NOQA
    cutensor_enabled = True
except ImportError:
    cutensor_enabled = False

def is_available():
    global _available
    if _available is None:
        _available = False
        try:
            _available = runtime.getDeviceCount() > 0
        except Exception as e:
            if (e.args[0] !=
                    'cudaErrorNoDevice: no CUDA-capable device is detected'):
                raise
    return _available

def get_cuda_path():
    global _cuda_path
    if _cuda_path is None:
        _cuda_path = os.getenv('CUDA_PATH', None)
        if _cuda_path is not None:
            return _cuda_path

        for p in os.getenv('PATH', '').split(os.pathsep):
            for cmd in ('nvcc', 'nvcc.exe'):
                nvcc_path = os.path.join(p, cmd)
                if not os.path.exists(nvcc_path):
                    continue
                nvcc_dir = os.path.dirname(os.path.abspath(nvcc_path))
                _cuda_path = os.path.normpath(os.path.join(nvcc_dir, '..'))
                return _cuda_path

        if os.path.exists('/usr/local/cuda'):
            _cuda_path = '/usr/local/cuda'

    return _cuda_path

# import class and function
from cupy.cuda.compiler import compile_with_cache  # NOQA
from cupy.cuda.device import Device  # NOQA
from cupy.cuda.device import get_cublas_handle  # NOQA
from cupy.cuda.device import get_device_id  # NOQA
from cupy.cuda.function import Function  # NOQA
from cupy.cuda.function import Module  # NOQA
from cupy.cuda.memory import alloc  # NOQA
from cupy.cuda.memory import BaseMemory  # NOQA
from cupy.cuda.memory import malloc_managed  # NOQA
from cupy.cuda.memory import ManagedMemory  # NOQA
from cupy.cuda.memory import Memory  # NOQA
from cupy.cuda.memory import MemoryPointer  # NOQA
from cupy.cuda.memory import MemoryPool  # NOQA
from cupy.cuda.memory import set_allocator  # NOQA
from cupy.cuda.memory import get_allocator  # NOQA
from cupy.cuda.memory import UnownedMemory  # NOQA
from cupy.cuda.memory_hook import MemoryHook  # NOQA
from cupy.cuda.pinned_memory import alloc_pinned_memory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPointer  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPool  # NOQA
from cupy.cuda.pinned_memory import set_pinned_memory_allocator  # NOQA
from cupy.cuda.stream import Event  # NOQA
from cupy.cuda.stream import get_current_stream  # NOQA
from cupy.cuda.stream import get_elapsed_time  # NOQA
from cupy.cuda.stream import Stream  # NOQA

@contextlib.contextmanager
def profile():
    """Enable CUDA profiling during with statement.

    This function enables profiling on entering a with statement, and disables
    profiling on leaving the statement.

    >>> with cupy.cuda.profile():
    ...    # do something you want to measure
    ...    pass

    """
    profiler.start()
    try:
        yield
    finally:
        profiler.stop()
<commit_msg>add comment to __init__.py too<commit_after>import contextlib
import os

from cupy.cuda import compiler  # NOQA
from cupy.cuda import device  # NOQA
from cupy.cuda import driver  # NOQA
from cupy.cuda import function  # NOQA
from cupy.cuda import memory  # NOQA
from cupy.cuda import memory_hook  # NOQA
from cupy.cuda import memory_hooks  # NOQA
from cupy.cuda import pinned_memory  # NOQA
from cupy.cuda import profiler  # NOQA
from cupy.cuda import runtime  # NOQA
from cupy.cuda import stream  # NOQA
from cupy.cuda import texture  # NOQA

_available = None
_cuda_path = None
_cub_disabled = None

from cupy.cuda import cusolver  # NOQA
# This flag is kept for backward compatibility.
# It is always True as cuSOLVER library is always available in CUDA 8.0+.
cusolver_enabled = True

try:
    from cupy.cuda import nvtx  # NOQA
    nvtx_enabled = True
except ImportError:
    nvtx_enabled = False

try:
    from cupy.cuda import thrust  # NOQA
    thrust_enabled = True
except ImportError:
    thrust_enabled = False

cub_enabled = False
if int(os.getenv('CUB_DISABLED', 0)) == 0:
    try:
        from cupy.cuda import cub  # NOQA
        cub_enabled = True
    except ImportError:
        pass

try:
    from cupy.cuda import nccl  # NOQA
    nccl_enabled = True
except ImportError:
    nccl_enabled = False

try:
    from cupy.cuda import cutensor  # NOQA
    cutensor_enabled = True
except ImportError:
    cutensor_enabled = False

def is_available():
    global _available
    if _available is None:
        _available = False
        try:
            _available = runtime.getDeviceCount() > 0
        except Exception as e:
            if (e.args[0] !=
                    'cudaErrorNoDevice: no CUDA-capable device is detected'):
                raise
    return _available

def get_cuda_path():
    global _cuda_path
    if _cuda_path is None:
        _cuda_path = os.getenv('CUDA_PATH', None)
        if _cuda_path is not None:
            return _cuda_path

        for p in os.getenv('PATH', '').split(os.pathsep):
            for cmd in ('nvcc', 'nvcc.exe'):
                nvcc_path = os.path.join(p, cmd)
                if not os.path.exists(nvcc_path):
                    continue
                nvcc_dir = os.path.dirname(os.path.abspath(nvcc_path))
                _cuda_path = os.path.normpath(os.path.join(nvcc_dir, '..'))
                return _cuda_path

        if os.path.exists('/usr/local/cuda'):
            _cuda_path = '/usr/local/cuda'

    return _cuda_path

# import class and function
from cupy.cuda.compiler import compile_with_cache  # NOQA
from cupy.cuda.device import Device  # NOQA
from cupy.cuda.device import get_cublas_handle  # NOQA
from cupy.cuda.device import get_device_id  # NOQA
from cupy.cuda.function import Function  # NOQA
from cupy.cuda.function import Module  # NOQA
from cupy.cuda.memory import alloc  # NOQA
from cupy.cuda.memory import BaseMemory  # NOQA
from cupy.cuda.memory import malloc_managed  # NOQA
from cupy.cuda.memory import ManagedMemory  # NOQA
from cupy.cuda.memory import Memory  # NOQA
from cupy.cuda.memory import MemoryPointer  # NOQA
from cupy.cuda.memory import MemoryPool  # NOQA
from cupy.cuda.memory import set_allocator  # NOQA
from cupy.cuda.memory import get_allocator  # NOQA
from cupy.cuda.memory import UnownedMemory  # NOQA
from cupy.cuda.memory_hook import MemoryHook  # NOQA
from cupy.cuda.pinned_memory import alloc_pinned_memory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPointer  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPool  # NOQA
from cupy.cuda.pinned_memory import set_pinned_memory_allocator  # NOQA
from cupy.cuda.stream import Event  # NOQA
from cupy.cuda.stream import get_current_stream  # NOQA
from cupy.cuda.stream import get_elapsed_time  # NOQA
from cupy.cuda.stream import Stream  # NOQA

@contextlib.contextmanager
def profile():
    """Enable CUDA profiling during with statement.

    This function enables profiling on entering a with statement, and disables
    profiling on leaving the statement.

    >>> with cupy.cuda.profile():
    ...    # do something you want to measure
    ...    pass

    """
    profiler.start()
    try:
        yield
    finally:
        profiler.stop()
<|endoftext|>


<commit_before>import contextlib
import warnings

from cupy._environment import get_cuda_path, get_nvcc_path  # NOQA
from cupy.cuda import compiler  # NOQA
from cupy.cuda import device  # NOQA
from cupy.cuda import function  # NOQA
from cupy.cuda import memory  # NOQA
from cupy.cuda import memory_hook  # NOQA
from cupy.cuda import memory_hooks  # NOQA
from cupy.cuda import pinned_memory  # NOQA
from cupy.cuda import stream  # NOQA
from cupy.cuda import texture  # NOQA
from cupy_backends.cuda.api import driver  # NOQA
from cupy_backends.cuda.api import runtime  # NOQA
from cupy_backends.cuda.libs import cublas  # NOQA
from cupy_backends.cuda.libs import curand  # NOQA
from cupy_backends.cuda.libs import cusolver  # NOQA
from cupy_backends.cuda.libs import cusparse  # NOQA
from cupy_backends.cuda.libs import nvrtc  # NOQA
from cupy_backends.cuda.libs import profiler  # NOQA

_available = None

class _UnavailableModule():
    available = False

    def __init__(self, name):
        self.__name__ = name

from cupy.cuda import cub  # NOQA
from cupy.cuda import jitify  # NOQA

try:
    from cupy.cuda import nvtx  # NOQA
except ImportError:
    nvtx = _UnavailableModule('cupy.cuda.nvtx')

try:
    from cupy.cuda import thrust  # NOQA
except ImportError:
    thrust = _UnavailableModule('cupy.cuda.thrust')

try:
    from cupy.cuda import nccl  # NOQA
except ImportError:
    nccl = _UnavailableModule('cupy.cuda.nccl')

try:
    from cupy_backends.cuda.libs import cutensor
except ImportError:
    cutensor = _UnavailableModule('cupy.cuda.cutensor')

def __getattr__(key):
    # `*_enabled` flags are kept for backward compatibility.
    # Note: module-level getattr only runs on Python 3.7+.
    if key == 'cusolver_enabled':
        # cuSOLVER is always available in CUDA 8.0+.
        warnings.warn('''
cupy.cuda.cusolver_enabled has been deprecated in CuPy v8 and will be removed in the future release.
This flag always returns True as cuSOLVER is always available in CUDA 8.0 or later.
            ''', DeprecationWarning)  # NOQA
        return True

    for mod in [nvtx, nccl, thrust, cub, cutensor]:
        flag = '{}_enabled'.format(mod.__name__.split('.')[-1])
        if key == flag:
            warnings.warn('''
cupy.cuda.{} has been deprecated in CuPy v8 and will be removed in the future release.
Use {}.available instead.
                '''.format(flag, mod.__name__), DeprecationWarning)  # NOQA
            return not isinstance(mod, _UnavailableModule)

    raise AttributeError(
        "module '{}' has no attribute '{}'".format(__name__, key))

def is_available():
    global _available
    if _available is None:
        _available = False
        try:
            _available = runtime.getDeviceCount() > 0
        except Exception as e:
            if (not runtime.is_hip and e.args[0] !=
                    'cudaErrorNoDevice: no CUDA-capable device is detected'):
                raise
            elif runtime.is_hip and 'hipErrorNoDevice' not in e.args[0]:
                raise
    return _available

# import class and function
from cupy.cuda.compiler import compile_with_cache  # NOQA
from cupy.cuda.device import Device  # NOQA
from cupy.cuda.device import get_cublas_handle  # NOQA
from cupy.cuda.device import get_device_id  # NOQA
from cupy.cuda.function import Function  # NOQA
from cupy.cuda.function import Module  # NOQA
from cupy.cuda.memory import alloc  # NOQA
from cupy.cuda.memory import BaseMemory  # NOQA
from cupy.cuda.memory import malloc_managed  # NOQA
from cupy.cuda.memory import ManagedMemory  # NOQA
from cupy.cuda.memory import Memory  # NOQA
from cupy.cuda.memory import MemoryPointer  # NOQA
from cupy.cuda.memory import MemoryPool  # NOQA
from cupy.cuda.memory import PythonFunctionAllocator  # NOQA
from cupy.cuda.memory import set_allocator  # NOQA
from cupy.cuda.memory import get_allocator  # NOQA
from cupy.cuda.memory import UnownedMemory  # NOQA
from cupy.cuda.memory_hook import MemoryHook  # NOQA
from cupy.cuda.pinned_memory import alloc_pinned_memory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPointer  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPool  # NOQA
from cupy.cuda.pinned_memory import set_pinned_memory_allocator  # NOQA
from cupy.cuda.stream import Event  # NOQA
from cupy.cuda.stream import get_current_stream  # NOQA
from cupy.cuda.stream import get_elapsed_time  # NOQA
from cupy.cuda.stream import Stream  # NOQA
from cupy.cuda.stream import ExternalStream  # NOQA

@contextlib.contextmanager
def using_allocator(allocator=None):
    """Sets a thread-local allocator for GPU memory inside
       context manager

    Args:
        allocator (function): CuPy memory allocator. It must have the same
            interface as the :func:`cupy.cuda.alloc` function, which takes the
            buffer size as an argument and returns the device buffer of that
            size. When ``None`` is specified, raw memory allocator will be
            used (i.e., memory pool is disabled).
    """
    # Note: cupy/memory.pyx would be the better place to implement this
    # function but `contextmanager` decoration doesn't behave well in Cython.
    if allocator is None:
        allocator = memory._malloc
    previous_allocator = memory._get_thread_local_allocator()
    memory._set_thread_local_allocator(allocator)
    try:
        yield
    finally:
        memory._set_thread_local_allocator(previous_allocator)

@contextlib.contextmanager
def profile():
    """Enable CUDA profiling during with statement.

    This function enables profiling on entering a with statement, and disables
    profiling on leaving the statement.

    >>> with cupy.cuda.profile():
    ...    # do something you want to measure
    ...    pass

    .. note::
        When starting ``nvprof`` from the command line, manually setting
        ``--profile-from-start off`` may be required for the desired behavior.

    """
    profiler.start()
    try:
        yield
    finally:
        profiler.stop()
<commit_msg>fix import<commit_after>import contextlib
import warnings

from cupy._environment import get_cuda_path, get_nvcc_path  # NOQA
from cupy.cuda import compiler  # NOQA
from cupy.cuda import device  # NOQA
from cupy.cuda import function  # NOQA
from cupy.cuda import memory  # NOQA
from cupy.cuda import memory_hook  # NOQA
from cupy.cuda import memory_hooks  # NOQA
from cupy.cuda import pinned_memory  # NOQA
from cupy.cuda import stream  # NOQA
from cupy.cuda import texture  # NOQA
from cupy_backends.cuda.api import driver  # NOQA
from cupy_backends.cuda.api import runtime  # NOQA
from cupy_backends.cuda.libs import cublas  # NOQA
from cupy_backends.cuda.libs import curand  # NOQA
from cupy_backends.cuda.libs import cusolver  # NOQA
from cupy_backends.cuda.libs import cusparse  # NOQA
from cupy_backends.cuda.libs import nvrtc  # NOQA
from cupy_backends.cuda.libs import profiler  # NOQA

_available = None

class _UnavailableModule():
    available = False

    def __init__(self, name):
        self.__name__ = name

from cupy.cuda import cub  # NOQA

if not runtime.is_hip and driver.get_build_version() > 0:
    from cupy.cuda import jitify  # NOQA
else:
    jitify = None

try:
    from cupy.cuda import nvtx  # NOQA
except ImportError:
    nvtx = _UnavailableModule('cupy.cuda.nvtx')

try:
    from cupy.cuda import thrust  # NOQA
except ImportError:
    thrust = _UnavailableModule('cupy.cuda.thrust')

try:
    from cupy.cuda import nccl  # NOQA
except ImportError:
    nccl = _UnavailableModule('cupy.cuda.nccl')

try:
    from cupy_backends.cuda.libs import cutensor
except ImportError:
    cutensor = _UnavailableModule('cupy.cuda.cutensor')

def __getattr__(key):
    # `*_enabled` flags are kept for backward compatibility.
    # Note: module-level getattr only runs on Python 3.7+.
    if key == 'cusolver_enabled':
        # cuSOLVER is always available in CUDA 8.0+.
        warnings.warn('''
cupy.cuda.cusolver_enabled has been deprecated in CuPy v8 and will be removed in the future release.
This flag always returns True as cuSOLVER is always available in CUDA 8.0 or later.
            ''', DeprecationWarning)  # NOQA
        return True

    for mod in [nvtx, nccl, thrust, cub, cutensor]:
        flag = '{}_enabled'.format(mod.__name__.split('.')[-1])
        if key == flag:
            warnings.warn('''
cupy.cuda.{} has been deprecated in CuPy v8 and will be removed in the future release.
Use {}.available instead.
                '''.format(flag, mod.__name__), DeprecationWarning)  # NOQA
            return not isinstance(mod, _UnavailableModule)

    raise AttributeError(
        "module '{}' has no attribute '{}'".format(__name__, key))

def is_available():
    global _available
    if _available is None:
        _available = False
        try:
            _available = runtime.getDeviceCount() > 0
        except Exception as e:
            if (not runtime.is_hip and e.args[0] !=
                    'cudaErrorNoDevice: no CUDA-capable device is detected'):
                raise
            elif runtime.is_hip and 'hipErrorNoDevice' not in e.args[0]:
                raise
    return _available

# import class and function
from cupy.cuda.compiler import compile_with_cache  # NOQA
from cupy.cuda.device import Device  # NOQA
from cupy.cuda.device import get_cublas_handle  # NOQA
from cupy.cuda.device import get_device_id  # NOQA
from cupy.cuda.function import Function  # NOQA
from cupy.cuda.function import Module  # NOQA
from cupy.cuda.memory import alloc  # NOQA
from cupy.cuda.memory import BaseMemory  # NOQA
from cupy.cuda.memory import malloc_managed  # NOQA
from cupy.cuda.memory import ManagedMemory  # NOQA
from cupy.cuda.memory import Memory  # NOQA
from cupy.cuda.memory import MemoryPointer  # NOQA
from cupy.cuda.memory import MemoryPool  # NOQA
from cupy.cuda.memory import PythonFunctionAllocator  # NOQA
from cupy.cuda.memory import set_allocator  # NOQA
from cupy.cuda.memory import get_allocator  # NOQA
from cupy.cuda.memory import UnownedMemory  # NOQA
from cupy.cuda.memory_hook import MemoryHook  # NOQA
from cupy.cuda.pinned_memory import alloc_pinned_memory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemory  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPointer  # NOQA
from cupy.cuda.pinned_memory import PinnedMemoryPool  # NOQA
from cupy.cuda.pinned_memory import set_pinned_memory_allocator  # NOQA
from cupy.cuda.stream import Event  # NOQA
from cupy.cuda.stream import get_current_stream  # NOQA
from cupy.cuda.stream import get_elapsed_time  # NOQA
from cupy.cuda.stream import Stream  # NOQA
from cupy.cuda.stream import ExternalStream  # NOQA

@contextlib.contextmanager
def using_allocator(allocator=None):
    """Sets a thread-local allocator for GPU memory inside
       context manager

    Args:
        allocator (function): CuPy memory allocator. It must have the same
            interface as the :func:`cupy.cuda.alloc` function, which takes the
            buffer size as an argument and returns the device buffer of that
            size. When ``None`` is specified, raw memory allocator will be
            used (i.e., memory pool is disabled).
    """
    # Note: cupy/memory.pyx would be the better place to implement this
    # function but `contextmanager` decoration doesn't behave well in Cython.
    if allocator is None:
        allocator = memory._malloc
    previous_allocator = memory._get_thread_local_allocator()
    memory._set_thread_local_allocator(allocator)
    try:
        yield
    finally:
        memory._set_thread_local_allocator(previous_allocator)

@contextlib.contextmanager
def profile():
    """Enable CUDA profiling during with statement.

    This function enables profiling on entering a with statement, and disables
    profiling on leaving the statement.

    >>> with cupy.cuda.profile():
    ...    # do something you want to measure
    ...    pass

    .. note::
        When starting ``nvprof`` from the command line, manually setting
        ``--profile-from-start off`` may be required for the desired behavior.

    """
    profiler.start()
    try:
        yield
    finally:
        profiler.stop()
<|endoftext|>


<commit_before>from cupy.random import distributions
from cupy.random import generator

def rand(*size, **kwarg):
    """Returns an array of uniform random values over the interval ``[0, 1)``.

    Each element of the array is uniformly distributed on the half-open
    interval ``[0, 1)``. All elements are identically and independently
    distributed (i.i.d.).

    Args:
        size (tuple of ints): The shape of the array.
        dtype: Data type specifier. Only float32 and float64 types are allowed.
            The default is float64.

    Returns:
        cupy.ndarray: A random array.

    .. seealso:: :func:`numpy.random.rand`

    """
    dtype = kwarg.pop('dtype', float)
    if kwarg:
        raise TypeError('rand() got unexpected keyward arguments %s'
                        % ', '.join(kwarg.keys()))
    return random_sample(size=size, dtype=dtype)

def randn(*size, **kwarg):
    """Returns an array of standand normal random values.

    Each element of the array is normally distributed with zero mean and unit
    variance. All elements are identically and independently distributed
    (i.i.d.).

    Args:
        size (tuple of ints): The shape of the array.
        dtype: Data type specifier. Only float32 and float64 types are allowed.
            The default is float64.

    Returns:
        cupy.ndarray: An array of standanr normal random values.

    .. seealso:: :func:`numpy.random.randn`

    """
    dtype = kwarg.pop('dtype', float)
    if kwarg:
        raise TypeError('randn() got unexpected keyward arguments %s'
                        % ', '.join(kwarg.keys()))
    return distributions.normal(size=size, dtype=dtype)

def randint(low, high=None, size=None):
    """Returns a scalar or an array of integer values over ``[low, high)``.

    Each element of returned values are independently sampled from
    uniform distribution over left-close and right-open interval
    ``[low, high)``.

    Args:
        low (int): If ``high`` is not ``None``,
        it is the lower bound of the interval.
        Otherwise, it is the **upper** bound of the interval
        and lower bound of the inteval is set to ``0``.
        high (int): Upper bound of the interval.
        size (None or int or tuple of ints): The shape of returned value.

    Returns:
        int or cupy.ndarray of ints: If size is ``None``,
        it is single integer sampled.
        If size is integer, it is the 1D-array of length ``size`` element.
        Otherwise, it is the array whose shape specified by ``size``.
    """
    if high is None:
        lo = 0
        hi = low
    else:
        lo = low
        hi = high

    if lo >= hi:
        raise ValueError('low >= high')

    diff = hi - lo - 1
    rs = generator.get_random_state()
    return lo + rs.interval(diff, size)

def random_integers(low, high=None, size=None):
    """Return a scalar or an array of interger values over ``[low, high]``

    Each element of returned values are independently sampled from
    uniform distribution over closed interval ``[low, high]``.

    Args:
        low (int): If ``high`` is not ``None``,
        it is the lower bound of the interval.
        Otherwise, it is the **upper** bound of the interval
        and the lower bound is set to ``1``.
        high (int): Upper bound of the interval.
        size (None or int or tuple of ints): The shape of returned value.

    Returns:
        int or cupy.ndarray of ints: If size is ``None``,
        it is single integer sampled.
        If size is integer, it is the 1D-array of length ``size`` element.
        Otherwise, it is the array whose shape specified by ``size``.
    """
    if high is None:
        high = low
        low = 1
    return randint(low, high+1, size)

def random_sample(size=None, dtype=float):
    """Returns an array of random values over the interval ``[0, 1)``.

    This is a variant of :func:`cupy.random.rand`.

    Args:
        size (int or tuple of ints): The shape of the array.
        dtype: Data type specifier. Only float32 and float64 types are allowed.

    Returns:
        cupy.ndarray: An array of uniformly distributed random values.

    .. seealso:: :func:`numpy.random.random_sample`

    """
    rs = generator.get_random_state()
    return rs.random_sample(size=size, dtype=dtype)

# TODO(okuta): Implement choice
<commit_msg>Fix document style<commit_after>from cupy.random import distributions
from cupy.random import generator

def rand(*size, **kwarg):
    """Returns an array of uniform random values over the interval ``[0, 1)``.

    Each element of the array is uniformly distributed on the half-open
    interval ``[0, 1)``. All elements are identically and independently
    distributed (i.i.d.).

    Args:
        size (tuple of ints): The shape of the array.
        dtype: Data type specifier. Only float32 and float64 types are allowed.
            The default is float64.

    Returns:
        cupy.ndarray: A random array.

    .. seealso:: :func:`numpy.random.rand`

    """
    dtype = kwarg.pop('dtype', float)
    if kwarg:
        raise TypeError('rand() got unexpected keyward arguments %s'
                        % ', '.join(kwarg.keys()))
    return random_sample(size=size, dtype=dtype)

def randn(*size, **kwarg):
    """Returns an array of standand normal random values.

    Each element of the array is normally distributed with zero mean and unit
    variance. All elements are identically and independently distributed
    (i.i.d.).

    Args:
        size (tuple of ints): The shape of the array.
        dtype: Data type specifier. Only float32 and float64 types are allowed.
            The default is float64.

    Returns:
        cupy.ndarray: An array of standanr normal random values.

    .. seealso:: :func:`numpy.random.randn`

    """
    dtype = kwarg.pop('dtype', float)
    if kwarg:
        raise TypeError('randn() got unexpected keyward arguments %s'
                        % ', '.join(kwarg.keys()))
    return distributions.normal(size=size, dtype=dtype)

def randint(low, high=None, size=None):
    """Returns a scalar or an array of integer values over ``[low, high)``.

    Each element of returned values are independently sampled from
    uniform distribution over left-close and right-open interval
    ``[low, high)``.

    Args:
        low (int): If ``high`` is not ``None``,
            it is the lower bound of the interval.
            Otherwise, it is the **upper** bound of the interval
            and lower bound of the inteval is set to ``0``.
        high (int): Upper bound of the interval.
        size (None or int or tuple of ints): The shape of returned value.

    Returns:
        int or cupy.ndarray of ints: If size is ``None``,
        it is single integer sampled.
        If size is integer, it is the 1D-array of length ``size`` element.
        Otherwise, it is the array whose shape specified by ``size``.
    """
    if high is None:
        lo = 0
        hi = low
    else:
        lo = low
        hi = high

    if lo >= hi:
        raise ValueError('low >= high')

    diff = hi - lo - 1
    rs = generator.get_random_state()
    return lo + rs.interval(diff, size)

def random_integers(low, high=None, size=None):
    """Return a scalar or an array of interger values over ``[low, high]``

    Each element of returned values are independently sampled from
    uniform distribution over closed interval ``[low, high]``.

    Args:
        low (int): If ``high`` is not ``None``,
            it is the lower bound of the interval.
            Otherwise, it is the **upper** bound of the interval
            and the lower bound is set to ``1``.
        high (int): Upper bound of the interval.
        size (None or int or tuple of ints): The shape of returned value.

    Returns:
        int or cupy.ndarray of ints: If size is ``None``,
        it is single integer sampled.
        If size is integer, it is the 1D-array of length ``size`` element.
        Otherwise, it is the array whose shape specified by ``size``.
    """
    if high is None:
        high = low
        low = 1
    return randint(low, high+1, size)

def random_sample(size=None, dtype=float):
    """Returns an array of random values over the interval ``[0, 1)``.

    This is a variant of :func:`cupy.random.rand`.

    Args:
        size (int or tuple of ints): The shape of the array.
        dtype: Data type specifier. Only float32 and float64 types are allowed.

    Returns:
        cupy.ndarray: An array of uniformly distributed random values.

    .. seealso:: :func:`numpy.random.random_sample`

    """
    rs = generator.get_random_state()
    return rs.random_sample(size=size, dtype=dtype)

# TODO(okuta): Implement choice
<|endoftext|>