<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
zotero.py

Created by Stephan Hügel on 2011-02-28
Copyright Stephan Hügel, 2011

License: http://www.gnu.org/licenses/gpl-3.0.txt
"""

__version__ = '0.5'

import sys
import os
import urllib
import urllib2
import feedparser
import xml.etree.ElementTree as xml
import zotero_errors as ze
import traceback

def open_file(to_read):
    """ Open a text file for reading, and strip the newlines
        returns a list, one list item per line
    """
    try:
        with open(to_read, 'r') as opened:
            return [got_lines.rstrip('\n') for got_lines in opened.readlines()]
    except IOError:
        print "Couldn't read values from %s\nCan't continue." % to_read
        raise

def dedup(suspects):
    """ Check for duplicate key entries (e.g. contributor) and append a number
        to these. This is a horrible hack, but the Zotero API returns items
        with non-unique key values (arbitrary no. of contributors/authors/eds)
    """
    seen = []
    counter = 2
    for candidate in suspects:
        if candidate in seen:
            candidate = '%s_%s' % (candidate, counter)
            counter += 1
            seen.append(candidate)
        else:
            seen.append(candidate)
    return seen

class Zotero(object):
    """ Zotero API methods
        A full list of methods can be found here:
        http://www.zotero.org/support/dev/server_api
        Most of the methods return Atom feed documents, which can be parsed
        using feedparser (http://www.feedparser.org/docs/)

        See the readme for details of valid URL and request parameters
    """

    def __init__(self, user_id = None, user_key = None):
        """ Store Zotero credentials
        """
        self.endpoint = 'https://api.zotero.org'
        if user_id and user_key:
            self.user_id = user_id
            self.user_key = user_key
        else:
            raise ze.MissingCredentials, \
            'Please provide both the user ID and the user key'
        self.url_params = None

    def retrieve_data(self, request = None):
        """ Method for retrieving Zotero items via the API
            Returns a dict containing feed items and lists of entries
        """
        full_url = '%s%s' % (self.endpoint, request)
        try:
            data = urllib2.urlopen(full_url).read()
        except urllib2.HTTPError, error:
            if error.code == 401 or error.code == 403:
                raise ze.UserNotAuthorised, \
"You are not authorised to retrieve this resource (%s)" % error.code
            if error.code == 400:
                raise ze.RateLimitExceeded, \
"Invalid request, probably due to unsupported parameters: %s" % \
                data
            else:
                raise ze.HTTPError, \
                "HTTP Error %s (%s)" % (error.msg, error.code)
        # parse the result into Python data structures
        self.url_params = None
        return feedparser.parse(data)

    def retrieve(output_func):
        """ Decorator for Zotero methods; calls retrieve_data() and passes
            the result to the function specified by output_func
        """
        def wrap(func):
            """ Wrapper for original function
            """
            def wrapped_f(self, *args, **kwargs):
                """ Returns result of retrieve_data(), then output_func
                """
                orig_func = func(self, *args, **kwargs)
                retr = self.retrieve_data(orig_func)
                return eval(output_func)(retr)
            return wrapped_f
        return wrap

    def add_parameters(self, **params):
        """ Set URL parameters. Will always add the user key
        """
        if params:
            params['key'] = self.user_key
        else:
            params = {'key': self.user_key}
        params = urllib.urlencode(params)
        self.url_params = params

    def build_query(self, query_string, params = None):
        """ Set request parameters. Will always add the user ID if it hasn't
            been specifically set by an API call
        """
        params = {'u': self.user_id}
        try:
            query = query_string.format(**params)
        except KeyError, err:
            raise ze.ParamNotPassed, \
            'There\'s a request parameter missing: %s' % err
        # Add the URL parameters and the user key, if necessary
        if not self.url_params:
            self.add_parameters()
        query = '%s?%s' % (query, self.url_params)
        return query

    @retrieve('self.items_data')
    def items(self):
        """ Get user items
        """
        query_string = '/users/{u}/items'
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def top(self):
        """ Get user top-level items
        """
        query_string = '/users/{u}/items/top'
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def item(self, item):
        """ Get a specific item
        """
        query_string = '/users/{u}/items/{i}'
        query_string = query_string.format(u = self.user_id, i = item)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def children(self, item):
        """ Get a specific item
        """
        query_string = '/users/{u}/items/{i}/children'
        query_string = query_string.format(u = self.user_id, i = item)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def tag_items(self, tag):
        """ Get items for a specific tag
        """
        query_string = '/users/{u}/tags/{t}/items'
        query_string = query_string.format(u = self.user_id, t = tag)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def collection_items(self, collection):
        """ Get a specific collection's items
        """
        query_string = '/users/{u}/collections/{c}/items'
        query_string = query_string.format(u = self.user_id, c = collection)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_items(self, group):
        """ Get a specific group's items
        """
        query_string = '/groups/{g}/items'
        query_string = query_string.format(g = group)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_top(self, group):
        """ Get a specific group's top-level items
        """
        query_string = '/groups/{g}/items/top'
        query_string = query_string.format(g = group)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_item(self, group, item):
        """ Get a specific group item
        """
        query_string = '/groups/{g}/items/{i}'
        query_string = query_string.format(g = group, i = item)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_item_children(self, group, item):
        """ Get a specific group item's child items
        """
        query_string = '/groups/{g}/items/{i}/children'
        query_string = query_string.format(g = group, i = item)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_items_tag(self, group, tag):
        """ Get a specific group's items for a specific tag
        """
        query_string = '/groups/{g}/tags/{t}/items'
        query_string = query_string.format(g = group, t = tag)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_collection_items(self, group, collection):
        """ Get a specific group's items for a specific collection
        """
        query_string = '/groups/{g}/collections/{c}/items'
        query_string = query_string.format(g = group, c = collection)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_collection_top(self, group, collection):
        """ Get a specific group's top-level items for a specific collection
        """
        query_string = '/groups/{g}/collections/{c}/items/top'
        query_string = query_string.format(g = group, c = collection)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_collection_item(self, group, collection, item):
        """ Get a specific collection's item from a specific group
        """
        query_string = '/groups/{g}/collections/{c}/items/{i}'
        query_string = query_string.format(g = group, c = collection, i = item)
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def collections(self):
        """ Get user collections
        """
        query_string = '/users/{u}/collections'
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def collections_sub(self, collection):
        """ Get subcollections for a specific collection
        """
        query_string = '/users/{u}/collections/{c}/collections'
        query_string = query_string.format(u = self.user_id, c = collection)
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def group_collections(self, group):
        """ Get collections for a specific group
        """
        query_string = '/groups/{group}/collections'
        query_string = query_string.format(u = self.user_id, g = group)
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def group_collection(self, group, collection):
        """ Get collections for a specific group
        """
        query_string = '/groups/{g}/collections/{c}'.format(
        u = self.user_id,
        g = group,
        c = collection)
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def group_collection_sub(self, group, collection):
        """ Get collections for a specific group
        """
        query_string = '/groups/{g}/collections/{c}/collections'.format(
        u = self.user_id,
        g = group,
        c = collection)
        return self.build_query(query_string)

    @retrieve('self.groups_data')
    def groups(self):
        """ Get user groups
        """
        query_string = '/users/{u}/groups'
        return self.build_query(query_string)

    @retrieve('self.tags_data')
    def tags(self):
        """ Get tags for a specific item
        """
        query_string = '/users/{u}/tags'
        return self.build_query(query_string)

    @retrieve('self.tags_data')
    def item_tags(self, item):
        """ Get tags for a specific item
        """
        query_string = '/users/{u}/items/{i}/tags'.format(
        u = self.user_id,
        i = item)
        return self.build_query(query_string)

    @retrieve('self.tags_data')
    def group_tags(self, group):
        """ Get tags for a specific group
        """
        query_string = '/groups/{g}/tags'.format(
        u = self.user_id,
        g = group)
        return self.build_query(query_string)

    @retrieve('self.tags_data')
    def group_item_tags(self, group, item):
        """ Get tags for a specific group
        """
        query_string = '/groups/{g}/items/{i}/tags'.format(
        u = self.user_id,
        g = group,
        i = item)
        return self.build_query(query_string)

    def items_data(self, retrieved):
        """ Format and return data from API calls which return Items
        """
        # Shunt each 'content' block into a list
        item_parsed = [a['content'][0]['value'] for a in retrieved.entries]
        # Shunt each 'title' and item ID value into a list
        item_title = [t['title'] for t in retrieved.entries]
        item_id = [i['zapi_key'] for i in retrieved.entries]
        items = []
        for index, content in enumerate(item_parsed):
            # define a utf-8 parser with which to parse the content HTML
            utf = xml.XMLParser(encoding = 'utf-8')
            elem = xml.XML(content, utf)
            keys = [e.text.lower().encode('utf-8') for e in elem.iter('th')]
            keys = dedup(keys)
            values = [v.text.encode('utf-8') for v in elem.iter('td')]
            zipped = dict(zip(keys, values))
            zipped['title'] = item_title[index].encode('utf-8')
            zipped['id'] = item_id[index].encode('utf-8')
            items.append(Item(zipped))
        return items

    def collections_data(self, retrieved):
        """ Format and return data from API calls which return collections
        """
        collections = []
        collection_key = [k['zapi_key'] for k in retrieved.entries]
        collection_title = [t['title'] for t in retrieved.entries]
        collection_sub = [s['zapi_numcollections'] for s in retrieved.entries]
        for index in range(len(collection_key)):
            collection_data = {}
            collection_data['id'] = collection_key[index].encode('utf-8')
            collection_data['title'] = collection_title[index].encode('utf-8')
            if int(collection_sub[index]) > 0:
                collection_data['subcollections'] = int(collection_sub[index])
            collections.append(collection_data)
        return collections

    def groups_data(self, retrieved):
        """ Format and return data from API calls which return Groups
        """
        groups = []
        group_id = [t['title'] for t in retrieved.entries]
        group_items = [i['zapi_numitems'] for i in retrieved.entries]
        group_author = [a['author'] for a in retrieved.entries]
        for index in range(len(group_id)):
            group_data = {}
            group_data['id'] = group_id[index].encode('utf-8')
            group_data['total_items'] = group_items[index].encode('utf-8')
            group_data['owner'] = group_author[index].encode('utf-8')
            groups.append(group_data)
        return groups

    def tags_data(self, retrieved):
        """ Format and return data from API call which return Tags
        """
        tags = [t['title'].encode('utf-8') for t in retrieved.entries]
        return tags

class Item(object):
    """ Adds all retrieved values as instance properties by key, value.
        Currently, this class is little more than a container for what would
        otherwise be dicts (and I'm not convinced they aren't the way to go)
    """
    def __init__(self, values):
        super(Item, self).__init__()
        # Hackish and bad; creates instance attributes on the fly
        self.__dict__ = dict(values)

    def __getattr__(self, name):
        """ Only called if __getattribute__ fails, so we can return
            None without too much hand-wringing
        """
        return None

def main():
    """ main function
    """
    # Read a file from your cwd. Expects user id on line 1, key on line 2, LF
    auth_values = open_file(os.path.join(os.path.expanduser('~'),
    'zotero_keys.txt'))
    zot_id = auth_values[0]
    zot_key = auth_values[1]
    zot = Zotero(zot_id, zot_key)
    zot.add_parameters(limit=10, start=50)
    items = zot.top()
    for item in items:
        print item.title, item.id

if __name__ == "__main__":
    try:
        main()
    except (KeyboardInterrupt, SystemExit):
        # actually raise these, for a clean exit
        raise
    except Exception, errm:
        # all other exceptions: display the error
        print errm
        print "Stack trace:\n", traceback.print_exc(file = sys.stdout)
    else:
        pass
    finally:
        # exit cleanly once we've done everything else
        sys.exit(0)<commit_msg>Removing unnecessary variable assignments from API methods<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
zotero.py

Created by Stephan Hügel on 2011-02-28
Copyright Stephan Hügel, 2011

License: http://www.gnu.org/licenses/gpl-3.0.txt
"""

__version__ = '0.5'

import sys
import os
import urllib
import urllib2
import feedparser
import xml.etree.ElementTree as xml
import zotero_errors as ze
import traceback

def open_file(to_read):
    """ Open a text file for reading, and strip the newlines
        returns a list, one list item per line
    """
    try:
        with open(to_read, 'r') as opened:
            return [got_lines.rstrip('\n') for got_lines in opened.readlines()]
    except IOError:
        print "Couldn't read values from %s\nCan't continue." % to_read
        raise

def dedup(suspects):
    """ Check for duplicate key entries (e.g. contributor) and append a number
        to these. This is a horrible hack, but the Zotero API returns items
        with non-unique key values (arbitrary no. of contributors/authors/eds)
    """
    seen = []
    counter = 2
    for candidate in suspects:
        if candidate in seen:
            candidate = '%s_%s' % (candidate, counter)
            counter += 1
            seen.append(candidate)
        else:
            seen.append(candidate)
    return seen

class Zotero(object):
    """ Zotero API methods
        A full list of methods can be found here:
        http://www.zotero.org/support/dev/server_api
        Most of the methods return Atom feed documents, which can be parsed
        using feedparser (http://www.feedparser.org/docs/)

        See the readme for details of valid URL and request parameters
    """

    def __init__(self, user_id = None, user_key = None):
        """ Store Zotero credentials
        """
        self.endpoint = 'https://api.zotero.org'
        if user_id and user_key:
            self.user_id = user_id
            self.user_key = user_key
        else:
            raise ze.MissingCredentials, \
            'Please provide both the user ID and the user key'
        self.url_params = None

    def retrieve_data(self, request = None):
        """ Method for retrieving Zotero items via the API
            Returns a dict containing feed items and lists of entries
        """
        full_url = '%s%s' % (self.endpoint, request)
        try:
            data = urllib2.urlopen(full_url).read()
        except urllib2.HTTPError, error:
            if error.code == 401 or error.code == 403:
                raise ze.UserNotAuthorised, \
"You are not authorised to retrieve this resource (%s)" % error.code
            if error.code == 400:
                raise ze.RateLimitExceeded, \
"Invalid request, probably due to unsupported parameters: %s" % \
                data
            else:
                raise ze.HTTPError, \
                "HTTP Error %s (%s)" % (error.msg, error.code)
        # parse the result into Python data structures
        self.url_params = None
        return feedparser.parse(data)

    def retrieve(output_func):
        """ Decorator for Zotero methods; calls retrieve_data() and passes
            the result to the function specified by output_func
        """
        def wrap(func):
            """ Wrapper for original function
            """
            def wrapped_f(self, *args, **kwargs):
                """ Returns result of retrieve_data(), then output_func
                """
                orig_func = func(self, *args, **kwargs)
                retr = self.retrieve_data(orig_func)
                return eval(output_func)(retr)
            return wrapped_f
        return wrap

    def add_parameters(self, **params):
        """ Set URL parameters. Will always add the user key
        """
        if params:
            params['key'] = self.user_key
        else:
            params = {'key': self.user_key}
        params = urllib.urlencode(params)
        self.url_params = params

    def build_query(self, query_string, params = None):
        """ Set request parameters. Will always add the user ID if it hasn't
            been specifically set by an API call
        """
        params = {'u': self.user_id}
        try:
            query = query_string.format(**params)
        except KeyError, err:
            raise ze.ParamNotPassed, \
            'There\'s a request parameter missing: %s' % err
        # Add the URL parameters and the user key, if necessary
        if not self.url_params:
            self.add_parameters()
        query = '%s?%s' % (query, self.url_params)
        return query

    @retrieve('self.items_data')
    def items(self):
        """ Get user items
        """
        query_string = '/users/{u}/items'
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def top(self):
        """ Get user top-level items
        """
        query_string = '/users/{u}/items/top'
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def item(self, item):
        """ Get a specific item
        """
        query_string = '/users/{u}/items/{i}'.format(
        u = self.user_id, i = item)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def children(self, item):
        """ Get a specific item
        """
        query_string = '/users/{u}/items/{i}/children'.format(
        u = self.user_id,
        i = item)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def tag_items(self, tag):
        """ Get items for a specific tag
        """
        query_string = '/users/{u}/tags/{t}/items'.format(
        u = self.user_id,
        t = tag)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def collection_items(self, collection):
        """ Get a specific collection's items
        """
        query_string = '/users/{u}/collections/{c}/items'.format(
        u = self.user_id,
        c = collection)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_items(self, group):
        """ Get a specific group's items
        """
        query_string = '/groups/{g}/items'.format(
        g = group)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_top(self, group):
        """ Get a specific group's top-level items
        """
        query_string = '/groups/{g}/items/top'.format(
        g = group)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_item(self, group, item):
        """ Get a specific group item
        """
        query_string = '/groups/{g}/items/{i}'.format(
        g = group,
        i = item)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_item_children(self, group, item):
        """ Get a specific group item's child items
        """
        query_string = '/groups/{g}/items/{i}/children'.format(
        g = group,
        i = item)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_items_tag(self, group, tag):
        """ Get a specific group's items for a specific tag
        """
        query_string = '/groups/{g}/tags/{t}/items'.format(
        g = group,
        t = tag)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_collection_items(self, group, collection):
        """ Get a specific group's items for a specific collection
        """
        query_string = '/groups/{g}/collections/{c}/items'.format(
        g = group,
        c = collection)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_collection_top(self, group, collection):
        """ Get a specific group's top-level items for a specific collection
        """
        query_string = '/groups/{g}/collections/{c}/items/top'.format(
        g = group,
        c = collection)
        return self.build_query(query_string)

    @retrieve('self.items_data')
    def group_collection_item(self, group, collection, item):
        """ Get a specific collection's item from a specific group
        """
        query_string = '/groups/{g}/collections/{c}/items/{i}'.format(
        g = group,
        c = collection,
        i = item)
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def collections(self):
        """ Get user collections
        """
        query_string = '/users/{u}/collections'
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def collections_sub(self, collection):
        """ Get subcollections for a specific collection
        """
        query_string = '/users/{u}/collections/{c}/collections'.format(
        u = self.user_id,
        c = collection)
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def group_collections(self, group):
        """ Get collections for a specific group
        """
        query_string = '/groups/{group}/collections'.format(
        u = self.user_id,
        g = group)
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def group_collection(self, group, collection):
        """ Get collections for a specific group
        """
        query_string = '/groups/{g}/collections/{c}'.format(
        u = self.user_id,
        g = group,
        c = collection)
        return self.build_query(query_string)

    @retrieve('self.collections_data')
    def group_collection_sub(self, group, collection):
        """ Get collections for a specific group
        """
        query_string = '/groups/{g}/collections/{c}/collections'.format(
        u = self.user_id,
        g = group,
        c = collection)
        return self.build_query(query_string)

    @retrieve('self.groups_data')
    def groups(self):
        """ Get user groups
        """
        query_string = '/users/{u}/groups'
        return self.build_query(query_string)

    @retrieve('self.tags_data')
    def tags(self):
        """ Get tags for a specific item
        """
        query_string = '/users/{u}/tags'
        return self.build_query(query_string)

    @retrieve('self.tags_data')
    def item_tags(self, item):
        """ Get tags for a specific item
        """
        query_string = '/users/{u}/items/{i}/tags'.format(
        u = self.user_id,
        i = item)
        return self.build_query(query_string)

    @retrieve('self.tags_data')
    def group_tags(self, group):
        """ Get tags for a specific group
        """
        query_string = '/groups/{g}/tags'.format(
        u = self.user_id,
        g = group)
        return self.build_query(query_string)

    @retrieve('self.tags_data')
    def group_item_tags(self, group, item):
        """ Get tags for a specific group
        """
        query_string = '/groups/{g}/items/{i}/tags'.format(
        u = self.user_id,
        g = group,
        i = item)
        return self.build_query(query_string)

    def items_data(self, retrieved):
        """ Format and return data from API calls which return Items
        """
        # Shunt each 'content' block into a list
        item_parsed = [a['content'][0]['value'] for a in retrieved.entries]
        # Shunt each 'title' and item ID value into a list
        item_title = [t['title'] for t in retrieved.entries]
        item_id = [i['zapi_key'] for i in retrieved.entries]
        items = []
        for index, content in enumerate(item_parsed):
            # define a utf-8 parser with which to parse the content HTML
            utf = xml.XMLParser(encoding = 'utf-8')
            elem = xml.XML(content, utf)
            keys = [e.text.lower().encode('utf-8') for e in elem.iter('th')]
            keys = dedup(keys)
            values = [v.text.encode('utf-8') for v in elem.iter('td')]
            zipped = dict(zip(keys, values))
            zipped['title'] = item_title[index].encode('utf-8')
            zipped['id'] = item_id[index].encode('utf-8')
            items.append(Item(zipped))
        return items

    def collections_data(self, retrieved):
        """ Format and return data from API calls which return collections
        """
        collections = []
        collection_key = [k['zapi_key'] for k in retrieved.entries]
        collection_title = [t['title'] for t in retrieved.entries]
        collection_sub = [s['zapi_numcollections'] for s in retrieved.entries]
        for index in range(len(collection_key)):
            collection_data = {}
            collection_data['id'] = collection_key[index].encode('utf-8')
            collection_data['title'] = collection_title[index].encode('utf-8')
            if int(collection_sub[index]) > 0:
                collection_data['subcollections'] = int(collection_sub[index])
            collections.append(collection_data)
        return collections

    def groups_data(self, retrieved):
        """ Format and return data from API calls which return Groups
        """
        groups = []
        group_id = [t['title'] for t in retrieved.entries]
        group_items = [i['zapi_numitems'] for i in retrieved.entries]
        group_author = [a['author'] for a in retrieved.entries]
        for index in range(len(group_id)):
            group_data = {}
            group_data['id'] = group_id[index].encode('utf-8')
            group_data['total_items'] = group_items[index].encode('utf-8')
            group_data['owner'] = group_author[index].encode('utf-8')
            groups.append(group_data)
        return groups

    def tags_data(self, retrieved):
        """ Format and return data from API call which return Tags
        """
        tags = [t['title'].encode('utf-8') for t in retrieved.entries]
        return tags

class Item(object):
    """ Adds all retrieved values as instance properties by key, value.
        Currently, this class is little more than a container for what would
        otherwise be dicts (and I'm not convinced they aren't the way to go)
    """
    def __init__(self, values):
        super(Item, self).__init__()
        # Hackish and bad; creates instance attributes on the fly
        self.__dict__ = dict(values)

    def __getattr__(self, name):
        """ Only called if __getattribute__ fails, so we can return
            None without too much hand-wringing
        """
        return None

def main():
    """ main function
    """
    # Read a file from your cwd. Expects user id on line 1, key on line 2, LF
    auth_values = open_file(os.path.join(os.path.expanduser('~'),
    'zotero_keys.txt'))
    zot_id = auth_values[0]
    zot_key = auth_values[1]
    zot = Zotero(zot_id, zot_key)
    zot.add_parameters(limit=10, start=50)
    items = zot.top()
    for item in items:
        print item.title, item.id

if __name__ == "__main__":
    try:
        main()
    except (KeyboardInterrupt, SystemExit):
        # actually raise these, for a clean exit
        raise
    except Exception, errm:
        # all other exceptions: display the error
        print errm
        print "Stack trace:\n", traceback.print_exc(file = sys.stdout)
    else:
        pass
    finally:
        # exit cleanly once we've done everything else
        sys.exit(0)<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
zotero.py

Created by Stephan Hügel on 2011-02-28
Copyright Stephan Hügel, 2011

License: http://www.gnu.org/licenses/gpl-3.0.txt
"""

__author__ = 'urschrei@gmail.com'
__version__ = '0.6.8'

import urllib
import urllib2
import socket
import feedparser
import json

import zotero_errors as ze

# Avoid hanging the application if there's no server response
timeout = 30
socket.setdefaulttimeout(timeout)

class Zotero(object):
    """ Zotero API methods
        A full list of methods can be found here:
        http://www.zotero.org/support/dev/server_api
    """

    def __init__(self, user_id = None, user_key = None):
        """ Store Zotero credentials
        """
        self.endpoint = 'https://api.zotero.org'
        if user_id and user_key:
            self.user_id = user_id
            self.user_key = user_key
        else:
            raise ze.MissingCredentials, \
            'Please provide both the user ID and the user key'
        self.url_params = None

    def retrieve_data(self, request = None):
        """ Retrieve Zotero items via the API
            Combine endpoint and request to access the specific resource
            Returns a dict containing feed items and lists of entries
        """
        full_url = '%s%s' % (self.endpoint, request)
        req = urllib2.Request(full_url)
        req.add_header('User-Agent', 'Pyzotero/%s' % __version__)
        try:
            response = urllib2.urlopen(req)
            data = response.read()
        except (urllib2.HTTPError, urllib2.URLError), error:
            # Distinguish between URL errors and HTTP status codes of 400+
            if hasattr(error, 'reason'):
                raise ze.CouldNotReachURL, \
    "Could not reach server. Reason: %s\nURL: %s" % (error, full_url)
            elif hasattr(error, 'code'):
                if error.code == 401 or error.code == 403:
                    raise ze.UserNotAuthorised, \
    "You are not authorised to retrieve this resource (%s)" % error.code
                elif error.code == 400:
                    raise ze.UnsupportedParams, \
    "Invalid request, probably due to unsupported parameters: %s" % \
                    full_url
                elif error.code == 404:
                    raise ze.ResourceNotFound, \
    "No results for the following query:\n%s" % full_url
                else:
                    raise ze.HTTPError, \
    "HTTP Error %s (%s)\nURL: %s" % (
                    error.msg, error.code, full_url)
        # parse the result into Python data structures
        return data

    def retrieve(func):
        """ Decorator for Zotero methods; calls retrieve_data() and passes
            the result to the JSON processor
        """
        def wrapped_f(self, *args, **kwargs):
            """ Returns result of retrieve_data()

                orig_func's return value is part of a URI, and it's this
                which is intercepted and passed to retrieve_data:
                '/users/123/items?key=abc123'
                the feed-parsed atom doc returned by retrieve_data is then
                passed to process_content
            """
            orig_func = func(self, *args, **kwargs)
            retrieved = self.retrieve_data(orig_func)
            return self.process_content(feedparser.parse(retrieved))
        return wrapped_f

    def add_parameters(self, **params):
        """ Set URL parameters. Will always add the user key
        """
        self.url_params = None
        if params:
            params['key'] = self.user_key
        else:
            params = {'key': self.user_key}
        # always return json
        if not params.get('content', None) == 'bib' or None:
            params['content'] = 'json'
        params = urllib.urlencode(params)
        self.url_params = params

    def _build_query(self, query_string):
        """ Set request parameters. Will always add the user ID if it hasn't
            been specifically set by an API method
        """
        try:
            query = urllib.quote(query_string.format(u = self.user_id))
        except KeyError, err:
            raise ze.ParamNotPassed, \
            'There\'s a request parameter missing: %s' % err
        # Add the URL parameters and the user key, if necessary
        if not self.url_params:
            self.add_parameters()
        query = '%s?%s' % (query, self.url_params)
        return query

    # The following methods are all Zotero API calls
    @retrieve
    def items(self):
        """ Get user items
        """
        query_string = '/users/{u}/items'
        return self._build_query(query_string)

    @retrieve
    def top(self):
        """ Get user top-level items
        """
        query_string = '/users/{u}/items/top'
        return self._build_query(query_string)

    @retrieve
    def item(self, item):
        """ Get a specific item
        """
        query_string = '/users/{u}/items/{i}'.format(
        u = self.user_id, i = item)
        return self._build_query(query_string)

    @retrieve
    def children(self, item):
        """ Get a specific item's child items
        """
        query_string = '/users/{u}/items/{i}/children'.format(
        u = self.user_id,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def tag_items(self, tag):
        """ Get items for a specific tag
        """
        query_string = '/users/{u}/tags/{t}/items'.format(
        u = self.user_id,
        t = tag)
        return self._build_query(query_string)

    @retrieve
    def collection_items(self, collection):
        """ Get a specific collection's items
        """
        query_string = '/users/{u}/collections/{c}/items'.format(
        u = self.user_id,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_items(self, group):
        """ Get a specific group's items
        """
        query_string = '/groups/{g}/items'.format(
        g = group)
        return self._build_query(query_string)

    @retrieve
    def group_top(self, group):
        """ Get a specific group's top-level items
        """
        query_string = '/groups/{g}/items/top'.format(
        g = group)
        return self._build_query(query_string)

    @retrieve
    def group_item(self, group, item):
        """ Get a specific group item
        """
        query_string = '/groups/{g}/items/{i}'.format(
        g = group,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def group_item_children(self, group, item):
        """ Get a specific group item's child items
        """
        query_string = '/groups/{g}/items/{i}/children'.format(
        g = group,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def group_items_tag(self, group, tag):
        """ Get a specific group's items for a specific tag
        """
        query_string = '/groups/{g}/tags/{t}/items'.format(
        g = group,
        t = tag)
        return self._build_query(query_string)

    @retrieve
    def group_collection_items(self, group, collection):
        """ Get a specific group's items from a specific collection
        """
        query_string = '/groups/{g}/collections/{c}/items'.format(
        g = group,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_collection_top(self, group, collection):
        """ Get a specific group's top-level items from a specific collection
        """
        query_string = '/groups/{g}/collections/{c}/items/top'.format(
        g = group,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_collection_item(self, group, collection, item):
        """ Get a specific collection's item from a specific group
        """
        query_string = '/groups/{g}/collections/{c}/items/{i}'.format(
        g = group,
        c = collection,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def collections(self):
        """ Get user collections
        """
        query_string = '/users/{u}/collections'
        return self._build_query(query_string)

    @retrieve
    def collections_sub(self, collection):
        """ Get subcollections for a specific collection
        """
        query_string = '/users/{u}/collections/{c}/collections'.format(
        u = self.user_id,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_collections(self, group):
        """ Get collections for a specific group
        """
        query_string = '/groups/{group}/collections'.format(
        u = self.user_id,
        g = group)
        return self._build_query(query_string)

    @retrieve
    def group_collection(self, group, collection):
        """ Get a specific collection for a specific group
        """
        query_string = '/groups/{g}/collections/{c}'.format(
        u = self.user_id,
        g = group,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_collection_sub(self, group, collection):
        """ Get collections for a specific group
        """
        query_string = '/groups/{g}/collections/{c}/collections'.format(
        u = self.user_id,
        g = group,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def groups(self):
        """ Get user groups
        """
        query_string = '/users/{u}/groups'
        return self._build_query(query_string)

    @retrieve
    def tags(self):
        """ Get tags for a specific item
        """
        query_string = '/users/{u}/tags'
        return self._build_query(query_string)

    @retrieve
    def item_tags(self, item):
        """ Get tags for a specific item
        """
        query_string = '/users/{u}/items/{i}/tags'.format(
        u = self.user_id,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def group_tags(self, group):
        """ Get tags for a specific group
        """
        query_string = '/groups/{g}/tags'.format(
        u = self.user_id,
        g = group)
        return self._build_query(query_string)

    @retrieve
    def group_item_tags(self, group, item):
        """ Get tags for a specific item in a specific group
        """
        query_string = '/groups/{g}/items/{i}/tags'.format(
        u = self.user_id,
        g = group,
        i = item)
        return self._build_query(query_string)

    def item_types(self):
        """ Get all available item types
        """
        query_string = '/itemTypes'
        retrieved = self.retrieve_data(query_string)
        return json.loads(retrieved)

    def item_fields(self):
        """ Get all available item fields
        """
        query_string = '/itemFields'
        retrieved = self.retrieve_data(query_string)
        return json.loads(retrieved)

    def item_creator_types(self, itemtype):
        """ Get all available creator types for an item
        """
        query_string = '/itemTypeCreatorTypes?itemType={i}'.format(
        i = itemtype)
        retrieved = self.retrieve_data(query_string)
        return json.loads(retrieved)

    def item_template(self, itemtype):
        """ Get a template for a new item
        """
        query_string = '/items/new?itemType={i}'.format(
        i = itemtype)
        retrieved = self.retrieve_data(query_string)
        return json.loads(retrieved)

    # These methods process the returned data from API calls, returning lists
    def process_content(self, retrieved):
        """ Call either standard_items or bib_items, depending on the URL param
        """
        # Content request in 'bib' format, so call bib_items
        if self.url_params.find('=bib') != -1:
            return self.bib_items(retrieved)
        else:
            return self.standard_items(retrieved)

    def standard_items(self, retrieved):
        """ Format and return data from API calls which return Items
        """
        # send entries to tags_data if there's no JSON
        try:
            items = [json.loads(e['content'][0]['value'])
                for e in retrieved.entries]
        except ValueError, err:
            return self.tags_data(retrieved)
        # Try to get an item ID, and add it to the dict
        try:
            item_id = [i['zapi_key'] for i in retrieved.entries]
            for k, val in enumerate(items):
                val['id'] = item_id[k]
        except KeyError:
            pass
        # Try to get a group ID, and add it to the dict
        try:
            group_id = [g['links'][0]['href'].split('/')[-1]
                    for g in retrieved.entries]
            for k, val in enumerate(items):
                val['group_id'] = group_id[k]
        except KeyError:
            pass
        self.url_params = None
        return items

    def bib_items(self, retrieved):
        """ Return a list of strings formatted as HTML bibliography entries
        """
        items = []
        for bib in retrieved.entries:
            items.append(bib['content'][0]['value'])
        self.url_params = None
        return items

    def tags_data(self, retrieved):
        """ Format and return data from API calls which return Tags
        """
        tags = [t['title'] for t in retrieved.entries]
        self.url_params = None
        return tags
<commit_msg>Override feedparser's buggy _isBase64 method<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
zotero.py

Created by Stephan Hügel on 2011-02-28
Copyright Stephan Hügel, 2011

License: http://www.gnu.org/licenses/gpl-3.0.txt
"""

__author__ = 'urschrei@gmail.com'
__version__ = '0.6.8'

import urllib
import urllib2
import socket
import feedparser
import json

import zotero_errors as ze

# Avoid hanging the application if there's no server response
timeout = 30
socket.setdefaulttimeout(timeout)

def ib64_patched(self, attrsD, contentparams):
    """
    Patch isBase64 to prevent Base64 encoding of JSON content
    """
    if attrsD.get('mode', '') == 'base64':
        return 1
    if self.contentparams['type'].startswith('text/'):
        return 0
    if self.contentparams['type'].endswith('+xml'):
        return 0
    if self.contentparams['type'].endswith('/xml'):
        return 0
    if self.contentparams['type'].endswith('/json'):
        return 0
    return 1

# Override feedparser's buggy isBase64 method until they fix it
feedparser._FeedParserMixin._isBase64 = ib64_patched

class Zotero(object):
    """ Zotero API methods
        A full list of methods can be found here:
        http://www.zotero.org/support/dev/server_api
    """

    def __init__(self, user_id = None, user_key = None):
        """ Store Zotero credentials
        """
        self.endpoint = 'https://api.zotero.org'
        if user_id and user_key:
            self.user_id = user_id
            self.user_key = user_key
        else:
            raise ze.MissingCredentials, \
            'Please provide both the user ID and the user key'
        self.url_params = None

    def retrieve_data(self, request = None):
        """ Retrieve Zotero items via the API
            Combine endpoint and request to access the specific resource
            Returns a dict containing feed items and lists of entries
        """
        full_url = '%s%s' % (self.endpoint, request)
        req = urllib2.Request(full_url)
        req.add_header('User-Agent', 'Pyzotero/%s' % __version__)
        try:
            response = urllib2.urlopen(req)
            data = response.read()
        except (urllib2.HTTPError, urllib2.URLError), error:
            # Distinguish between URL errors and HTTP status codes of 400+
            if hasattr(error, 'reason'):
                raise ze.CouldNotReachURL, \
    "Could not reach server. Reason: %s\nURL: %s" % (error, full_url)
            elif hasattr(error, 'code'):
                if error.code == 401 or error.code == 403:
                    raise ze.UserNotAuthorised, \
    "You are not authorised to retrieve this resource (%s)" % error.code
                elif error.code == 400:
                    raise ze.UnsupportedParams, \
    "Invalid request, probably due to unsupported parameters: %s" % \
                    full_url
                elif error.code == 404:
                    raise ze.ResourceNotFound, \
    "No results for the following query:\n%s" % full_url
                else:
                    raise ze.HTTPError, \
    "HTTP Error %s (%s)\nURL: %s" % (
                    error.msg, error.code, full_url)
        # parse the result into Python data structures
        return data

    def retrieve(func):
        """ Decorator for Zotero methods; calls retrieve_data() and passes
            the result to the JSON processor
        """
        def wrapped_f(self, *args, **kwargs):
            """ Returns result of retrieve_data()

                orig_func's return value is part of a URI, and it's this
                which is intercepted and passed to retrieve_data:
                '/users/123/items?key=abc123'
                the feed-parsed atom doc returned by retrieve_data is then
                passed to process_content
            """
            orig_func = func(self, *args, **kwargs)
            retrieved = self.retrieve_data(orig_func)
            return self.process_content(feedparser.parse(retrieved))
        return wrapped_f

    def add_parameters(self, **params):
        """ Set URL parameters. Will always add the user key
        """
        self.url_params = None
        if params:
            params['key'] = self.user_key
        else:
            params = {'key': self.user_key}
        # always return json
        if not params.get('content', None) == 'bib' or None:
            params['content'] = 'json'
        params = urllib.urlencode(params)
        self.url_params = params

    def _build_query(self, query_string):
        """ Set request parameters. Will always add the user ID if it hasn't
            been specifically set by an API method
        """
        try:
            query = urllib.quote(query_string.format(u = self.user_id))
        except KeyError, err:
            raise ze.ParamNotPassed, \
            'There\'s a request parameter missing: %s' % err
        # Add the URL parameters and the user key, if necessary
        if not self.url_params:
            self.add_parameters()
        query = '%s?%s' % (query, self.url_params)
        return query

    # The following methods are all Zotero API calls
    @retrieve
    def items(self):
        """ Get user items
        """
        query_string = '/users/{u}/items'
        return self._build_query(query_string)

    @retrieve
    def top(self):
        """ Get user top-level items
        """
        query_string = '/users/{u}/items/top'
        return self._build_query(query_string)

    @retrieve
    def item(self, item):
        """ Get a specific item
        """
        query_string = '/users/{u}/items/{i}'.format(
        u = self.user_id, i = item)
        return self._build_query(query_string)

    @retrieve
    def children(self, item):
        """ Get a specific item's child items
        """
        query_string = '/users/{u}/items/{i}/children'.format(
        u = self.user_id,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def tag_items(self, tag):
        """ Get items for a specific tag
        """
        query_string = '/users/{u}/tags/{t}/items'.format(
        u = self.user_id,
        t = tag)
        return self._build_query(query_string)

    @retrieve
    def collection_items(self, collection):
        """ Get a specific collection's items
        """
        query_string = '/users/{u}/collections/{c}/items'.format(
        u = self.user_id,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_items(self, group):
        """ Get a specific group's items
        """
        query_string = '/groups/{g}/items'.format(
        g = group)
        return self._build_query(query_string)

    @retrieve
    def group_top(self, group):
        """ Get a specific group's top-level items
        """
        query_string = '/groups/{g}/items/top'.format(
        g = group)
        return self._build_query(query_string)

    @retrieve
    def group_item(self, group, item):
        """ Get a specific group item
        """
        query_string = '/groups/{g}/items/{i}'.format(
        g = group,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def group_item_children(self, group, item):
        """ Get a specific group item's child items
        """
        query_string = '/groups/{g}/items/{i}/children'.format(
        g = group,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def group_items_tag(self, group, tag):
        """ Get a specific group's items for a specific tag
        """
        query_string = '/groups/{g}/tags/{t}/items'.format(
        g = group,
        t = tag)
        return self._build_query(query_string)

    @retrieve
    def group_collection_items(self, group, collection):
        """ Get a specific group's items from a specific collection
        """
        query_string = '/groups/{g}/collections/{c}/items'.format(
        g = group,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_collection_top(self, group, collection):
        """ Get a specific group's top-level items from a specific collection
        """
        query_string = '/groups/{g}/collections/{c}/items/top'.format(
        g = group,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_collection_item(self, group, collection, item):
        """ Get a specific collection's item from a specific group
        """
        query_string = '/groups/{g}/collections/{c}/items/{i}'.format(
        g = group,
        c = collection,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def collections(self):
        """ Get user collections
        """
        query_string = '/users/{u}/collections'
        return self._build_query(query_string)

    @retrieve
    def collections_sub(self, collection):
        """ Get subcollections for a specific collection
        """
        query_string = '/users/{u}/collections/{c}/collections'.format(
        u = self.user_id,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_collections(self, group):
        """ Get collections for a specific group
        """
        query_string = '/groups/{group}/collections'.format(
        u = self.user_id,
        g = group)
        return self._build_query(query_string)

    @retrieve
    def group_collection(self, group, collection):
        """ Get a specific collection for a specific group
        """
        query_string = '/groups/{g}/collections/{c}'.format(
        u = self.user_id,
        g = group,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def group_collection_sub(self, group, collection):
        """ Get collections for a specific group
        """
        query_string = '/groups/{g}/collections/{c}/collections'.format(
        u = self.user_id,
        g = group,
        c = collection)
        return self._build_query(query_string)

    @retrieve
    def groups(self):
        """ Get user groups
        """
        query_string = '/users/{u}/groups'
        return self._build_query(query_string)

    @retrieve
    def tags(self):
        """ Get tags for a specific item
        """
        query_string = '/users/{u}/tags'
        return self._build_query(query_string)

    @retrieve
    def item_tags(self, item):
        """ Get tags for a specific item
        """
        query_string = '/users/{u}/items/{i}/tags'.format(
        u = self.user_id,
        i = item)
        return self._build_query(query_string)

    @retrieve
    def group_tags(self, group):
        """ Get tags for a specific group
        """
        query_string = '/groups/{g}/tags'.format(
        u = self.user_id,
        g = group)
        return self._build_query(query_string)

    @retrieve
    def group_item_tags(self, group, item):
        """ Get tags for a specific item in a specific group
        """
        query_string = '/groups/{g}/items/{i}/tags'.format(
        u = self.user_id,
        g = group,
        i = item)
        return self._build_query(query_string)

    def item_types(self):
        """ Get all available item types
        """
        query_string = '/itemTypes'
        retrieved = self.retrieve_data(query_string)
        return json.loads(retrieved)

    def item_fields(self):
        """ Get all available item fields
        """
        query_string = '/itemFields'
        retrieved = self.retrieve_data(query_string)
        return json.loads(retrieved)

    def item_creator_types(self, itemtype):
        """ Get all available creator types for an item
        """
        query_string = '/itemTypeCreatorTypes?itemType={i}'.format(
        i = itemtype)
        retrieved = self.retrieve_data(query_string)
        return json.loads(retrieved)

    def item_template(self, itemtype):
        """ Get a template for a new item
        """
        query_string = '/items/new?itemType={i}'.format(
        i = itemtype)
        retrieved = self.retrieve_data(query_string)
        return json.loads(retrieved)

    # These methods process the returned data from API calls, returning lists
    def process_content(self, retrieved):
        """ Call either standard_items or bib_items, depending on the URL param
        """
        # Content request in 'bib' format, so call bib_items
        if self.url_params.find('=bib') != -1:
            return self.bib_items(retrieved)
        else:
            return self.standard_items(retrieved)

    def standard_items(self, retrieved):
        """ Format and return data from API calls which return Items
        """
        # send entries to tags_data if there's no JSON
        try:
            items = [json.loads(e['content'][0]['value'])
                for e in retrieved.entries]
        except ValueError, err:
            return self.tags_data(retrieved)
        # Try to get an item ID, and add it to the dict
        try:
            item_id = [i['zapi_key'] for i in retrieved.entries]
            for k, val in enumerate(items):
                val['id'] = item_id[k]
        except KeyError:
            pass
        # Try to get a group ID, and add it to the dict
        try:
            group_id = [g['links'][0]['href'].split('/')[-1]
                    for g in retrieved.entries]
            for k, val in enumerate(items):
                val['group_id'] = group_id[k]
        except KeyError:
            pass
        self.url_params = None
        return items

    def bib_items(self, retrieved):
        """ Return a list of strings formatted as HTML bibliography entries
        """
        items = []
        for bib in retrieved.entries:
            items.append(bib['content'][0]['value'])
        self.url_params = None
        return items

    def tags_data(self, retrieved):
        """ Format and return data from API calls which return Tags
        """
        tags = [t['title'] for t in retrieved.entries]
        self.url_params = None
        return tags
<|endoftext|>


<commit_before>import unittest

import cupy
from cupy import testing
import cupyx.scipy.special  # NOQA

class _TestBase(object):

    def test_erf(self):
        self.check_unary('erf')

    def test_erfc(self):
        self.check_unary('erfc')

    def test_erfcx(self):
        self.check_unary('erfcx')

    def test_erfinv(self):
        self.check_unary('erfinv')

    def test_erfcinv(self):
        self.check_unary('erfcinv')

@testing.gpu
@testing.with_requires('scipy')
class TestSpecial(unittest.TestCase, _TestBase):

    @testing.for_dtypes(['e', 'f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
    def check_unary(self, name, xp, scp, dtype):
        import scipy.special  # NOQA

        a = testing.shaped_arange((2, 3), xp, dtype)
        return getattr(scp.special, name)(a)

@testing.gpu
@testing.with_requires('scipy')
class TestFusionSpecial(unittest.TestCase, _TestBase):

    @testing.for_dtypes(['e', 'f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
    def check_unary(self, name, xp, scp, dtype):
        import scipy.special  # NOQA

        a = testing.shaped_arange((2, 3), xp, dtype)

        @cupy.fuse()
        def f(x):
            return getattr(scp.special, name)(x)

        return f(a)
<commit_msg>Add tests for erfinv and erfcinv<commit_after>import unittest

import cupy
from cupy import testing
import cupyx.scipy.special  # NOQA

class _TestBase(object):

    def test_erf(self):
        self.check_unary('erf')

    def test_erfc(self):
        self.check_unary('erfc')

    def test_erfcx(self):
        self.check_unary('erfcx')

    def test_erfinv(self):
        self.check_unary('erfinv')
        self.check_unary_random('erfinv', scale=2, offset=-1)
        self.check_unary_boundary('erfinv', boundary=-1)
        self.check_unary_boundary('erfinv', boundary=1)

    def test_erfcinv(self):
        self.check_unary('erfcinv')
        self.check_unary_random('erfcinv', scale=2, offset=0)
        self.check_unary_boundary('erfcinv', boundary=0)
        self.check_unary_boundary('erfcinv', boundary=2)

@testing.gpu
@testing.with_requires('scipy')
class TestSpecial(unittest.TestCase, _TestBase):

    @testing.for_dtypes(['e', 'f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
    def check_unary(self, name, xp, scp, dtype):
        import scipy.special  # NOQA

        a = testing.shaped_arange((2, 3), xp, dtype)
        return getattr(scp.special, name)(a)

    @testing.for_dtypes(['f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
    def check_unary_random(self, name, xp, scp, dtype, scale, offset):
        import scipy.special  # NOQA

        a = testing.shaped_random((2, 3), xp, dtype, scale=scale) + offset
        return getattr(scp.special, name)(a)

    @testing.for_dtypes(['f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
    def check_unary_boundary(self, name, xp, scp, dtype, boundary):
        import scipy.special  # NOQA

        x = boundary * (1 - 1.0 / 1024)
        y = boundary
        z = boundary * (1 + 1.0 / 1024)
        a = xp.array([x, y, z], dtype=dtype)
        return getattr(scp.special, name)(a)

@testing.gpu
@testing.with_requires('scipy')
class TestFusionSpecial(unittest.TestCase, _TestBase):

    @testing.for_dtypes(['e', 'f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
    def check_unary(self, name, xp, scp, dtype):
        import scipy.special  # NOQA

        a = testing.shaped_arange((2, 3), xp, dtype)

        @cupy.fuse()
        def f(x):
            return getattr(scp.special, name)(x)

        return f(a)

    @testing.for_dtypes(['f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
    def check_unary_random(self, name, xp, scp, dtype, scale, offset):
        import scipy.special  # NOQA

        a = testing.shaped_random((2, 3), xp, dtype, scale=scale) + offset

        @cupy.fuse()
        def f(x):
            return getattr(scp.special, name)(x)

        return f(a)

    @testing.for_dtypes(['f', 'd'])
    @testing.numpy_cupy_allclose(atol=1e-5, scipy_name='scp')
    def check_unary_boundary(self, name, xp, scp, dtype, boundary):
        import scipy.special  # NOQA

        x = boundary * (1 - 1.0 / 1024)
        y = boundary
        z = boundary * (1 + 1.0 / 1024)
        a = xp.array([x, y, z], dtype=dtype)

        @cupy.fuse()
        def f(x):
            return getattr(scp.special, name)(x)

        return f(a)
<|endoftext|>


<commit_before>import os
import time

import pandas as pd

from .datasource import Datasource
from .filesystem_path_generator import FilesystemPathGenerator
from ...dataset.pandas_dataset import PandasDataset

from ...exceptions import BatchKwargsError

class PandasCSVDatasource(Datasource):
    """
    A PandasDataSource makes it easy to create, manage and validate expectations on
    Pandas dataframes.

    Use with the FilesystemPathGenerator for simple cases.
    """

    def __init__(self, name="default", data_context=None, generators=None, read_csv_kwargs=None, **kwargs):
        if generators is None:
            # Provide a gentle way to build a datasource with a sane default, including ability to specify the base_directory
            base_directory = kwargs.pop("base_directory", "/data")
            generators = {
                "default": {"type": "filesystem", "base_directory": base_directory}
        }
        super(PandasCSVDatasource, self).__init__(name, type_="pandas", data_context=data_context, generators=generators)
        self._datasource_config.update(
            {
                "read_csv_kwargs": read_csv_kwargs or {}
            }
        )
        self._build_generators()

    def _get_generator_class(self, type_):
        if type_ == "filesystem":
            return FilesystemPathGenerator
        else:
            raise ValueError("Unrecognized BatchGenerator type %s" % type_)

    def _get_data_asset(self, data_asset_name, batch_kwargs, expectations_config, **kwargs):
        try:
            full_path = os.path.join(batch_kwargs["path"])
        except KeyError:
            raise BatchKwargsError("Invalid batch_kwargs: path is required for a PandasCSVDatasource", batch_kwargs)

        all_kwargs = dict(**self._datasource_config["read_csv_kwargs"])
        all_kwargs.update(**kwargs)

        df = pd.read_csv(full_path, all_kwargs)

        return PandasDataset(df,
            expectations_config=expectations_config,
            data_context=self._data_context,
            data_asset_name=data_asset_name,
            batch_kwargs=batch_kwargs)

    def build_batch_kwargs(self, filepath, **kwargs):
        batch_kwargs = {
            "path": filepath,
            "timestamp": time.time()
        }
        batch_kwargs.update(dict(**kwargs))
        return batch_kwargs<commit_msg>Fix pd.read_csv kwargs passing<commit_after>import os
import time

import pandas as pd

from .datasource import Datasource
from .filesystem_path_generator import FilesystemPathGenerator
from ...dataset.pandas_dataset import PandasDataset

from ...exceptions import BatchKwargsError

class PandasCSVDatasource(Datasource):
    """
    A PandasDataSource makes it easy to create, manage and validate expectations on
    Pandas dataframes.

    Use with the FilesystemPathGenerator for simple cases.
    """

    def __init__(self, name="default", data_context=None, generators=None, read_csv_kwargs=None, **kwargs):
        if generators is None:
            # Provide a gentle way to build a datasource with a sane default, including ability to specify the base_directory
            base_directory = kwargs.pop("base_directory", "/data")
            generators = {
                "default": {"type": "filesystem", "base_directory": base_directory}
        }
        super(PandasCSVDatasource, self).__init__(name, type_="pandas", data_context=data_context, generators=generators)
        self._datasource_config.update(
            {
                "read_csv_kwargs": read_csv_kwargs or {}
            }
        )
        self._build_generators()

    def _get_generator_class(self, type_):
        if type_ == "filesystem":
            return FilesystemPathGenerator
        else:
            raise ValueError("Unrecognized BatchGenerator type %s" % type_)

    def _get_data_asset(self, data_asset_name, batch_kwargs, expectations_config, **kwargs):
        try:
            full_path = os.path.join(batch_kwargs["path"])
        except KeyError:
            raise BatchKwargsError("Invalid batch_kwargs: path is required for a PandasCSVDatasource", batch_kwargs)

        all_kwargs = dict(**self._datasource_config["read_csv_kwargs"])
        all_kwargs.update(**kwargs)

        df = pd.read_csv(full_path, **all_kwargs)

        return PandasDataset(df,
            expectations_config=expectations_config,
            data_context=self._data_context,
            data_asset_name=data_asset_name,
            batch_kwargs=batch_kwargs)

    def build_batch_kwargs(self, filepath, **kwargs):
        batch_kwargs = {
            "path": filepath,
            "timestamp": time.time()
        }
        batch_kwargs.update(dict(**kwargs))
        return batch_kwargs<|endoftext|>


<commit_before>from hazelcast.lifecycle import LifecycleState
from tests.base import HazelcastTestCase
from tests.util import event_collector

class LifecycleTest(HazelcastTestCase):
    rc = None

    def setUp(self):
        self.rc = self.create_rc()
        self.cluster = self.create_cluster(self.rc)

    def tearDown(self):
        self.shutdown_all_clients()
        self.rc.exit()

    def test_lifecycle_listener_receives_events_in_order(self):
        collector = event_collector()
        self.cluster.start_member()
        client = self.create_client(
            {
                "cluster_name": self.cluster.id,
                "lifecycle_listeners": [
                    collector,
                ],
            }
        )
        client.shutdown()

        self.assertEqual(
            collector.events,
            [
                LifecycleState.STARTING,
                LifecycleState.STARTED,
                LifecycleState.CONNECTED,
                LifecycleState.SHUTTING_DOWN,
                LifecycleState.DISCONNECTED,
                LifecycleState.SHUTDOWN,
            ],
        )

    def test_lifecycle_listener_receives_events_in_order_after_startup(self):
        self.cluster.start_member()

        collector = event_collector()
        client = self.create_client(
            {
                "cluster_name": self.cluster.id,
            }
        )
        client.lifecycle_service.add_listener(collector)
        client.shutdown()

        self.assertEqual(
            collector.events,
            [LifecycleState.SHUTTING_DOWN, LifecycleState.DISCONNECTED, LifecycleState.SHUTDOWN],
        )

    def test_lifecycle_listener_receives_disconnected_event(self):
        member = self.cluster.start_member()

        collector = event_collector()
        client = self.create_client(
            {
                "cluster_name": self.cluster.id,
            }
        )
        client.lifecycle_service.add_listener(collector)
        member.shutdown()
        self.assertEqual(collector.events, [LifecycleState.DISCONNECTED])
        client.shutdown()

    def test_remove_lifecycle_listener(self):
        collector = event_collector()

        self.cluster.start_member()
        client = self.create_client(
            {
                "cluster_name": self.cluster.id,
            }
        )
        registration_id = client.lifecycle_service.add_listener(collector)
        client.lifecycle_service.remove_listener(registration_id)
        client.shutdown()

        self.assertEqual(collector.events, [])

    def test_exception_in_listener(self):
        def listener(_):
            raise RuntimeError("error")

        self.cluster.start_member()
        self.create_client(
            {
                "cluster_name": self.cluster.id,
                "lifecycle_listeners": [
                    listener,
                ],
            }
        )
<commit_msg>Fix `test_lifecycle_listener_receives_disconnected_event` (#417)<commit_after>from hazelcast.lifecycle import LifecycleState
from tests.base import HazelcastTestCase
from tests.util import event_collector

class LifecycleTest(HazelcastTestCase):
    rc = None

    def setUp(self):
        self.rc = self.create_rc()
        self.cluster = self.create_cluster(self.rc)

    def tearDown(self):
        self.shutdown_all_clients()
        self.rc.exit()

    def test_lifecycle_listener_receives_events_in_order(self):
        collector = event_collector()
        self.cluster.start_member()
        client = self.create_client(
            {
                "cluster_name": self.cluster.id,
                "lifecycle_listeners": [
                    collector,
                ],
            }
        )
        client.shutdown()

        self.assertEqual(
            collector.events,
            [
                LifecycleState.STARTING,
                LifecycleState.STARTED,
                LifecycleState.CONNECTED,
                LifecycleState.SHUTTING_DOWN,
                LifecycleState.DISCONNECTED,
                LifecycleState.SHUTDOWN,
            ],
        )

    def test_lifecycle_listener_receives_events_in_order_after_startup(self):
        self.cluster.start_member()

        collector = event_collector()
        client = self.create_client(
            {
                "cluster_name": self.cluster.id,
            }
        )
        client.lifecycle_service.add_listener(collector)
        client.shutdown()

        self.assertEqual(
            collector.events,
            [LifecycleState.SHUTTING_DOWN, LifecycleState.DISCONNECTED, LifecycleState.SHUTDOWN],
        )

    def test_lifecycle_listener_receives_disconnected_event(self):
        member = self.cluster.start_member()

        collector = event_collector()
        client = self.create_client(
            {
                "cluster_name": self.cluster.id,
            }
        )
        client.lifecycle_service.add_listener(collector)
        member.shutdown()

        def assertion():
            self.assertEqual(collector.events, [LifecycleState.DISCONNECTED])

        self.assertTrueEventually(assertion)

        client.shutdown()

    def test_remove_lifecycle_listener(self):
        collector = event_collector()

        self.cluster.start_member()
        client = self.create_client(
            {
                "cluster_name": self.cluster.id,
            }
        )
        registration_id = client.lifecycle_service.add_listener(collector)
        client.lifecycle_service.remove_listener(registration_id)
        client.shutdown()

        self.assertEqual(collector.events, [])

    def test_exception_in_listener(self):
        def listener(_):
            raise RuntimeError("error")

        self.cluster.start_member()
        self.create_client(
            {
                "cluster_name": self.cluster.id,
                "lifecycle_listeners": [
                    listener,
                ],
            }
        )
<|endoftext|>


<commit_before>import atexit
import threading

import numpy
import six

import cupy
from cupy.core import internal
from cupy import cuda
from cupy.cuda import cudnn

_cudnn_version = cudnn.getVersion()
_thread_local = threading.local()

_handles = {}

def get_handle():
    dev = cuda.get_device_id()
    if dev in _handles:
        return _handles[dev]
    handle = cudnn.create()
    _handles[dev] = handle
    return handle

@atexit.register
def reset_handles():
    global _handles
    handles = _handles
    _handles = {}

    for handle in six.itervalues(handles):
        cudnn.destroy(handle)

def _get_nd_tensor_cache():
    if not hasattr(_thread_local, 'cudnn_nd_tensor_cache'):
        _thread_local.cudnn_nd_tensor_cache = {}
    return _thread_local.cudnn_nd_tensor_cache

class Descriptor(object):

    def __init__(self, descriptor, destroyer):
        self.value = descriptor
        self.destroy = destroyer

    def __del__(self):
        if self.value:
            self.destroy(self.value)
            self.value = None

def get_data_type(dtype):
    if dtype.type == numpy.float32:
        return cudnn.CUDNN_DATA_FLOAT
    elif dtype.type == numpy.float64:
        return cudnn.CUDNN_DATA_DOUBLE
    elif dtype.type == numpy.float16:
        return cudnn.CUDNN_DATA_HALF
    else:
        raise TypeError('Dtype {} is not supported in cuDNN'.format(dtype))

def _to_ctypes_array(tup, dtype=numpy.intc):
    return numpy.array(tup, dtype=dtype).ctypes

def create_tensor_descriptor(arr, format=cudnn.CUDNN_TENSOR_NCHW):
    desc = Descriptor(cudnn.createTensorDescriptor(),
                      cudnn.destroyTensorDescriptor)
    if not arr.flags.c_contiguous:
        raise ValueError('cupy.cudnn supports c-contiguous arrays only')
    data_type = get_data_type(arr.dtype)
    if arr.ndim == 4:
        cudnn.setTensor4dDescriptor(desc.value, format, data_type, *arr.shape)
    else:
        strides = [s // arr.itemsize for s in arr.strides]
        c_shape = _to_ctypes_array(arr.shape)
        c_strides = _to_ctypes_array(strides)
        cudnn.setTensorNdDescriptor(desc.value, data_type, arr.ndim,
                                    c_shape.data, c_strides.data)
    return desc

def create_uninitialized_tensor_descriptor():
    """Create uninitialized tensor descriptor.

    Create a cudnnCreateTensorDescriptor_t that is not yet initialized.
    This is used by the batch normalization functions.
    """
    desc = Descriptor(cudnn.createTensorDescriptor(),
                      cudnn.destroyTensorDescriptor)
    return desc

def create_tensor_nd_descriptor(arr):
    desc = Descriptor(cudnn.createTensorDescriptor(),
                      cudnn.destroyTensorDescriptor)
    if not arr.flags.c_contiguous:
        raise ValueError('cupy.cudnn supports c-contiguous arrays only')
    data_type = get_data_type(arr.dtype)
    shape = arr.shape
    key = (data_type, shape)
    cache = _get_nd_tensor_cache()
    if key in cache:
        return cache[key]

    # numpy's stride is defined in bytes, but cudnn's stride is defined in
    # size of element
    strides = [s // arr.itemsize for s in arr.strides]

    c_shape = _to_ctypes_array(shape)
    c_strides = _to_ctypes_array(strides)
    cudnn.setTensorNdDescriptor(desc.value, data_type,
                                arr.ndim, c_shape.data, c_strides.data)
    cache = _get_nd_tensor_cache()
    cache[key] = desc
    return desc

def create_filter_descriptor(arr, format=cudnn.CUDNN_TENSOR_NCHW):
    desc = Descriptor(cudnn.createFilterDescriptor(),
                      cudnn.destroyFilterDescriptor)
    data_type = get_data_type(arr.dtype)
    if _cudnn_version >= 4000:
        if arr.ndim == 4:
            cudnn.setFilter4dDescriptor_v4(desc.value, data_type, format,
                                           *arr.shape)
        else:
            c_shape = _to_ctypes_array(arr.shape)
            cudnn.setFilterNdDescriptor_v4(desc.value, data_type, format,
                                           arr.ndim, c_shape.data)
    else:
        if arr.ndim == 4:
            cudnn.setFilter4dDescriptor_v3(desc.value, data_type, *arr.shape)
        else:
            c_shape = _to_ctypes_array(arr.shape)
            cudnn.setFilterNdDescriptor_v3(desc.value, data_type, arr.ndim,
                                           c_shape.data)

    return desc

def create_convolution_descriptor(pad, stride, dtype, dilation=(1, 1),
                                  mode=cudnn.CUDNN_CROSS_CORRELATION):
    desc = Descriptor(cudnn.createConvolutionDescriptor(),
                      cudnn.destroyConvolutionDescriptor)
    ndim = len(pad)
    if ndim != len(stride):
        raise ValueError('pad and stride must be of same length')

    if ndim == 2:
        if _cudnn_version < 6000:
            if dilation[0] != 1 or dilation[1] != 1:
                raise ValueError('dilation must be one when cudnn < 6.0')
        if _cudnn_version >= 5000:
            data_type = get_data_type(dtype)
            # TODO(takagi) Temporarily use computing precision of FP32 for
            #     storing precision of FP16.
            if dtype == numpy.float16:
                data_type = cudnn.CUDNN_DATA_FLOAT
            cudnn.setConvolution2dDescriptor_v5(
                desc.value, pad[0], pad[1], stride[0], stride[1],
                dilation[0], dilation[1], mode, data_type)
        else:
            cudnn.setConvolution2dDescriptor_v4(
                desc.value, pad[0], pad[1], stride[0], stride[1], 1, 1, mode)
    else:
        c_pad = _to_ctypes_array(pad)
        c_stride = _to_ctypes_array(stride)
        c_dilation = _to_ctypes_array((1,) * ndim)
        if _cudnn_version >= 3000:
            data_type = get_data_type(dtype)
            # TODO(takagi) Temporarily use computing precision of FP32 for
            #     storing precision of FP16.
            if dtype == numpy.float16:
                data_type = cudnn.CUDNN_DATA_FLOAT
            cudnn.setConvolutionNdDescriptor_v3(
                desc.value, ndim, c_pad.data, c_stride.data, c_dilation.data,
                mode, data_type)
        else:
            cudnn.setConvolutionNdDescriptor_v2(
                desc.value, ndim, c_pad.data, c_stride.data, c_dilation.data,
                mode)

    return desc

def create_pooling_descriptor(ksize, stride, pad, mode):
    desc = Descriptor(cudnn.createPoolingDescriptor(),
                      cudnn.destroyPoolingDescriptor)
    ndim = len(ksize)
    if ndim != len(stride) or ndim != len(pad):
        raise ValueError('ksize, stride, and pad must be of same length')

    if ndim == 2:
        if _cudnn_version >= 4000:
            cudnn.setPooling2dDescriptor_v4(
                desc.value, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, ksize[0],
                ksize[1], pad[0], pad[1], stride[0], stride[1])
        else:
            cudnn.setPooling2dDescriptor_v3(
                desc.value, mode, ksize[0], ksize[1], pad[0], pad[1],
                stride[0], stride[1])
    else:
        c_ksize = _to_ctypes_array(ksize)
        c_pad = _to_ctypes_array(pad)
        c_stride = _to_ctypes_array(stride)
        if _cudnn_version >= 4000:
            cudnn.setPoolingNdDescriptor_v4(
                desc.value, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, ndim,
                c_ksize.data, c_pad.data, c_stride.data)
        else:
            cudnn.setPoolingNdDescriptor_v3(
                desc.value, mode, ndim, c_ksize.data, c_pad.data,
                c_stride.data)

    return desc

def _as4darray(arr):
    if arr.ndim == 0:
        return arr.reshape(1, 1, 1, 1)
    else:
        return arr.reshape(arr.shape[0], -1, 1, 1)

def activation_forward(x, mode):
    x = cupy.ascontiguousarray(x)
    y = cupy.empty_like(x)

    dtype = 'd' if x.dtype == 'd' else 'f'
    one = numpy.array(1, dtype=dtype).ctypes
    zero = numpy.array(0, dtype=dtype).ctypes
    handle = get_handle()
    x_mat = _as4darray(x)
    desc = create_tensor_descriptor(x_mat)
    if _cudnn_version >= 4000:
        act_desc = Descriptor(cudnn.createActivationDescriptor(),
                              cudnn.destroyActivationDescriptor)
        cudnn.setActivationDescriptor(
            act_desc.value, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, 0.0)
        cudnn.activationForward_v4(
            handle, act_desc.value, one.data, desc.value, x_mat.data.ptr,
            zero.data, desc.value, y.data.ptr)
    else:
        cudnn.activationForward_v3(
            handle, mode, one.data, desc.value, x_mat.data.ptr,
            zero.data, desc.value, y.data.ptr)
    return y

def activation_backward(x, y, gy, mode):
    x = cupy.ascontiguousarray(x)
    gy = cupy.ascontiguousarray(gy)

    gx = cupy.empty_like(x)
    dtype = 'd' if x.dtype == 'd' else 'f'
    one = numpy.array(1, dtype=dtype).ctypes
    zero = numpy.array(0, dtype=dtype).ctypes
    handle = get_handle()
    y_mat = _as4darray(y)
    desc = create_tensor_descriptor(y_mat)
    if _cudnn_version >= 4000:
        act_desc = Descriptor(cudnn.createActivationDescriptor(),
                              cudnn.destroyActivationDescriptor)
        cudnn.setActivationDescriptor(
            act_desc.value, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, 0.0)
        cudnn.activationBackward_v4(
            handle, act_desc.value, one.data, desc.value, y.data.ptr,
            desc.value, gy.data.ptr, desc.value, x.data.ptr,
            zero.data, desc.value, gx.data.ptr)
    else:
        cudnn.activationBackward_v3(
            handle, mode, one.data, desc.value, y.data.ptr,
            desc.value, gy.data.ptr, desc.value, x.data.ptr,
            zero.data, desc.value, gx.data.ptr)
    return gx

def create_dropout_descriptor(
        handle, dropout, states, state_size_in_bytes, seed):
    desc = Descriptor(cudnn.createDropoutDescriptor(),
                      cudnn.destroyDropoutDescriptor)
    cudnn.setDropoutDescriptor(desc.value, handle, dropout,
                               states, state_size_in_bytes, seed)
    return desc

def set_dropout_descriptor(desc, handle, dropout):
    # When the fourth argument is NULL, random state is not updated.
    cudnn.setDropoutDescriptor(desc.value, handle, dropout, 0, 0, 0)

def create_rnn_descriptor(hidden_size, num_layers, dropout_desc,
                          input_mode, direction, mode, data_type):
    desc = Descriptor(cudnn.createRNNDescriptor(),
                      cudnn.destroyRNNDescriptor)
    cudnn.setRNNDescriptor(
        desc.value, hidden_size, num_layers, dropout_desc.value,
        input_mode, direction, mode, data_type)
    return desc

def get_rnn_lin_layer_matrix_params(
        handle, rnn_desc, layer, x_desc, w_desc, w, lin_layer_id):
    mat_desc = Descriptor(cudnn.createFilterDescriptor(),
                          cudnn.destroyFilterDescriptor)
    ptr = numpy.array(0, dtype=numpy.intp)
    cudnn.getRNNLinLayerMatrixParams(
        handle, rnn_desc.value, layer, x_desc.value, w_desc.value, w.data.ptr,
        lin_layer_id, mat_desc.value, ptr.ctypes.data)
    offset = (ptr - w.data.ptr) // 4
    _, _, _, dim = cudnn.getFilterNdDescriptor(mat_desc.value, 3)
    size = internal.prod(dim)
    mat = w[offset: offset + size]
    return mat

def get_rnn_lin_layer_bias_params(
        handle, rnn_desc, layer, x_desc, w_desc, w, lin_layer_id):
    bias_desc = Descriptor(cudnn.createFilterDescriptor(),
                           cudnn.destroyFilterDescriptor)
    ptr = numpy.array(0, dtype=numpy.intp)
    cudnn.getRNNLinLayerBiasParams(
        handle, rnn_desc.value, layer, x_desc.value, w_desc.value, w.data.ptr,
        lin_layer_id, bias_desc.value, ptr.ctypes.data)
    offset = (ptr - w.data.ptr) // 4
    _, _, _, dim = cudnn.getFilterNdDescriptor(bias_desc.value, 3)
    size = internal.prod(dim)
    bias = w[offset: offset + size]
    return bias

def create_dropout_states(handle):
    state_size = cudnn.dropoutGetStatesSize(handle)
    return cupy.empty((state_size,), dtype='b')

def create_spatial_transformer_descriptor(sampler_type, dtype, nb_dims, dim_A):
    desc = Descriptor(cudnn.createSpatialTransformerDescriptor(),
                      cudnn.destroySpatialTransformerDescriptor)
    data_type = get_data_type(dtype)

    cudnn.setSpatialTransformerDescriptor(
        desc.value, sampler_type, data_type, nb_dims, dim_A)
    return desc

if _cudnn_version >= 3000:
    def add_tensor(handle, alpha, biasDesc, biasData, beta, srcDestDesc,
                   srcDestData):
        cudnn.addTensor_v3(handle, alpha, biasDesc,
                           biasData, beta, srcDestDesc, srcDestData)
else:
    def add_tensor(handle, alpha, biasDesc, biasData, beta, srcDestDesc,
                   srcDestData):
        cudnn.addTensor_v2(handle, cudnn.CUDNN_ADD_SAME_C, alpha, biasDesc,
                           biasData, beta, srcDestDesc, srcDestData)
<commit_msg>Fix for compatibility<commit_after>import atexit
import threading

import numpy
import six

import cupy
from cupy.core import internal
from cupy import cuda
from cupy.cuda import cudnn

_cudnn_version = cudnn.getVersion()
_thread_local = threading.local()

_handles = {}

def get_handle():
    dev = cuda.get_device_id()
    if dev in _handles:
        return _handles[dev]
    handle = cudnn.create()
    _handles[dev] = handle
    return handle

@atexit.register
def reset_handles():
    global _handles
    handles = _handles
    _handles = {}

    for handle in six.itervalues(handles):
        cudnn.destroy(handle)

def _get_nd_tensor_cache():
    if not hasattr(_thread_local, 'cudnn_nd_tensor_cache'):
        _thread_local.cudnn_nd_tensor_cache = {}
    return _thread_local.cudnn_nd_tensor_cache

class Descriptor(object):

    def __init__(self, descriptor, destroyer):
        self.value = descriptor
        self.destroy = destroyer

    def __del__(self):
        if self.value:
            self.destroy(self.value)
            self.value = None

def get_data_type(dtype):
    if dtype.type == numpy.float32:
        return cudnn.CUDNN_DATA_FLOAT
    elif dtype.type == numpy.float64:
        return cudnn.CUDNN_DATA_DOUBLE
    elif dtype.type == numpy.float16:
        return cudnn.CUDNN_DATA_HALF
    else:
        raise TypeError('Dtype {} is not supported in cuDNN'.format(dtype))

def _to_ctypes_array(tup, dtype=numpy.intc):
    return numpy.array(tup, dtype=dtype).ctypes

def create_tensor_descriptor(arr, format=cudnn.CUDNN_TENSOR_NCHW):
    desc = Descriptor(cudnn.createTensorDescriptor(),
                      cudnn.destroyTensorDescriptor)
    if not arr.flags.c_contiguous:
        raise ValueError('cupy.cudnn supports c-contiguous arrays only')
    data_type = get_data_type(arr.dtype)
    if arr.ndim == 4:
        cudnn.setTensor4dDescriptor(desc.value, format, data_type, *arr.shape)
    else:
        strides = [s // arr.itemsize for s in arr.strides]
        c_shape = _to_ctypes_array(arr.shape)
        c_strides = _to_ctypes_array(strides)
        cudnn.setTensorNdDescriptor(desc.value, data_type, arr.ndim,
                                    c_shape.data, c_strides.data)
    return desc

def create_uninitialized_tensor_descriptor():
    """Create uninitialized tensor descriptor.

    Create a cudnnCreateTensorDescriptor_t that is not yet initialized.
    This is used by the batch normalization functions.
    """
    desc = Descriptor(cudnn.createTensorDescriptor(),
                      cudnn.destroyTensorDescriptor)
    return desc

def create_tensor_nd_descriptor(arr):
    desc = Descriptor(cudnn.createTensorDescriptor(),
                      cudnn.destroyTensorDescriptor)
    if not arr.flags.c_contiguous:
        raise ValueError('cupy.cudnn supports c-contiguous arrays only')
    data_type = get_data_type(arr.dtype)
    shape = arr.shape
    key = (data_type, shape)
    cache = _get_nd_tensor_cache()
    if key in cache:
        return cache[key]

    # numpy's stride is defined in bytes, but cudnn's stride is defined in
    # size of element
    strides = [s // arr.itemsize for s in arr.strides]

    c_shape = _to_ctypes_array(shape)
    c_strides = _to_ctypes_array(strides)
    cudnn.setTensorNdDescriptor(desc.value, data_type,
                                arr.ndim, c_shape.data, c_strides.data)
    cache = _get_nd_tensor_cache()
    cache[key] = desc
    return desc

def create_filter_descriptor(arr, format=cudnn.CUDNN_TENSOR_NCHW):
    desc = Descriptor(cudnn.createFilterDescriptor(),
                      cudnn.destroyFilterDescriptor)
    data_type = get_data_type(arr.dtype)
    if _cudnn_version >= 4000:
        if arr.ndim == 4:
            cudnn.setFilter4dDescriptor_v4(desc.value, data_type, format,
                                           *arr.shape)
        else:
            c_shape = _to_ctypes_array(arr.shape)
            cudnn.setFilterNdDescriptor_v4(desc.value, data_type, format,
                                           arr.ndim, c_shape.data)
    else:
        if arr.ndim == 4:
            cudnn.setFilter4dDescriptor_v3(desc.value, data_type, *arr.shape)
        else:
            c_shape = _to_ctypes_array(arr.shape)
            cudnn.setFilterNdDescriptor_v3(desc.value, data_type, arr.ndim,
                                           c_shape.data)

    return desc

def create_convolution_descriptor(pad, stride, dtype,
                                  mode=cudnn.CUDNN_CROSS_CORRELATION,
                                  dilation=(1, 1)):
    desc = Descriptor(cudnn.createConvolutionDescriptor(),
                      cudnn.destroyConvolutionDescriptor)
    ndim = len(pad)
    if ndim != len(stride):
        raise ValueError('pad and stride must be of same length')

    if ndim == 2:
        if _cudnn_version < 6000:
            if dilation[0] != 1 or dilation[1] != 1:
                raise ValueError('dilation must be one when cudnn < 6.0')
        if _cudnn_version >= 5000:
            data_type = get_data_type(dtype)
            # TODO(takagi) Temporarily use computing precision of FP32 for
            #     storing precision of FP16.
            if dtype == numpy.float16:
                data_type = cudnn.CUDNN_DATA_FLOAT
            cudnn.setConvolution2dDescriptor_v5(
                desc.value, pad[0], pad[1], stride[0], stride[1],
                dilation[0], dilation[1], mode, data_type)
        else:
            cudnn.setConvolution2dDescriptor_v4(
                desc.value, pad[0], pad[1], stride[0], stride[1], 1, 1, mode)
    else:
        c_pad = _to_ctypes_array(pad)
        c_stride = _to_ctypes_array(stride)
        c_dilation = _to_ctypes_array((1,) * ndim)
        if _cudnn_version >= 3000:
            data_type = get_data_type(dtype)
            # TODO(takagi) Temporarily use computing precision of FP32 for
            #     storing precision of FP16.
            if dtype == numpy.float16:
                data_type = cudnn.CUDNN_DATA_FLOAT
            cudnn.setConvolutionNdDescriptor_v3(
                desc.value, ndim, c_pad.data, c_stride.data, c_dilation.data,
                mode, data_type)
        else:
            cudnn.setConvolutionNdDescriptor_v2(
                desc.value, ndim, c_pad.data, c_stride.data, c_dilation.data,
                mode)

    return desc

def create_pooling_descriptor(ksize, stride, pad, mode):
    desc = Descriptor(cudnn.createPoolingDescriptor(),
                      cudnn.destroyPoolingDescriptor)
    ndim = len(ksize)
    if ndim != len(stride) or ndim != len(pad):
        raise ValueError('ksize, stride, and pad must be of same length')

    if ndim == 2:
        if _cudnn_version >= 4000:
            cudnn.setPooling2dDescriptor_v4(
                desc.value, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, ksize[0],
                ksize[1], pad[0], pad[1], stride[0], stride[1])
        else:
            cudnn.setPooling2dDescriptor_v3(
                desc.value, mode, ksize[0], ksize[1], pad[0], pad[1],
                stride[0], stride[1])
    else:
        c_ksize = _to_ctypes_array(ksize)
        c_pad = _to_ctypes_array(pad)
        c_stride = _to_ctypes_array(stride)
        if _cudnn_version >= 4000:
            cudnn.setPoolingNdDescriptor_v4(
                desc.value, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, ndim,
                c_ksize.data, c_pad.data, c_stride.data)
        else:
            cudnn.setPoolingNdDescriptor_v3(
                desc.value, mode, ndim, c_ksize.data, c_pad.data,
                c_stride.data)

    return desc

def _as4darray(arr):
    if arr.ndim == 0:
        return arr.reshape(1, 1, 1, 1)
    else:
        return arr.reshape(arr.shape[0], -1, 1, 1)

def activation_forward(x, mode):
    x = cupy.ascontiguousarray(x)
    y = cupy.empty_like(x)

    dtype = 'd' if x.dtype == 'd' else 'f'
    one = numpy.array(1, dtype=dtype).ctypes
    zero = numpy.array(0, dtype=dtype).ctypes
    handle = get_handle()
    x_mat = _as4darray(x)
    desc = create_tensor_descriptor(x_mat)
    if _cudnn_version >= 4000:
        act_desc = Descriptor(cudnn.createActivationDescriptor(),
                              cudnn.destroyActivationDescriptor)
        cudnn.setActivationDescriptor(
            act_desc.value, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, 0.0)
        cudnn.activationForward_v4(
            handle, act_desc.value, one.data, desc.value, x_mat.data.ptr,
            zero.data, desc.value, y.data.ptr)
    else:
        cudnn.activationForward_v3(
            handle, mode, one.data, desc.value, x_mat.data.ptr,
            zero.data, desc.value, y.data.ptr)
    return y

def activation_backward(x, y, gy, mode):
    x = cupy.ascontiguousarray(x)
    gy = cupy.ascontiguousarray(gy)

    gx = cupy.empty_like(x)
    dtype = 'd' if x.dtype == 'd' else 'f'
    one = numpy.array(1, dtype=dtype).ctypes
    zero = numpy.array(0, dtype=dtype).ctypes
    handle = get_handle()
    y_mat = _as4darray(y)
    desc = create_tensor_descriptor(y_mat)
    if _cudnn_version >= 4000:
        act_desc = Descriptor(cudnn.createActivationDescriptor(),
                              cudnn.destroyActivationDescriptor)
        cudnn.setActivationDescriptor(
            act_desc.value, mode, cudnn.CUDNN_NOT_PROPAGATE_NAN, 0.0)
        cudnn.activationBackward_v4(
            handle, act_desc.value, one.data, desc.value, y.data.ptr,
            desc.value, gy.data.ptr, desc.value, x.data.ptr,
            zero.data, desc.value, gx.data.ptr)
    else:
        cudnn.activationBackward_v3(
            handle, mode, one.data, desc.value, y.data.ptr,
            desc.value, gy.data.ptr, desc.value, x.data.ptr,
            zero.data, desc.value, gx.data.ptr)
    return gx

def create_dropout_descriptor(
        handle, dropout, states, state_size_in_bytes, seed):
    desc = Descriptor(cudnn.createDropoutDescriptor(),
                      cudnn.destroyDropoutDescriptor)
    cudnn.setDropoutDescriptor(desc.value, handle, dropout,
                               states, state_size_in_bytes, seed)
    return desc

def set_dropout_descriptor(desc, handle, dropout):
    # When the fourth argument is NULL, random state is not updated.
    cudnn.setDropoutDescriptor(desc.value, handle, dropout, 0, 0, 0)

def create_rnn_descriptor(hidden_size, num_layers, dropout_desc,
                          input_mode, direction, mode, data_type):
    desc = Descriptor(cudnn.createRNNDescriptor(),
                      cudnn.destroyRNNDescriptor)
    cudnn.setRNNDescriptor(
        desc.value, hidden_size, num_layers, dropout_desc.value,
        input_mode, direction, mode, data_type)
    return desc

def get_rnn_lin_layer_matrix_params(
        handle, rnn_desc, layer, x_desc, w_desc, w, lin_layer_id):
    mat_desc = Descriptor(cudnn.createFilterDescriptor(),
                          cudnn.destroyFilterDescriptor)
    ptr = numpy.array(0, dtype=numpy.intp)
    cudnn.getRNNLinLayerMatrixParams(
        handle, rnn_desc.value, layer, x_desc.value, w_desc.value, w.data.ptr,
        lin_layer_id, mat_desc.value, ptr.ctypes.data)
    offset = (ptr - w.data.ptr) // 4
    _, _, _, dim = cudnn.getFilterNdDescriptor(mat_desc.value, 3)
    size = internal.prod(dim)
    mat = w[offset: offset + size]
    return mat

def get_rnn_lin_layer_bias_params(
        handle, rnn_desc, layer, x_desc, w_desc, w, lin_layer_id):
    bias_desc = Descriptor(cudnn.createFilterDescriptor(),
                           cudnn.destroyFilterDescriptor)
    ptr = numpy.array(0, dtype=numpy.intp)
    cudnn.getRNNLinLayerBiasParams(
        handle, rnn_desc.value, layer, x_desc.value, w_desc.value, w.data.ptr,
        lin_layer_id, bias_desc.value, ptr.ctypes.data)
    offset = (ptr - w.data.ptr) // 4
    _, _, _, dim = cudnn.getFilterNdDescriptor(bias_desc.value, 3)
    size = internal.prod(dim)
    bias = w[offset: offset + size]
    return bias

def create_dropout_states(handle):
    state_size = cudnn.dropoutGetStatesSize(handle)
    return cupy.empty((state_size,), dtype='b')

def create_spatial_transformer_descriptor(sampler_type, dtype, nb_dims, dim_A):
    desc = Descriptor(cudnn.createSpatialTransformerDescriptor(),
                      cudnn.destroySpatialTransformerDescriptor)
    data_type = get_data_type(dtype)

    cudnn.setSpatialTransformerDescriptor(
        desc.value, sampler_type, data_type, nb_dims, dim_A)
    return desc

if _cudnn_version >= 3000:
    def add_tensor(handle, alpha, biasDesc, biasData, beta, srcDestDesc,
                   srcDestData):
        cudnn.addTensor_v3(handle, alpha, biasDesc,
                           biasData, beta, srcDestDesc, srcDestData)
else:
    def add_tensor(handle, alpha, biasDesc, biasData, beta, srcDestDesc,
                   srcDestData):
        cudnn.addTensor_v2(handle, cudnn.CUDNN_ADD_SAME_C, alpha, biasDesc,
                           biasData, beta, srcDestDesc, srcDestData)
<|endoftext|>