<commit_before># -*- coding: utf-8 -*-
"""
    Pygments basic API tests
    ~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import os
import random
import unittest

from pygments import lexers, formatters, filters, format
from pygments.token import _TokenType, Text
from pygments.lexer import RegexLexer
from pygments.formatters.img import FontNotFound
from pygments.util import BytesIO, StringIO, bytes, b

import support

TESTFILE, TESTDIR = support.location(__file__)

test_content = [chr(i) for i in xrange(33, 128)] * 5
random.shuffle(test_content)
test_content = ''.join(test_content) + '\n'

def test_lexer_import_all():
    # instantiate every lexer, to see if the token type defs are correct
    for x in lexers.LEXERS.keys():
        c = getattr(lexers, x)()

def test_lexer_classes():
    # test that every lexer class has the correct public API
    def verify(cls):
        assert type(cls.name) is str
        for attr in 'aliases', 'filenames', 'alias_filenames', 'mimetypes':
            assert hasattr(cls, attr)
            assert type(getattr(cls, attr)) is list, \
                   "%s: %s attribute wrong" % (cls, attr)
        result = cls.analyse_text("abc")
        assert isinstance(result, float) and 0.0 <= result <= 1.0

        inst = cls(opt1="val1", opt2="val2")
        if issubclass(cls, RegexLexer):
            if not hasattr(cls, '_tokens'):
                # if there's no "_tokens", the lexer has to be one with
                # multiple tokendef variants
                assert cls.token_variants
                for variant in cls.tokens:
                    assert 'root' in cls.tokens[variant]
            else:
                assert 'root' in cls._tokens, \
                       '%s has no root state' % cls

        if cls.name == 'XQuery':   # XXX temporary
            return

        tokens = list(inst.get_tokens(test_content))
        txt = ""
        for token in tokens:
            assert isinstance(token, tuple)
            assert isinstance(token[0], _TokenType)
            if isinstance(token[1], str):
                print repr(token[1])
            assert isinstance(token[1], unicode)
            txt += token[1]
        assert txt == test_content, "%s lexer roundtrip failed: %r != %r" % \
               (cls.name, test_content, txt)

    for lexer in lexers._iter_lexerclasses():
        yield verify, lexer

def test_lexer_options():
    # test that the basic options work
    def ensure(tokens, output):
        concatenated = ''.join(token[1] for token in tokens)
        assert concatenated == output, \
               '%s: %r != %r' % (lexer, concatenated, output)
    def verify(cls):
        inst = cls(stripnl=False)
        ensure(inst.get_tokens('a\nb'), 'a\nb\n')
        ensure(inst.get_tokens('\n\n\n'), '\n\n\n')
        inst = cls(stripall=True)
        ensure(inst.get_tokens('   \n  b\n\n\n'), 'b\n')
        # some lexers require full lines in input
        if cls.__name__ not in (
            'PythonConsoleLexer', 'RConsoleLexer', 'RubyConsoleLexer',
            'SqliteConsoleLexer', 'MatlabSessionLexer', 'ErlangShellLexer',
            'BashSessionLexer', 'LiterateHaskellLexer'):
            inst = cls(ensurenl=False)
            ensure(inst.get_tokens('a\nb'), 'a\nb')
            inst = cls(ensurenl=False, stripall=True)
            ensure(inst.get_tokens('a\nb\n\n'), 'a\nb')

    for lexer in lexers._iter_lexerclasses():
        if lexer.__name__ == 'RawTokenLexer':
            # this one is special
            continue
        yield verify, lexer

def test_get_lexers():
    # test that the lexers functions work
    def verify(func, args):
        x = func(opt='val', *args)
        assert isinstance(x, lexers.PythonLexer)
        assert x.options["opt"] == "val"

    for func, args in [(lexers.get_lexer_by_name, ("python",)),
                       (lexers.get_lexer_for_filename, ("test.py",)),
                       (lexers.get_lexer_for_mimetype, ("text/x-python",)),
                       (lexers.guess_lexer, ("#!/usr/bin/python -O\nprint",)),
                       (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>"))
                       ]:
        yield verify, func, args

def test_formatter_public_api():
    ts = list(lexers.PythonLexer().get_tokens("def f(): pass"))
    out = StringIO()
    # test that every formatter class has the correct public API
    def verify(formatter, info):
        assert len(info) == 4
        assert info[0], "missing formatter name"
        assert info[1], "missing formatter aliases"
        assert info[3], "missing formatter docstring"

        if formatter.name == 'Raw tokens':
            # will not work with Unicode output file
            return

        try:
            inst = formatter(opt1="val1")
        except (ImportError, FontNotFound):
            return
        try:
            inst.get_style_defs()
        except NotImplementedError:
            # may be raised by formatters for which it doesn't make sense
            pass
        inst.format(ts, out)

    for formatter, info in formatters.FORMATTERS.iteritems():
        yield verify, formatter, info

def test_formatter_encodings():
    from pygments.formatters import HtmlFormatter

    # unicode output
    fmt = HtmlFormatter()
    tokens = [(Text, u"ä")]
    out = format(tokens, fmt)
    assert type(out) is unicode
    assert u"ä" in out

    # encoding option
    fmt = HtmlFormatter(encoding="latin1")
    tokens = [(Text, u"ä")]
    assert u"ä".encode("latin1") in format(tokens, fmt)

    # encoding and outencoding option
    fmt = HtmlFormatter(encoding="latin1", outencoding="utf8")
    tokens = [(Text, u"ä")]
    assert u"ä".encode("utf8") in format(tokens, fmt)

def test_formatter_unicode_handling():
    # test that the formatter supports encoding and Unicode
    tokens = list(lexers.PythonLexer(encoding='utf-8').
                  get_tokens("def f(): 'ä'"))

    def verify(formatter):
        try:
            inst = formatter(encoding=None)
        except (ImportError, FontNotFound):
            # some dependency or font not installed
            return

        if formatter.name != 'Raw tokens':
            out = format(tokens, inst)
            if formatter.unicodeoutput:
                assert type(out) is unicode

            inst = formatter(encoding='utf-8')
            out = format(tokens, inst)
            assert type(out) is bytes, '%s: %r' % (formatter, out)
            # Cannot test for encoding, since formatters may have to escape
            # non-ASCII characters.
        else:
            inst = formatter()
            out = format(tokens, inst)
            assert type(out) is bytes, '%s: %r' % (formatter, out)

    for formatter, info in formatters.FORMATTERS.iteritems():
        yield verify, formatter

def test_get_formatters():
    # test that the formatters functions work
    x = formatters.get_formatter_by_name("html", opt="val")
    assert isinstance(x, formatters.HtmlFormatter)
    assert x.options["opt"] == "val"

    x = formatters.get_formatter_for_filename("a.html", opt="val")
    assert isinstance(x, formatters.HtmlFormatter)
    assert x.options["opt"] == "val"

def test_styles():
    # minimal style test
    from pygments.formatters import HtmlFormatter
    fmt = HtmlFormatter(style="pastie")

class FiltersTest(unittest.TestCase):

    def test_basic(self):
        filter_args = {
            'whitespace': {'spaces': True, 'tabs': True, 'newlines': True},
            'highlight': {'names': ['isinstance', 'lexers', 'x']},
        }
        for x in filters.FILTERS.keys():
            lx = lexers.PythonLexer()
            lx.add_filter(x, **filter_args.get(x, {}))
            text = open(TESTFILE, 'rb').read().decode('utf-8')
            tokens = list(lx.get_tokens(text))
            roundtext = ''.join([t[1] for t in tokens])
            if x not in ('whitespace', 'keywordcase'):
                # these filters change the text
                self.assertEquals(roundtext, text,
                                  "lexer roundtrip with %s filter failed" % x)

    def test_raiseonerror(self):
        lx = lexers.PythonLexer()
        lx.add_filter('raiseonerror', excclass=RuntimeError)
        self.assertRaises(RuntimeError, list, lx.get_tokens('$'))

    def test_whitespace(self):
        lx = lexers.PythonLexer()
        lx.add_filter('whitespace', spaces='%')
        text = open(TESTFILE, 'rb').read().decode('utf-8')
        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
        self.failIf(' ' in lxtext)

    def test_keywordcase(self):
        lx = lexers.PythonLexer()
        lx.add_filter('keywordcase', case='capitalize')
        text = open(TESTFILE, 'rb').read().decode('utf-8')
        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
        self.assert_('Def' in lxtext and 'Class' in lxtext)

    def test_codetag(self):
        lx = lexers.PythonLexer()
        lx.add_filter('codetagify')
        text = u'# BUG: text'
        tokens = list(lx.get_tokens(text))
        self.assertEquals('# ', tokens[0][1])
        self.assertEquals('BUG', tokens[1][1])

    def test_codetag_boundary(self):
        # ticket #368
        lx = lexers.PythonLexer()
        lx.add_filter('codetagify')
        text = u'# DEBUG: text'
        tokens = list(lx.get_tokens(text))
        self.assertEquals('# DEBUG: text', tokens[0][1])
<commit_msg>Test all lexers can be found by name or alias<commit_after># -*- coding: utf-8 -*-
"""
    Pygments basic API tests
    ~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import os
import random
import unittest

from pygments import lexers, formatters, filters, format
from pygments.token import _TokenType, Text
from pygments.lexer import RegexLexer
from pygments.formatters.img import FontNotFound
from pygments.util import BytesIO, StringIO, bytes, b

import support

TESTFILE, TESTDIR = support.location(__file__)

test_content = [chr(i) for i in xrange(33, 128)] * 5
random.shuffle(test_content)
test_content = ''.join(test_content) + '\n'

def test_lexer_import_all():
    # instantiate every lexer, to see if the token type defs are correct
    for x in lexers.LEXERS.keys():
        c = getattr(lexers, x)()

def test_lexer_classes():
    # test that every lexer class has the correct public API
    def verify(cls):
        assert type(cls.name) is str
        for attr in 'aliases', 'filenames', 'alias_filenames', 'mimetypes':
            assert hasattr(cls, attr)
            assert type(getattr(cls, attr)) is list, \
                   "%s: %s attribute wrong" % (cls, attr)
        result = cls.analyse_text("abc")
        assert isinstance(result, float) and 0.0 <= result <= 1.0

        inst = cls(opt1="val1", opt2="val2")
        if issubclass(cls, RegexLexer):
            if not hasattr(cls, '_tokens'):
                # if there's no "_tokens", the lexer has to be one with
                # multiple tokendef variants
                assert cls.token_variants
                for variant in cls.tokens:
                    assert 'root' in cls.tokens[variant]
            else:
                assert 'root' in cls._tokens, \
                       '%s has no root state' % cls

        if cls.name == 'XQuery':   # XXX temporary
            return

        tokens = list(inst.get_tokens(test_content))
        txt = ""
        for token in tokens:
            assert isinstance(token, tuple)
            assert isinstance(token[0], _TokenType)
            if isinstance(token[1], str):
                print repr(token[1])
            assert isinstance(token[1], unicode)
            txt += token[1]
        assert txt == test_content, "%s lexer roundtrip failed: %r != %r" % \
               (cls.name, test_content, txt)

    for lexer in lexers._iter_lexerclasses():
        yield verify, lexer

def test_lexer_options():
    # test that the basic options work
    def ensure(tokens, output):
        concatenated = ''.join(token[1] for token in tokens)
        assert concatenated == output, \
               '%s: %r != %r' % (lexer, concatenated, output)
    def verify(cls):
        inst = cls(stripnl=False)
        ensure(inst.get_tokens('a\nb'), 'a\nb\n')
        ensure(inst.get_tokens('\n\n\n'), '\n\n\n')
        inst = cls(stripall=True)
        ensure(inst.get_tokens('   \n  b\n\n\n'), 'b\n')
        # some lexers require full lines in input
        if cls.__name__ not in (
            'PythonConsoleLexer', 'RConsoleLexer', 'RubyConsoleLexer',
            'SqliteConsoleLexer', 'MatlabSessionLexer', 'ErlangShellLexer',
            'BashSessionLexer', 'LiterateHaskellLexer'):
            inst = cls(ensurenl=False)
            ensure(inst.get_tokens('a\nb'), 'a\nb')
            inst = cls(ensurenl=False, stripall=True)
            ensure(inst.get_tokens('a\nb\n\n'), 'a\nb')

    for lexer in lexers._iter_lexerclasses():
        if lexer.__name__ == 'RawTokenLexer':
            # this one is special
            continue
        yield verify, lexer

def test_get_lexers():
    # test that the lexers functions work
    def verify(func, args):
        x = func(opt='val', *args)
        assert isinstance(x, lexers.PythonLexer)
        assert x.options["opt"] == "val"

    for func, args in [(lexers.get_lexer_by_name, ("python",)),
                       (lexers.get_lexer_for_filename, ("test.py",)),
                       (lexers.get_lexer_for_mimetype, ("text/x-python",)),
                       (lexers.guess_lexer, ("#!/usr/bin/python -O\nprint",)),
                       (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>"))
                       ]:
        yield verify, func, args

    for cls, (_, lname, aliases, _, mimetypes) in lexers.LEXERS.iteritems():
        assert cls == lexers.find_lexer_class(lname).__name__

        for alias in aliases:
            assert cls == lexers.get_lexer_by_name(alias).__class__.__name__

def test_formatter_public_api():
    ts = list(lexers.PythonLexer().get_tokens("def f(): pass"))
    out = StringIO()
    # test that every formatter class has the correct public API
    def verify(formatter, info):
        assert len(info) == 4
        assert info[0], "missing formatter name"
        assert info[1], "missing formatter aliases"
        assert info[3], "missing formatter docstring"

        if formatter.name == 'Raw tokens':
            # will not work with Unicode output file
            return

        try:
            inst = formatter(opt1="val1")
        except (ImportError, FontNotFound):
            return
        try:
            inst.get_style_defs()
        except NotImplementedError:
            # may be raised by formatters for which it doesn't make sense
            pass
        inst.format(ts, out)

    for formatter, info in formatters.FORMATTERS.iteritems():
        yield verify, formatter, info

def test_formatter_encodings():
    from pygments.formatters import HtmlFormatter

    # unicode output
    fmt = HtmlFormatter()
    tokens = [(Text, u"ä")]
    out = format(tokens, fmt)
    assert type(out) is unicode
    assert u"ä" in out

    # encoding option
    fmt = HtmlFormatter(encoding="latin1")
    tokens = [(Text, u"ä")]
    assert u"ä".encode("latin1") in format(tokens, fmt)

    # encoding and outencoding option
    fmt = HtmlFormatter(encoding="latin1", outencoding="utf8")
    tokens = [(Text, u"ä")]
    assert u"ä".encode("utf8") in format(tokens, fmt)

def test_formatter_unicode_handling():
    # test that the formatter supports encoding and Unicode
    tokens = list(lexers.PythonLexer(encoding='utf-8').
                  get_tokens("def f(): 'ä'"))

    def verify(formatter):
        try:
            inst = formatter(encoding=None)
        except (ImportError, FontNotFound):
            # some dependency or font not installed
            return

        if formatter.name != 'Raw tokens':
            out = format(tokens, inst)
            if formatter.unicodeoutput:
                assert type(out) is unicode

            inst = formatter(encoding='utf-8')
            out = format(tokens, inst)
            assert type(out) is bytes, '%s: %r' % (formatter, out)
            # Cannot test for encoding, since formatters may have to escape
            # non-ASCII characters.
        else:
            inst = formatter()
            out = format(tokens, inst)
            assert type(out) is bytes, '%s: %r' % (formatter, out)

    for formatter, info in formatters.FORMATTERS.iteritems():
        yield verify, formatter

def test_get_formatters():
    # test that the formatters functions work
    x = formatters.get_formatter_by_name("html", opt="val")
    assert isinstance(x, formatters.HtmlFormatter)
    assert x.options["opt"] == "val"

    x = formatters.get_formatter_for_filename("a.html", opt="val")
    assert isinstance(x, formatters.HtmlFormatter)
    assert x.options["opt"] == "val"

def test_styles():
    # minimal style test
    from pygments.formatters import HtmlFormatter
    fmt = HtmlFormatter(style="pastie")

class FiltersTest(unittest.TestCase):

    def test_basic(self):
        filter_args = {
            'whitespace': {'spaces': True, 'tabs': True, 'newlines': True},
            'highlight': {'names': ['isinstance', 'lexers', 'x']},
        }
        for x in filters.FILTERS.keys():
            lx = lexers.PythonLexer()
            lx.add_filter(x, **filter_args.get(x, {}))
            text = open(TESTFILE, 'rb').read().decode('utf-8')
            tokens = list(lx.get_tokens(text))
            roundtext = ''.join([t[1] for t in tokens])
            if x not in ('whitespace', 'keywordcase'):
                # these filters change the text
                self.assertEquals(roundtext, text,
                                  "lexer roundtrip with %s filter failed" % x)

    def test_raiseonerror(self):
        lx = lexers.PythonLexer()
        lx.add_filter('raiseonerror', excclass=RuntimeError)
        self.assertRaises(RuntimeError, list, lx.get_tokens('$'))

    def test_whitespace(self):
        lx = lexers.PythonLexer()
        lx.add_filter('whitespace', spaces='%')
        text = open(TESTFILE, 'rb').read().decode('utf-8')
        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
        self.failIf(' ' in lxtext)

    def test_keywordcase(self):
        lx = lexers.PythonLexer()
        lx.add_filter('keywordcase', case='capitalize')
        text = open(TESTFILE, 'rb').read().decode('utf-8')
        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
        self.assert_('Def' in lxtext and 'Class' in lxtext)

    def test_codetag(self):
        lx = lexers.PythonLexer()
        lx.add_filter('codetagify')
        text = u'# BUG: text'
        tokens = list(lx.get_tokens(text))
        self.assertEquals('# ', tokens[0][1])
        self.assertEquals('BUG', tokens[1][1])

    def test_codetag_boundary(self):
        # ticket #368
        lx = lexers.PythonLexer()
        lx.add_filter('codetagify')
        text = u'# DEBUG: text'
        tokens = list(lx.get_tokens(text))
        self.assertEquals('# DEBUG: text', tokens[0][1])
<|endoftext|>


<commit_before>#!/usr/bin/python

"""Download images from DCC using media API
Allows download of images using the media API provided by the DCC instead
of the solr core which the original downloadimages.py script uses. This is
necessary sometimes to get a headstart on the data-release.

See also downloadimages.py and downloadimages_using_xml.py

"""
import sys
import os
import requests
import json
import argparse

# Import helper functions from original downloadimages.py
from downloadimages import createDestinationFilePath, createNotDownloadedOutputPath, processFile

uniqueUris=set()

def main(argv):
    """Download images using DCC media API onto holding area on disk"""

    parser = argparse.ArgumentParser(
        description='Download images using DCC media API onto holding area on disk'
    )
    parser.add_argument('-d1', '--initialDestinationDir', required=True,
                        dest='initialDestinationDir',
                        help='Directory for root of holding destination to store images'
    )
    parser.add_argument('-d2', '--finalDestinationDir', required=True,
                        dest='finalDestinationDir',
                        help='Directory for root of final destination to store images'
    )
    parser.add_argument('--not-downloaded', dest='notDownloadedOutputPath',
                        help='path to save list of files that could not be downloaded'
    )

    args = parser.parse_args()

    rootDestinationDir = args.initialDestinationDir
    finalDestinationDir = args.finalDestinationDir

    print "running python image download script for impc images"

    print 'rootDestinationDir is "', rootDestinationDir

    notDownloaded = runWithMediaApiAsDataSource(rootDestinationDir, finalDestinationDir)
    print str(len(notDownloaded)) + " files could not be downloaded"
    if len(notDownloaded) > 0:
        notDownloadedOutputPath =  args.notDownloadedOutputPath if args.notDownloadedOutputPath <> None else createNotDownloadedOutputPath(rootDestinationDir)
        with open(notDownloadedOutputPath, 'wt') as fid:
            fid.writelines(notDownloaded)

        print "Written files that could not be downloaded to " + notDownloadedOutputPath

def runWithMediaApiAsDataSource(rootDestinationDir, finalDestinationDir):
    """
        Download images using Media API as the datasource.
        Return urls that cannot be downloaded
    """

    notDownloaded = []
    numFound=0

    # We get the files we are interested in for each site using the
    # media API
    sites = [
        ('bcm', 'BCM',),
        ('gmc','HMGU',),
        ('h', 'MRC Harwell'),
        ('ics', 'ICS',),
        ('j', 'JAX',),
        ('tcp', 'TCP'),
        ('ning', 'NING',),
        ('rbrc', 'RBRC',),
        ('ucd', 'UC Davis',),
        ('wtsi', 'WTSI',),
        ('kmpc', 'KMPC',),
        ('ccpcz', 'CCP-IMG',),
    ]

    # Preset observation ID. This is passed to processFile in the
    # loop below but is not used by the function.
    # ToDo: remove from function definition
    observation_id=None

    for site, phenotyping_center in sites:
        query_string = "https://api.mousephenotype.org/media/dccUrl/" +\
            site # + "?start=0&resultsize=2"
        print query_string

        v = json.loads(requests.get(query_string).text)
        try:
            docs = v['mediaFiles']
        except KeyError as key_error:
            print "WARNING - no media files returned for site: " + site
            continue

        numFound += len(docs)

        for doc in docs:
            download_file_path=doc['dccUrl']
            download_file_path=download_file_path.lower()
            if download_file_path.find('mousephenotype.org') < 0 or \
                    download_file_path.endswith('.mov') or \
                    download_file_path.endswith('.bz2'):
                continue

            # On 13/11/2019 got a KeyError for phenotyping centre. This
            # should not happen, but code modified appropriately
            try:
                pipeline_stable_id=doc['pipelineKey']
                procedure_stable_id=doc['procedureKey']
                parameter_stable_id=doc['parameterKey']
            except KeyError as e:
                print "Key " + str(e)+  " not returned by solr - not downloading " + download_file_path
                notDownloaded.append(download_file_path+'\n')
                continue
            downloaded = processFile(observation_id, rootDestinationDir, finalDestinationDir, phenotyping_center,pipeline_stable_id, procedure_stable_id, parameter_stable_id, download_file_path)
            if not downloaded:
                notDownloaded.append(download_file_path+'\n')

    print 'number found in media API ='+str(numFound)+' number of images not downloaded = '+str(len(notDownloaded))

    return notDownloaded

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>Add 'done' status to api query so only processed images are downloaded<commit_after>#!/usr/bin/python

"""Download images from DCC using media API
Allows download of images using the media API provided by the DCC instead
of the solr core which the original downloadimages.py script uses. This is
necessary sometimes to get a headstart on the data-release.

See also downloadimages.py and downloadimages_using_xml.py

"""
import sys
import os
import requests
import json
import argparse

# Import helper functions from original downloadimages.py
from downloadimages import createDestinationFilePath, createNotDownloadedOutputPath, processFile

uniqueUris=set()

def main(argv):
    """Download images using DCC media API onto holding area on disk"""

    parser = argparse.ArgumentParser(
        description='Download images using DCC media API onto holding area on disk'
    )
    parser.add_argument('-d1', '--initialDestinationDir', required=True,
                        dest='initialDestinationDir',
                        help='Directory for root of holding destination to store images'
    )
    parser.add_argument('-d2', '--finalDestinationDir', required=True,
                        dest='finalDestinationDir',
                        help='Directory for root of final destination to store images'
    )
    parser.add_argument('--not-downloaded', dest='notDownloadedOutputPath',
                        help='path to save list of files that could not be downloaded'
    )

    args = parser.parse_args()

    rootDestinationDir = args.initialDestinationDir
    finalDestinationDir = args.finalDestinationDir

    print "running python image download script for impc images"

    print 'rootDestinationDir is "', rootDestinationDir

    notDownloaded = runWithMediaApiAsDataSource(rootDestinationDir, finalDestinationDir)
    print str(len(notDownloaded)) + " files could not be downloaded"
    if len(notDownloaded) > 0:
        notDownloadedOutputPath =  args.notDownloadedOutputPath if args.notDownloadedOutputPath <> None else createNotDownloadedOutputPath(rootDestinationDir)
        with open(notDownloadedOutputPath, 'wt') as fid:
            fid.writelines(notDownloaded)

        print "Written files that could not be downloaded to " + notDownloadedOutputPath

def runWithMediaApiAsDataSource(rootDestinationDir, finalDestinationDir):
    """
        Download images using Media API as the datasource.
        Return urls that cannot be downloaded
    """

    notDownloaded = []
    numFound=0

    # We get the files we are interested in for each site using the
    # media API
    sites = [
        ('bcm', 'BCM',),
        ('gmc','HMGU',),
        ('h', 'MRC Harwell'),
        ('ics', 'ICS',),
        ('j', 'JAX',),
        ('tcp', 'TCP'),
        ('ning', 'NING',),
        ('rbrc', 'RBRC',),
        ('ucd', 'UC Davis',),
        ('wtsi', 'WTSI',),
        ('kmpc', 'KMPC',),
        ('ccpcz', 'CCP-IMG',),
    ]

    # Preset observation ID. This is passed to processFile in the
    # loop below but is not used by the function.
    # ToDo: remove from function definition
    observation_id=None

    for site, phenotyping_center in sites:
        query_string = "https://api.mousephenotype.org/media/dccUrl/" +\
            site + "?status=done" # +"&start=0&resultsize=2"
        print query_string

        v = json.loads(requests.get(query_string).text)
        try:
            docs = v['mediaFiles']
        except KeyError as key_error:
            print "WARNING - no media files returned for site: " + site
            continue

        numFound += len(docs)

        for doc in docs:
            download_file_path=doc['dccUrl']
            download_file_path=download_file_path.lower()
            if download_file_path.find('mousephenotype.org') < 0 or \
                    download_file_path.endswith('.mov') or \
                    download_file_path.endswith('.bz2'):
                continue

            # On 13/11/2019 got a KeyError for phenotyping centre. This
            # should not happen, but code modified appropriately
            try:
                pipeline_stable_id=doc['pipelineKey']
                procedure_stable_id=doc['procedureKey']
                parameter_stable_id=doc['parameterKey']
            except KeyError as e:
                print "Key " + str(e)+  " not returned by solr - not downloading " + download_file_path
                notDownloaded.append(download_file_path+'\n')
                continue
            downloaded = processFile(observation_id, rootDestinationDir, finalDestinationDir, phenotyping_center,pipeline_stable_id, procedure_stable_id, parameter_stable_id, download_file_path)
            if not downloaded:
                notDownloaded.append(download_file_path+'\n')

    print 'number found in media API ='+str(numFound)+' number of images not downloaded = '+str(len(notDownloaded))

    return notDownloaded

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>


<commit_before><commit_msg>fix for view query error after collections upgrade<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
# author: @netmanchris
# -*- coding: utf-8 -*-
"""
This module contains class objects for working with the specific types of network
infrastructure equipment in the HPE IMC NMS using the RESTful API. These objects
rely on various other functions from within this library.

"""

# This section imports required libraries
import json
import requests
from pyhpeimc.plat.device import *
from pyhpeimc.plat.vlanm import *
from pyhpeimc.plat.termaccess import *
from pyhpeimc.plat.netassets import *
from pyhpeimc.plat.alarms import *
from requests.auth import HTTPDigestAuth

headers = {'Accept': 'application/json', 'Content-Type':
    'application/json', 'Accept-encoding': 'application/json'}

# IMC Device Class

class IMCDev:
    """
    imc_dev class takes in the ip_address which is used as the primary key to gather the following attributes
         for a device which as been previously discovered in the HP IMC Network Management platform.
         Each instance of this class should have the following attributes

         ip: The IP address used to manage the device in HP IMC
         description: returns the description of the device as discovered in HP IMC
         location: returns the location of the device as discovered in HP IMC
         contact: returns the contact of the device as discovered in HP IMC
         type: returns the type of the device as discovered in HP IMC
         name: returns the name of the device as discovered in HP IMC
         status: returns the current alarm status as discovered in HP IMC
         devid: returns the current devid used to internally identify the device as discovered in HP IMC
         interfacelist: returns the current list of interfaces for the device as discovered in HP IMC
         numinterface: returns a count of the number of interfaces in the interfacelist attribute
         vlans: returns the current vlans existing in the device as discovered in HP IMC. Device must be supported in the
                HP IMC Platform VLAN manager module.
         accessinterfaces: returns the device interfaces configured as access interfaces. Device must be supported in the
                HP IMC Platform VLAN manager module.
         trunkinterfaces: returns the device interfaces configured as trunk interfaces. Device must be supported in the
                HP IMC Platform VLAN manager module.
         alarm: returns the current unrecovered alarms as known by HP IMC.
         num alarms: returns a count of the number of alarms as returned by the alarm attribute
         serial: returns the network assets, including serial numbers for the device as discovered by HP IMC. The device
                must support the ENTITY MIB ( rfc 4133 ) for this value to be returned.
         runconfig: returns the most recent running configuration for the device as known by HP IMC. The device must be
                be supported in the HP IMC platform ICC module.
         startconfig: returns the most recent startup configuration for the device as known by HP IMC. The device must be
                be supported in the HP IMC platform ICC module.
         ipmacarp: returns the current device maciparp table as discovered by HP IMC.

         The imc_dev class supports the following methods which can be called upon an instance of this class

         addvlan: This method executes the addvlan function on the specific instance of the imc_dev object. Devices must
                  supported in the HP IMC Platform VLAN Manager module.

         """

    def __init__(self, ip_address, auth, url):
        self.ip = get_dev_details(ip_address, auth, url)['ip']
        self.description = get_dev_details(ip_address, auth, url)['sysDescription']
        self.location = get_dev_details(ip_address, auth, url)['location']
        self.contact = get_dev_details(ip_address, auth, url)['contact']
        self.type = get_dev_details(ip_address, auth, url)['typeName']
        self.name = get_dev_details(ip_address, auth, url)['sysName']
        self.status = get_dev_details(ip_address, auth, url)['statusDesc']
        self.devid = get_dev_details(ip_address, auth, url)['id']
        self.interfacelist = get_dev_interface(self.devid, auth, url)
        self.numinterface = len(get_dev_interface(self.devid, auth, url))
        self.vlans = get_dev_vlans(self.devid, auth, url)
        self.accessinterfaces = get_device_access_interfaces(self.devid, auth, url)
        self.trunkinterfaces = get_trunk_interfaces(self.devid, auth, url)
        self.alarm = get_dev_alarms(self.devid, auth, url)
        self.numalarm = len(get_dev_alarms(self.devid, auth, url))
        self.assets = get_dev_asset_details(self.ip, auth, url)
        self.serials = [({'name' : asset['name'], 'serialNum': asset['serialNum']}) for asset in self.assets]
        self.runconfig = get_dev_run_config(self.devid, auth, url)
        self.startconfig = get_dev_start_config(self.devid, auth, url)
        self.ipmacarp = get_ip_mac_arp_list(self.devid, auth, url)
        self.auth = auth
        self.url = url

    def getvlans(self):
        self.vlans = get_dev_vlans(self.devid, self.auth, self.url)

    def addvlan(self, vlanid, vlan_name,auth, url):
        create_dev_vlan(self.devid, vlanid, vlan_name, auth, url)

    def delvlan(self, vlanid, auth, url):
        delete_dev_vlans(self.devid, vlanid, auth, url)

    def getipmacarp(self):
        self.ipmacarp = get_ip_mac_arp_list(self.devid, auth, url)

class IMCInterface:
    def __init__(self, ip_address, ifIndex, auth, url):
        self.ip = get_dev_details(ip_address,auth, url)['ip']
        self.devid = get_dev_details(ip_address,auth, url)['id']
        self.ifIndex = get_interface_details(self.devid, ifIndex,auth, url)['ifIndex']
        self.macaddress = get_interface_details(self.devid, ifIndex,auth, url)['phyAddress']
        self.status = get_interface_details(self.devid, ifIndex,auth, url)['statusDesc']
        self.adminstatus = get_interface_details(self.devid, ifIndex,auth, url)['adminStatusDesc']
        self.name = get_interface_details(self.devid, ifIndex,auth, url)['ifDescription']
        self.description = get_interface_details(self.devid, ifIndex,auth, url)['ifAlias']
        self.mtu = get_interface_details(self.devid, ifIndex,auth, url)['mtu']
        self.speed = get_interface_details(self.devid, ifIndex,auth, url)['ifspeed']
        self.accessinterfaces = get_device_access_interfaces(self.devid,auth, url)
        self.pvid = get_access_interface_vlan(self.ifIndex, self.accessinterfaces,auth, url)
        self.auth = auth
        self.url = url

class IPScope:
    def __init__(self, netaddr, auth, url):
        self.id = get_scope_id(netaddr, auth, url)
        self.details = get_ip_scope_detail(auth, url, self.id)
        self.hosts = get_ip_scope_hosts(auth, url, self.id)
        self.auth = auth
        self.url = url
        self.netaddr = ipaddress.ip_network(netaddr)
        self.startip = get_ip_scope_detail(auth, url, self.id)['startIp']
        self.endip = get_ip_scope_detail(auth, url, self.id)['endIp']
        if 'assignedIpScope' in self.details:
            self.child = get_ip_scope_detail(auth, url, self.id)['assignedIpScope']

    def allocateIp(self, ipaddress, name, description):
        add_scope_ip(ipaddress, name, description, self.id, self.auth, self.url)

    def deallocateIp(self, hostid):
        remove_scope_ip( hostid, self.auth, self.url)

    def gethosts(self):
        self.hosts = get_ip_scope_hosts(self.auth, self.url, self.id)

    def nextfreeip(self):
        allocated_ips = [ ipaddress.ip_address(host['ip']) for host in self.hosts]
        for ip in self.netaddr:
            if str(ip).split('.')[-1] == '0':
                continue
            if ip not in allocated_ips:
                return ip

    def addchild(self, startIp, endIp, name, description):
        add_child_ip_scope(self.auth, self.url,startIp, endIp, name, description,self.id)

<commit_msg>Making pep8 modifications to increase code quality<commit_after>#!/usr/bin/env python3
# author: @netmanchris
# -*- coding: utf-8 -*-
"""
This module contains class objects for working with the specific types of network
infrastructure equipment in the HPE IMC NMS using the RESTful API. These objects
rely on various other functions from within this library.

"""

# This section imports required libraries
import json
import requests
from pyhpeimc.plat.device import *
from pyhpeimc.plat.vlanm import *
from pyhpeimc.plat.termaccess import *
from pyhpeimc.plat.netassets import *
from pyhpeimc.plat.alarms import *

headers = {'Accept': 'application/json', 'Content-Type':
    'application/json', 'Accept-encoding': 'application/json'}

# IMC Device Class

class IMCDev:
    """
    imc_dev class takes in the ip_address which is used as the primary key to gather
    the following attributes
    for a device which as been previously discovered in the HP IMC Network Management
    platform.

     Each instance of this class should have the following attributes

     ip: The IP address used to manage the device in HP IMC
     description: returns the description of the device as discovered in HP IMC
     location: returns the location of the device as discovered in HP IMC
     contact: returns the contact of the device as discovered in HP IMC
     type: returns the type of the device as discovered in HP IMC
     name: returns the name of the device as discovered in HP IMC
     status: returns the current alarm status as discovered in HP IMC
     devid: returns the current devid used to internally identify the device
     as discovered in HP IMC
     interfacelist: returns the current list of interfaces for the device as
     discovered in HP IMC
     numinterface: returns a count of the number of interfaces in the interfacelist
     attribute
     vlans: returns the current vlans existing in the device as discovered in HP IMC.
     Device must be supported in the HP IMC Platform VLAN manager module.
     accessinterfaces: returns the device interfaces configured as access interfaces.
     Device must be supported in the HP IMC Platform VLAN manager module.
     trunkinterfaces: returns the device interfaces configured as trunk interfaces.
     Device must be supported in the HP IMC Platform VLAN manager module.
     alarm: returns the current unrecovered alarms as known by HP IMC.
     num alarms: returns a count of the number of alarms as returned by the alarm
     attribute
     serial: returns the network assets, including serial numbers for the device as
     discovered by HP IMC. The device must support the ENTITY MIB ( rfc 4133 ) for
     this value to be returned.
     runconfig: returns the most recent running configuration for the device as known
     by HP IMC. The device must be be supported in the HP IMC platform ICC module.
     startconfig: returns the most recent startup configuration for the device as known
     by HP IMC. The device must be supported in the HP IMC platform ICC module.
     ipmacarp: returns the current device maciparp table as discovered by HP IMC.

     The imc_dev class supports the following methods which can be called upon an instance of this class

     getvlans: This method executes the getvlans function on the specific instance of the imc_dev
     object and populates the return into the self.vlans attribute. Devices must be supported in
     the HPE IMC Platform VLAN Manager module
     addvlan: This method executes the addvlan function on the specific instance of the imc_dev
     object. Devices must supported in the HP IMC Platform VLAN Manager module.

     """

    # pylint: disable=too-many-instance-attributes

    def __init__(self, ip_address, auth, url):
        """
        Function take in input of ipv4 address, auth and url and returns and object of type IMCDev
        :param ip_address: valid IPv4 address
        :param auth: requests auth object #usually auth.creds from auth pyhpeimc.auth.class
        :param url: base url of IMC RS interface #usually auth.url from pyhpeimc.auth.authclass
        :return IMCDev object
        :rtype IMCDev
        """
        self.ip = get_dev_details(ip_address, auth, url)['ip']
        self.description = get_dev_details(ip_address, auth, url)['sysDescription']
        self.location = get_dev_details(ip_address, auth, url)['location']
        self.contact = get_dev_details(ip_address, auth, url)['contact']
        self.type = get_dev_details(ip_address, auth, url)['typeName']
        self.name = get_dev_details(ip_address, auth, url)['sysName']
        self.status = get_dev_details(ip_address, auth, url)['statusDesc']
        self.devid = get_dev_details(ip_address, auth, url)['id']
        self.interfacelist = get_dev_interface(self.devid, auth, url)
        self.numinterface = len(get_dev_interface(self.devid, auth, url))
        self.vlans = get_dev_vlans(self.devid, auth, url)
        self.accessinterfaces = get_device_access_interfaces(self.devid, auth, url)
        self.trunkinterfaces = get_trunk_interfaces(self.devid, auth, url)
        self.alarm = get_dev_alarms(self.devid, auth, url)
        self.numalarm = len(get_dev_alarms(self.devid, auth, url))
        self.assets = get_dev_asset_details(self.ip, auth, url)
        self.serials = [({'name' : asset['name'], 'serialNum': asset['serialNum']}) for asset in self.assets]
        self.runconfig = get_dev_run_config(self.devid, auth, url)
        self.startconfig = get_dev_start_config(self.devid, auth, url)
        self.ipmacarp = get_ip_mac_arp_list(self.devid, auth, url)
        self.auth = auth
        self.url = url

    def getvlans(self):
        """
        Function operates on the IMCDev object and updates the vlans attribute
        :return:
        """
        self.vlans = get_dev_vlans(self.devid, self.auth, self.url)

    def addvlan(self, vlanid, vlan_name,auth, url):
        """
        Function operates on the IMCDev object. Takes input of vlanid (1-4094), str of vlan_name,
        auth and url to execute the create_dev_vlan method on the IMCDev object. Device must be
        supported in the HPE IMC Platform VLAN Manager module.
        :param vlanid: str of VLANId ( valid 1-4094 )
        :param vlan_name: str of vlan_name
        :param auth: requests auth object #usually auth.creds from auth pyhpeimc.auth.class
        :param url: base url of IMC RS interface #usually auth.url from pyhpeimc.auth.authclass
        :return:
        """
        create_dev_vlan(self.devid, vlanid, vlan_name, auth, url)

    def delvlan(self, vlanid, auth, url):
        """
        Function operates on the IMCDev object. Takes input of vlanid (1-4094),
        auth and url to execute the delete_dev_vlans method on the IMCDev object. Device must be
        supported in the HPE IMC Platform VLAN Manager module.
        :param vlanid: str of VLANId ( valid 1-4094 )
        :param auth: requests auth object #usually auth.creds from auth pyhpeimc.auth.class
        :param url: base url of IMC RS interface #usually auth.url from pyhpeimc.auth.authclass
        :return:
        """
        delete_dev_vlans(self.devid, vlanid, auth, url)

    def getipmacarp(self):
        """
        Function operates on the IMCDev object and updates the ipmacarp attribute
        :return:
        """
        self.ipmacarp = get_ip_mac_arp_list(self.devid, auth, url)

class IMCInterface:
    def __init__(self, ip_address, ifIndex, auth, url):
        self.ip = get_dev_details(ip_address,auth, url)['ip']
        self.devid = get_dev_details(ip_address,auth, url)['id']
        self.ifIndex = get_interface_details(self.devid, ifIndex,auth, url)['ifIndex']
        self.macaddress = get_interface_details(self.devid, ifIndex,auth, url)['phyAddress']
        self.status = get_interface_details(self.devid, ifIndex,auth, url)['statusDesc']
        self.adminstatus = get_interface_details(self.devid, ifIndex,auth, url)['adminStatusDesc']
        self.name = get_interface_details(self.devid, ifIndex,auth, url)['ifDescription']
        self.description = get_interface_details(self.devid, ifIndex,auth, url)['ifAlias']
        self.mtu = get_interface_details(self.devid, ifIndex,auth, url)['mtu']
        self.speed = get_interface_details(self.devid, ifIndex,auth, url)['ifspeed']
        self.accessinterfaces = get_device_access_interfaces(self.devid,auth, url)
        self.pvid = get_access_interface_vlan(self.ifIndex, self.accessinterfaces,auth, url)
        self.auth = auth
        self.url = url

class IPScope:
    def __init__(self, netaddr, auth, url):
        self.id = get_scope_id(netaddr, auth, url)
        self.details = get_ip_scope_detail(auth, url, self.id)
        self.hosts = get_ip_scope_hosts(auth, url, self.id)
        self.auth = auth
        self.url = url
        self.netaddr = ipaddress.ip_network(netaddr)
        self.startip = get_ip_scope_detail(auth, url, self.id)['startIp']
        self.endip = get_ip_scope_detail(auth, url, self.id)['endIp']
        if 'assignedIpScope' in self.details:
            self.child = get_ip_scope_detail(auth, url, self.id)['assignedIpScope']

    def allocateIp(self, ipaddress, name, description):
        add_scope_ip(ipaddress, name, description, self.id, self.auth, self.url)

    def deallocateIp(self, hostid):
        remove_scope_ip( hostid, self.auth, self.url)

    def gethosts(self):
        self.hosts = get_ip_scope_hosts(self.auth, self.url, self.id)

    def nextfreeip(self):
        allocated_ips = [ ipaddress.ip_address(host['ip']) for host in self.hosts]
        for ip in self.netaddr:
            if str(ip).split('.')[-1] == '0':
                continue
            if ip not in allocated_ips:
                return ip

    def addchild(self, startIp, endIp, name, description):
        add_child_ip_scope(self.auth, self.url,startIp, endIp, name, description,self.id)

<|endoftext|>


<commit_before># coding: utf-8

"""
This module is used to estimate the cost of various compounds. Costs are taken
from the a CostDB instance, for example a CSV file via CostDBCSV.
For compounds with no cost listed, a Phase Diagram style convex hull
optimization is performed to determine a set of compositions that can be mixed
to give the desired compound with lowest total cost.
"""

from __future__ import division, unicode_literals
import abc
from collections import defaultdict
import csv
import os
import itertools
from monty.design_patterns import singleton
from monty.string import unicode2str
import six
from pymatgen import Composition, Element
from pymatgen.core.physical_constants import AVOGADROS_CONST
from pymatgen.matproj.snl import is_valid_bibtex
from pymatgen.phasediagram.entries import PDEntry
from pymatgen.phasediagram.pdanalyzer import PDAnalyzer
from pymatgen.phasediagram.pdmaker import PhaseDiagram
from io import open

__author__ = 'Anubhav Jain'
__copyright__ = 'Copyright 2013, The Materials Project'
__version__ = '0.1'
__maintainer__ = 'Anubhav Jain'
__email__ = 'ajain@lbl.gov'
__date__ = 'Aug 27, 2013'

module_dir = os.path.dirname(os.path.abspath(__file__))

class CostEntry(PDEntry):
    """
    Extends PDEntry to include a BibTeX reference and include language about
    cost
    """

    def __init__(self, composition, cost, name, reference):
        """
        Args:
            composition:
                Composition as a pymatgen.core.structure.Composition
            cost:
                Cost (per mol, NOT per kg) of the full Composition
            name:
                Optional parameter to name the entry. Defaults to the reduced
                chemical formula as in PDEntry.
            reference:
                Reference data as BiBTeX string
        """
        super(CostEntry, self).__init__(composition, cost, name)
        if reference and not is_valid_bibtex(reference):
            raise ValueError(
                "Invalid format for cost reference! Should be BibTeX string.")
        self.reference = reference

    def __repr__(self):
        return "CostEntry : {} with cost = {:.4f}".format(self.composition,
                                                          self.energy)

class CostDB(six.with_metaclass(abc.ABCMeta)):
    """
    Abstract class for representing a Cost database.
    Can be extended, e.g. for file-based or REST-based databases
    """

    @abc.abstractmethod
    def get_entries(self, chemsys):
        """
        For a given chemical system, return an array of CostEntries

        Args:
            chemsys:
                array of Elements defining the chemical system.

        Returns:
            array of CostEntries
        """
        return

class CostDBCSV(CostDB):
    """
    Read a CSV file to get costs
    Format is formula,cost_per_kg,name,BibTeX
    """

    def __init__(self, filename):
        # read in data from file
        self._chemsys_entries = defaultdict(list)
        filename = os.path.join(os.path.dirname(__file__), filename)
        reader = csv.reader(open(filename, "rt"), quotechar=unicode2str("|"))
        for row in reader:
            comp = Composition(row[0])
            cost_per_mol = float(row[1]) * comp.weight.to("kg") * \
                           AVOGADROS_CONST
            pde = CostEntry(comp.formula, cost_per_mol, row[2], row[3])
            chemsys = "-".join(sorted([el.symbol
                                       for el in pde.composition.elements]))
            self._chemsys_entries[chemsys].append(pde)

    def get_entries(self, chemsys):
        chemsys = "-".join(sorted([el.symbol for el in chemsys]))
        return self._chemsys_entries[chemsys]

@singleton
class CostDBElements(CostDBCSV):
    """
    Singleton object that provides the cost data for elements
    """
    def __init__(self):
        CostDBElements.__init__(
            self, os.path.join(module_dir, "costdb_elements.csv"))

class CostAnalyzer(object):
    """
    Given a CostDB, figures out the minimum cost solutions via convex hull
    """

    def __init__(self, costdb):
        self.costdb = costdb

    def get_lowest_decomposition(self, composition):
        """
        Get the decomposition leading to lowest cost

        Args:
            composition:
                Composition as a pymatgen.core.structure.Composition
        Returns:
            Decomposition as a dict of {Entry: amount}
        """

        entries_list = []
        elements = [e.symbol for e in composition.elements]
        for i in range(len(elements)):
            for combi in itertools.combinations(elements, i + 1):
                chemsys = [Element(e) for e in combi]
                x = self.costdb.get_entries(chemsys)
                entries_list.extend(x)
        try:
            pd = PhaseDiagram(entries_list)
            return PDAnalyzer(pd).get_decomposition(composition)
        except IndexError:
            raise ValueError("Error during PD building; most likely, "
                             "cost data does not exist!")

    def get_cost_per_mol(self, comp):
        """
        Get best estimate of minimum cost/mol based on known data

        Args:
            comp:
                Composition as a pymatgen.core.structure.Composition
        Returns:
            float of cost/mol
        """

        comp = comp if isinstance(comp, Composition) else Composition(comp)
        decomp = self.get_lowest_decomposition(comp)
        return sum(k.energy_per_atom * v * comp.num_atoms for k, v in
                   decomp.items())

    def get_cost_per_kg(self, comp):
        """
        Get best estimate of minimum cost/kg based on known data

        Args:
            comp:
                Composition as a pymatgen.core.structure.Composition
        Returns:
            float of cost/kg
        """
        comp = comp if isinstance(comp, Composition) else Composition(comp)
        return self.get_cost_per_mol(comp) / (
            comp.weight.to("kg") * AVOGADROS_CONST)<commit_msg>fix cost again.<commit_after># coding: utf-8

"""
This module is used to estimate the cost of various compounds. Costs are taken
from the a CostDB instance, for example a CSV file via CostDBCSV.
For compounds with no cost listed, a Phase Diagram style convex hull
optimization is performed to determine a set of compositions that can be mixed
to give the desired compound with lowest total cost.
"""

from __future__ import division, unicode_literals
import abc
from collections import defaultdict
import csv
import os
import itertools
from monty.design_patterns import singleton
from monty.string import unicode2str
import six
from pymatgen import Composition, Element
from pymatgen.core.physical_constants import AVOGADROS_CONST
from pymatgen.matproj.snl import is_valid_bibtex
from pymatgen.phasediagram.entries import PDEntry
from pymatgen.phasediagram.pdanalyzer import PDAnalyzer
from pymatgen.phasediagram.pdmaker import PhaseDiagram
from io import open

__author__ = 'Anubhav Jain'
__copyright__ = 'Copyright 2013, The Materials Project'
__version__ = '0.1'
__maintainer__ = 'Anubhav Jain'
__email__ = 'ajain@lbl.gov'
__date__ = 'Aug 27, 2013'

module_dir = os.path.dirname(os.path.abspath(__file__))

class CostEntry(PDEntry):
    """
    Extends PDEntry to include a BibTeX reference and include language about
    cost
    """

    def __init__(self, composition, cost, name, reference):
        """
        Args:
            composition:
                Composition as a pymatgen.core.structure.Composition
            cost:
                Cost (per mol, NOT per kg) of the full Composition
            name:
                Optional parameter to name the entry. Defaults to the reduced
                chemical formula as in PDEntry.
            reference:
                Reference data as BiBTeX string
        """
        super(CostEntry, self).__init__(composition, cost, name)
        if reference and not is_valid_bibtex(reference):
            raise ValueError(
                "Invalid format for cost reference! Should be BibTeX string.")
        self.reference = reference

    def __repr__(self):
        return "CostEntry : {} with cost = {:.4f}".format(self.composition,
                                                          self.energy)

class CostDB(six.with_metaclass(abc.ABCMeta)):
    """
    Abstract class for representing a Cost database.
    Can be extended, e.g. for file-based or REST-based databases
    """

    @abc.abstractmethod
    def get_entries(self, chemsys):
        """
        For a given chemical system, return an array of CostEntries

        Args:
            chemsys:
                array of Elements defining the chemical system.

        Returns:
            array of CostEntries
        """
        return

class CostDBCSV(CostDB):
    """
    Read a CSV file to get costs
    Format is formula,cost_per_kg,name,BibTeX
    """

    def __init__(self, filename):
        # read in data from file
        self._chemsys_entries = defaultdict(list)
        filename = os.path.join(os.path.dirname(__file__), filename)
        reader = csv.reader(open(filename, "rt"), quotechar=unicode2str("|"))
        for row in reader:
            comp = Composition(row[0])
            cost_per_mol = float(row[1]) * comp.weight.to("kg") * \
                           AVOGADROS_CONST
            pde = CostEntry(comp.formula, cost_per_mol, row[2], row[3])
            chemsys = "-".join(sorted([el.symbol
                                       for el in pde.composition.elements]))
            self._chemsys_entries[chemsys].append(pde)

    def get_entries(self, chemsys):
        chemsys = "-".join(sorted([el.symbol for el in chemsys]))
        return self._chemsys_entries[chemsys]

@singleton
class CostDBElements(CostDBCSV):
    """
    Singleton object that provides the cost data for elements
    """
    def __init__(self):
        CostDBCSV.__init__(
            self, os.path.join(module_dir, "costdb_elements.csv"))

class CostAnalyzer(object):
    """
    Given a CostDB, figures out the minimum cost solutions via convex hull
    """

    def __init__(self, costdb):
        self.costdb = costdb

    def get_lowest_decomposition(self, composition):
        """
        Get the decomposition leading to lowest cost

        Args:
            composition:
                Composition as a pymatgen.core.structure.Composition
        Returns:
            Decomposition as a dict of {Entry: amount}
        """

        entries_list = []
        elements = [e.symbol for e in composition.elements]
        for i in range(len(elements)):
            for combi in itertools.combinations(elements, i + 1):
                chemsys = [Element(e) for e in combi]
                x = self.costdb.get_entries(chemsys)
                entries_list.extend(x)
        try:
            pd = PhaseDiagram(entries_list)
            return PDAnalyzer(pd).get_decomposition(composition)
        except IndexError:
            raise ValueError("Error during PD building; most likely, "
                             "cost data does not exist!")

    def get_cost_per_mol(self, comp):
        """
        Get best estimate of minimum cost/mol based on known data

        Args:
            comp:
                Composition as a pymatgen.core.structure.Composition
        Returns:
            float of cost/mol
        """

        comp = comp if isinstance(comp, Composition) else Composition(comp)
        decomp = self.get_lowest_decomposition(comp)
        return sum(k.energy_per_atom * v * comp.num_atoms for k, v in
                   decomp.items())

    def get_cost_per_kg(self, comp):
        """
        Get best estimate of minimum cost/kg based on known data

        Args:
            comp:
                Composition as a pymatgen.core.structure.Composition
        Returns:
            float of cost/kg
        """
        comp = comp if isinstance(comp, Composition) else Composition(comp)
        return self.get_cost_per_mol(comp) / (
            comp.weight.to("kg") * AVOGADROS_CONST)<|endoftext|>


<commit_before>from main import *

class FeedHandlerBase(CachedPage):
    selfurl = "feed"
    mimeType = "application/atom+xml"
    def generatePage(self):
        output = []
        output.append( """<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
 <title>Mathblogging.org</title>
 <link href="http://www.mathblogging.org/%(url)s" rel="self"/>
 <link href="http://www.mathblogging.org/"/>
 <updated>2010-11-10T21:50:23-05:00</updated>
 <id>http://www.mathblogging.org/</id>
 <author>
   <name>mathblogging.org</name>
   <email>mathblogging.network@gmail.com</email>
 </author>""" % {'url':self.selfurl})
        for entry in self.query():
            output.append(""" <entry>
   <title> [[%(service)s]] %(title)s </title>
   <author>
      <name>%(service)s</name>
      <uri>%(homepage)s</uri>
   </author>

   <link href="%(link)s"/>
   <published>%(tcreated)s</published>
   <updated>%(tupdated)s</updated>
   <id>%(link)s</id>
   <content type="html"> %(content_trunc)s [Length: approx %(length)d words.] </content>
 </entry>""" % {'service': html_escape(entry.service),'title': entry.title, 'homepage': entry.homepage , 'link':entry.link, 'tcreated': entry.printTime_created_rfc3339(), 'tupdated':entry.printTime_updated_rfc3339(), 'length': entry.length/5, 'content_trunc': truncate_html_words(entry.content, 40)} )
        output.append("</feed>")
        return "".join(output)

class FeedHandlerAll(FeedHandlerBase):
    selfurl = "feed_all"
    cacheName = "FeedAll"
    def query(self):
        return Post.gql("ORDER BY timestamp_created DESC LIMIT 150")

class FeedHandlerCategory(FeedHandlerBase):
    def query(self):
        return Post.gql("WHERE category = :1 ORDER BY timestamp_created DESC LIMIT 150", self.thecategory)

class FeedHandlerCategories(FeedHandlerBase):
    def query(self):
        return Post.gql("WHERE category IN :1 ORDER BY timestamp_created DESC LIMIT 150", self.thecategories)

class PlanetMOfeed(FeedHandlerBase):
    selfurl = "planetmo-feed"
    cacheName = "PlanetMOfeed"
    def query(self):
        return Post.gql("WHERE tags IN :1 ORDER BY timestamp_created DESC LIMIT 150", ['Planetmo','Mathoverflow', 'Math Overflow','Mo','MO'])

class FeedHandlerResearchers(FeedHandlerCategories):
    selfurl = "feed_researchers"
    cacheName = "FeedResearchers"
    thecategories = ['pure','applied','history']

class FeedHandlerPure(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedPure"
    thecategory = "pure"

class FeedHandlerApplied(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedApplied"
    thecategory = "applied"

class FeedHandlerHistory(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedHistory"
    thecategory = "history"

class FeedHandlerVisual(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedVisual"
    thecategory = "visual"

class FeedHandlerArt(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedArt"
    thecategory = "art"

class FeedHandlerFun(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedFun"
    thecategory = "fun"

class FeedHandlerGeneral(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedGeneral"
    thecategory = "general"

class FeedHandlerJournals(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedJournals"
    thecategory = "journal"

class FeedHandlerTeachers(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedTeachers"
    thecategory = "teacher"

class FeedHandlerJournalism(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedJournalism"
    thecategory = "journalism"

class FeedHandlerInstitutions(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedInstitutions"
    thecategory = "institution"

class FeedHandlerCommunities(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedCommunities"
    thecategory = "community"

class FeedHandlerCommercial(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedCommercial"
    thecategory = "commercial"

class FeedHandlerNewssite(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedNewssite"
    thecategory = "news"

class FeedHandlerCarnival(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedCarnival"
    thecategory = "carnival"

class FeedHandlerPeople(FeedHandlerCategories):
    selfurl = ""
    cacheName = "FeedPeople"
    thecategories = ["pure","applied","visual","history","fun","teacher","journalism","general","art"]
<commit_msg>add nbsp entity declaration to atom feeds<commit_after>from main import *

class FeedHandlerBase(CachedPage):
    selfurl = "feed"
    mimeType = "application/atom+xml"
    def generatePage(self):
        output = []
        output.append( """<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE feed [<!ENTITY nbsp " ">]>
<feed xmlns="http://www.w3.org/2005/Atom">
 <title>Mathblogging.org</title>
 <link href="http://www.mathblogging.org/%(url)s" rel="self"/>
 <link href="http://www.mathblogging.org/"/>
 <updated>2010-11-10T21:50:23-05:00</updated>
 <id>http://www.mathblogging.org/</id>
 <author>
   <name>mathblogging.org</name>
   <email>mathblogging.network@gmail.com</email>
 </author>""" % {'url':self.selfurl})
        for entry in self.query():
            output.append(""" <entry>
   <title> [[%(service)s]] %(title)s </title>
   <author>
      <name>%(service)s</name>
      <uri>%(homepage)s</uri>
   </author>

   <link href="%(link)s"/>
   <published>%(tcreated)s</published>
   <updated>%(tupdated)s</updated>
   <id>%(link)s</id>
   <content type="html"> %(content_trunc)s [Length: approx %(length)d words.] </content>
 </entry>""" % {'service': html_escape(entry.service),'title': entry.title, 'homepage': entry.homepage , 'link':entry.link, 'tcreated': entry.printTime_created_rfc3339(), 'tupdated':entry.printTime_updated_rfc3339(), 'length': entry.length/5, 'content_trunc': truncate_html_words(entry.content, 40)} )
        output.append("</feed>")
        return "".join(output)

class FeedHandlerAll(FeedHandlerBase):
    selfurl = "feed_all"
    cacheName = "FeedAll"
    def query(self):
        return Post.gql("ORDER BY timestamp_created DESC LIMIT 150")

class FeedHandlerCategory(FeedHandlerBase):
    def query(self):
        return Post.gql("WHERE category = :1 ORDER BY timestamp_created DESC LIMIT 150", self.thecategory)

class FeedHandlerCategories(FeedHandlerBase):
    def query(self):
        return Post.gql("WHERE category IN :1 ORDER BY timestamp_created DESC LIMIT 150", self.thecategories)

class PlanetMOfeed(FeedHandlerBase):
    selfurl = "planetmo-feed"
    cacheName = "PlanetMOfeed"
    def query(self):
        return Post.gql("WHERE tags IN :1 ORDER BY timestamp_created DESC LIMIT 150", ['Planetmo','Mathoverflow', 'Math Overflow','Mo','MO'])

class FeedHandlerResearchers(FeedHandlerCategories):
    selfurl = "feed_researchers"
    cacheName = "FeedResearchers"
    thecategories = ['pure','applied','history']

class FeedHandlerPure(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedPure"
    thecategory = "pure"

class FeedHandlerApplied(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedApplied"
    thecategory = "applied"

class FeedHandlerHistory(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedHistory"
    thecategory = "history"

class FeedHandlerVisual(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedVisual"
    thecategory = "visual"

class FeedHandlerArt(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedArt"
    thecategory = "art"

class FeedHandlerFun(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedFun"
    thecategory = "fun"

class FeedHandlerGeneral(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedGeneral"
    thecategory = "general"

class FeedHandlerJournals(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedJournals"
    thecategory = "journal"

class FeedHandlerTeachers(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedTeachers"
    thecategory = "teacher"

class FeedHandlerJournalism(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedJournalism"
    thecategory = "journalism"

class FeedHandlerInstitutions(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedInstitutions"
    thecategory = "institution"

class FeedHandlerCommunities(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedCommunities"
    thecategory = "community"

class FeedHandlerCommercial(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedCommercial"
    thecategory = "commercial"

class FeedHandlerNewssite(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedNewssite"
    thecategory = "news"

class FeedHandlerCarnival(FeedHandlerCategory):
    selfurl = ""
    cacheName = "FeedCarnival"
    thecategory = "carnival"

class FeedHandlerPeople(FeedHandlerCategories):
    selfurl = ""
    cacheName = "FeedPeople"
    thecategories = ["pure","applied","visual","history","fun","teacher","journalism","general","art"]
<|endoftext|>


<commit_before>import unittest

import cupy
from cupy import _util
from cupy import core
from cupy import testing

# TODO(leofang): test PTDS in this file

class DummyObjectWithCudaArrayInterface(object):

    def __init__(self, a):
        self.a = a

    @property
    def __cuda_array_interface__(self):
        stream = cupy.cuda.get_current_stream()
        if stream.ptr == 0:
            if _util.CUDA_ARRAY_INTERFACE_SYNC:
                stream_ptr = 1
            else:
                stream_ptr = None
        else:
            stream_ptr = stream.ptr
        desc = {
            'shape': self.a.shape,
            'strides': self.a.strides,
            'typestr': self.a.dtype.str,
            'descr': self.a.dtype.descr,
            'data': (self.a.data.ptr, False),
            'stream': stream_ptr,
            'version': 3,
        }
        return desc

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
}))
@testing.gpu
class TestArrayUfunc(unittest.TestCase):

    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
    @testing.numpy_cupy_allclose(rtol=1e-6, accept_error=TypeError,
                                 contiguous_check=False)
    def check_array_scalar_op(self, op, xp, x_type, y_type, trans=False):
        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
        if trans:
            a = a.T

        if xp is cupy:
            a = DummyObjectWithCudaArrayInterface(a)
        return getattr(xp, op)(a, y_type(3))

    def test_add_scalar(self):
        self.check_array_scalar_op('add')

    def test_add_scalar_with_strides(self):
        self.check_array_scalar_op('add', trans=True)

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
}))
@testing.gpu
class TestElementwiseKernel(unittest.TestCase):

    @testing.for_all_dtypes_combination()
    @testing.numpy_cupy_allclose(rtol=1e-6, accept_error=TypeError,
                                 contiguous_check=False)
    def check_array_scalar_op(self, op, xp, dtyes, trans=False):
        a = xp.array([[1, 2, 3], [4, 5, 6]], dtyes)
        if trans:
            a = a.T

        if xp is cupy:
            a = DummyObjectWithCudaArrayInterface(a)
            f = cupy.ElementwiseKernel('T x, T y', 'T z', 'z = x + y')
            return f(a, dtyes(3))
        else:
            return a + dtyes(3)

    def test_add_scalar(self):
        self.check_array_scalar_op('add')

    def test_add_scalar_with_strides(self):
        self.check_array_scalar_op('add', trans=True)

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
}))
@testing.gpu
class SimpleReductionFunction(unittest.TestCase):

    def setUp(self):
        self.my_int8_sum = core.create_reduction_func(
            'my_sum', ('b->b',), ('in0', 'a + b', 'out0 = a', None))

    @testing.numpy_cupy_allclose()
    def check_int8_sum(self, shape, xp, axis=None, keepdims=False,
                       trans=False):
        a = testing.shaped_random(shape, xp, 'b')
        if trans:
            a = a.T

        if xp == cupy:
            a = DummyObjectWithCudaArrayInterface(a)
            return self.my_int8_sum(
                a, axis=axis, keepdims=keepdims)
        else:
            return a.sum(axis=axis, keepdims=keepdims, dtype='b')

    def test_shape(self):
        self.check_int8_sum((2 ** 10,))

    def test_shape_with_strides(self):
        self.check_int8_sum((2 ** 10, 16), trans=True)

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
}))
@testing.gpu
class TestReductionKernel(unittest.TestCase):

    def setUp(self):
        self.my_sum = core.ReductionKernel(
            'T x', 'T out', 'x', 'a + b', 'out = a', '0', 'my_sum')

    @testing.numpy_cupy_allclose()
    def check_int8_sum(self, shape, xp, axis=None, keepdims=False,
                       trans=False):
        a = testing.shaped_random(shape, xp, 'b')
        if trans:
            a = a.T

        if xp == cupy:
            a = DummyObjectWithCudaArrayInterface(a)
            return self.my_sum(
                a, axis=axis, keepdims=keepdims)
        else:
            return a.sum(axis=axis, keepdims=keepdims, dtype='b')

    def test_shape(self):
        self.check_int8_sum((2 ** 10,))

    def test_shape_with_strides(self):
        self.check_int8_sum((2 ** 10, 16), trans=True)

@testing.parameterize(
    {'shape': (10,), 'slices': (slice(0, None),)},
    {'shape': (10,), 'slices': (slice(2, None),)},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(0, None))},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(2, None))},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(0, None))},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(2, None))},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(4, None))},
)
@testing.gpu
class TestSlicingMemoryPointer(unittest.TestCase):

    @testing.for_all_dtypes_combination(names=['dtype'])
    @testing.for_orders('CF')
    def test_shape_with_strides(self, dtype, order):
        x = cupy.zeros(self.shape, dtype=dtype, order=order)

        start = [s.start for s in self.slices]
        itemsize = cupy.dtype(dtype).itemsize
        dimsize = [s * itemsize for s in start]
        if len(self.shape) == 1:
            offset = start[0] * itemsize
        else:
            if order == 'C':
                offset = self.shape[0] * dimsize[0] + dimsize[1]
            else:
                offset = self.shape[0] * dimsize[1] + dimsize[0]

        cai_ptr, _ = x.__cuda_array_interface__['data']
        slice_cai_ptr, _ = x[self.slices].__cuda_array_interface__['data']
        cupy_data_ptr = x.data.ptr
        sliced_cupy_data_ptr = x[self.slices].data.ptr

        assert cai_ptr == cupy_data_ptr
        assert slice_cai_ptr == sliced_cupy_data_ptr
        assert slice_cai_ptr == cai_ptr+offset

@testing.parameterize(
    {'shape': (10,), 'slices': (slice(0, None),), 'stream': cupy.cuda.Stream.null},
    {'shape': (10,), 'slices': (slice(2, None),), 'stream': cupy.cuda.Stream.null},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(0, None)), 'stream': cupy.cuda.Stream.null},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(2, None)), 'stream': cupy.cuda.Stream.null},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(0, None)), 'stream': cupy.cuda.Stream.null},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(2, None)), 'stream': cupy.cuda.Stream.null},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(4, None)), 'stream': cupy.cuda.Stream.null},
    {'shape': (10,), 'slices': (slice(0, None),), 'stream': cupy.cuda.Stream()},
    {'shape': (10,), 'slices': (slice(2, None),), 'stream': cupy.cuda.Stream()},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(0, None)), 'stream': cupy.cuda.Stream()},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(2, None)), 'stream': cupy.cuda.Stream()},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(0, None)), 'stream': cupy.cuda.Stream()},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(2, None)), 'stream': cupy.cuda.Stream()},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(4, None)), 'stream': cupy.cuda.Stream()},
)
@testing.gpu
class TestCUDAArrayInterfaceCompliance(unittest.TestCase):

    @testing.for_all_dtypes_combination(names=['dtype'])
    @testing.for_orders('CF')
    def test_value_type(self, dtype, order):
        x = cupy.zeros(self.shape, dtype=dtype, order=order)
        y = x[self.slices]

        # mandatory entries
        shape = y.__cuda_array_interface__['shape']
        typestr = y.__cuda_array_interface__['typestr']
        ptr, readonly = y.__cuda_array_interface__['data']
        version = y.__cuda_array_interface__['version']
        strides = y.__cuda_array_interface__['strides']

        # optional entries
        if 'descr' in y.__cuda_array_interface__:
            descr = y.__cuda_array_interface__['descr']
        else:
            descr = None
        with self.stream:
            if 'stream' in y.__cuda_array_interface__:
                stream = y.__cuda_array_interface__['stream']
            else:
                stream = None

        # Don't validate correctness of data here, just their types
        assert version == 3  # bump this when the protocol is updated!
        assert isinstance(y.__cuda_array_interface__, dict)
        assert isinstance(shape, tuple)
        assert isinstance(typestr, str)
        assert isinstance(ptr, int)
        assert isinstance(readonly, bool)
        assert (strides is None) or isinstance(strides, tuple)
        assert (descr is None) or isinstance(descr, list)
        if isinstance(descr, list):
            for item in descr:
                assert isinstance(item, tuple)
        assert (stream is None) or isinstance(stream, int)

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
    'sync': (True, False),
}))
@testing.gpu
class TestCUDAArrayInterfaceStream(unittest.TestCase):
    def setUp(self):
        self.sync_config = _util.CUDA_ARRAY_INTERFACE_SYNC
        _util.CUDA_ARRAY_INTERFACE_SYNC = self.sync

    def tearDown(self):
        _util.CUDA_ARRAY_INTERFACE_SYNC = self.sync_config

    def test_stream_export(self):
        a = cupy.empty(100)

        # the stream context should export the stream
        with self.stream:
            stream_ptr = a.__cuda_array_interface__['stream']
        if self.stream is cupy.cuda.Stream.null:
            if self.sync:
                assert stream_ptr == 1
            else:
                assert stream_ptr is None
        else:
            assert stream_ptr == self.stream.ptr

        # without a stream context, it's always the default stream
        stream_ptr = a.__cuda_array_interface__['stream']
        if self.sync:
            assert stream_ptr == 1
        else:
            assert stream_ptr is None
<commit_msg>flake8<commit_after>import unittest

import cupy
from cupy import _util
from cupy import core
from cupy import testing

# TODO(leofang): test PTDS in this file

class DummyObjectWithCudaArrayInterface(object):

    def __init__(self, a):
        self.a = a

    @property
    def __cuda_array_interface__(self):
        stream = cupy.cuda.get_current_stream()
        if stream.ptr == 0:
            if _util.CUDA_ARRAY_INTERFACE_SYNC:
                stream_ptr = 1
            else:
                stream_ptr = None
        else:
            stream_ptr = stream.ptr
        desc = {
            'shape': self.a.shape,
            'strides': self.a.strides,
            'typestr': self.a.dtype.str,
            'descr': self.a.dtype.descr,
            'data': (self.a.data.ptr, False),
            'stream': stream_ptr,
            'version': 3,
        }
        return desc

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
}))
@testing.gpu
class TestArrayUfunc(unittest.TestCase):

    @testing.for_all_dtypes_combination(names=['x_type', 'y_type'])
    @testing.numpy_cupy_allclose(rtol=1e-6, accept_error=TypeError,
                                 contiguous_check=False)
    def check_array_scalar_op(self, op, xp, x_type, y_type, trans=False):
        a = xp.array([[1, 2, 3], [4, 5, 6]], x_type)
        if trans:
            a = a.T

        if xp is cupy:
            a = DummyObjectWithCudaArrayInterface(a)
        return getattr(xp, op)(a, y_type(3))

    def test_add_scalar(self):
        self.check_array_scalar_op('add')

    def test_add_scalar_with_strides(self):
        self.check_array_scalar_op('add', trans=True)

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
}))
@testing.gpu
class TestElementwiseKernel(unittest.TestCase):

    @testing.for_all_dtypes_combination()
    @testing.numpy_cupy_allclose(rtol=1e-6, accept_error=TypeError,
                                 contiguous_check=False)
    def check_array_scalar_op(self, op, xp, dtyes, trans=False):
        a = xp.array([[1, 2, 3], [4, 5, 6]], dtyes)
        if trans:
            a = a.T

        if xp is cupy:
            a = DummyObjectWithCudaArrayInterface(a)
            f = cupy.ElementwiseKernel('T x, T y', 'T z', 'z = x + y')
            return f(a, dtyes(3))
        else:
            return a + dtyes(3)

    def test_add_scalar(self):
        self.check_array_scalar_op('add')

    def test_add_scalar_with_strides(self):
        self.check_array_scalar_op('add', trans=True)

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
}))
@testing.gpu
class SimpleReductionFunction(unittest.TestCase):

    def setUp(self):
        self.my_int8_sum = core.create_reduction_func(
            'my_sum', ('b->b',), ('in0', 'a + b', 'out0 = a', None))

    @testing.numpy_cupy_allclose()
    def check_int8_sum(self, shape, xp, axis=None, keepdims=False,
                       trans=False):
        a = testing.shaped_random(shape, xp, 'b')
        if trans:
            a = a.T

        if xp == cupy:
            a = DummyObjectWithCudaArrayInterface(a)
            return self.my_int8_sum(
                a, axis=axis, keepdims=keepdims)
        else:
            return a.sum(axis=axis, keepdims=keepdims, dtype='b')

    def test_shape(self):
        self.check_int8_sum((2 ** 10,))

    def test_shape_with_strides(self):
        self.check_int8_sum((2 ** 10, 16), trans=True)

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
}))
@testing.gpu
class TestReductionKernel(unittest.TestCase):

    def setUp(self):
        self.my_sum = core.ReductionKernel(
            'T x', 'T out', 'x', 'a + b', 'out = a', '0', 'my_sum')

    @testing.numpy_cupy_allclose()
    def check_int8_sum(self, shape, xp, axis=None, keepdims=False,
                       trans=False):
        a = testing.shaped_random(shape, xp, 'b')
        if trans:
            a = a.T

        if xp == cupy:
            a = DummyObjectWithCudaArrayInterface(a)
            return self.my_sum(
                a, axis=axis, keepdims=keepdims)
        else:
            return a.sum(axis=axis, keepdims=keepdims, dtype='b')

    def test_shape(self):
        self.check_int8_sum((2 ** 10,))

    def test_shape_with_strides(self):
        self.check_int8_sum((2 ** 10, 16), trans=True)

@testing.parameterize(
    {'shape': (10,), 'slices': (slice(0, None),)},
    {'shape': (10,), 'slices': (slice(2, None),)},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(0, None))},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(2, None))},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(0, None))},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(2, None))},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(4, None))},
)
@testing.gpu
class TestSlicingMemoryPointer(unittest.TestCase):

    @testing.for_all_dtypes_combination(names=['dtype'])
    @testing.for_orders('CF')
    def test_shape_with_strides(self, dtype, order):
        x = cupy.zeros(self.shape, dtype=dtype, order=order)

        start = [s.start for s in self.slices]
        itemsize = cupy.dtype(dtype).itemsize
        dimsize = [s * itemsize for s in start]
        if len(self.shape) == 1:
            offset = start[0] * itemsize
        else:
            if order == 'C':
                offset = self.shape[0] * dimsize[0] + dimsize[1]
            else:
                offset = self.shape[0] * dimsize[1] + dimsize[0]

        cai_ptr, _ = x.__cuda_array_interface__['data']
        slice_cai_ptr, _ = x[self.slices].__cuda_array_interface__['data']
        cupy_data_ptr = x.data.ptr
        sliced_cupy_data_ptr = x[self.slices].data.ptr

        assert cai_ptr == cupy_data_ptr
        assert slice_cai_ptr == sliced_cupy_data_ptr
        assert slice_cai_ptr == cai_ptr+offset

test_cases = [
    {'shape': (10,), 'slices': (slice(0, None),)},
    {'shape': (10,), 'slices': (slice(2, None),)},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(0, None))},
    {'shape': (10, 10), 'slices': (slice(0, None), slice(2, None))},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(0, None))},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(2, None))},
    {'shape': (10, 10), 'slices': (slice(2, None), slice(4, None))}]
test_streams = (cupy.cuda.Stream.null, cupy.cuda.Stream())
test_cases_with_stream = [
    {'stream': s, **t} for t in test_cases for s in test_streams]

@testing.parameterize(*test_cases_with_stream)
@testing.gpu
class TestCUDAArrayInterfaceCompliance(unittest.TestCase):

    @testing.for_all_dtypes_combination(names=['dtype'])
    @testing.for_orders('CF')
    def test_value_type(self, dtype, order):
        x = cupy.zeros(self.shape, dtype=dtype, order=order)
        y = x[self.slices]

        # mandatory entries
        shape = y.__cuda_array_interface__['shape']
        typestr = y.__cuda_array_interface__['typestr']
        ptr, readonly = y.__cuda_array_interface__['data']
        version = y.__cuda_array_interface__['version']
        strides = y.__cuda_array_interface__['strides']

        # optional entries
        if 'descr' in y.__cuda_array_interface__:
            descr = y.__cuda_array_interface__['descr']
        else:
            descr = None
        with self.stream:
            if 'stream' in y.__cuda_array_interface__:
                stream = y.__cuda_array_interface__['stream']
            else:
                stream = None

        # Don't validate correctness of data here, just their types
        assert version == 3  # bump this when the protocol is updated!
        assert isinstance(y.__cuda_array_interface__, dict)
        assert isinstance(shape, tuple)
        assert isinstance(typestr, str)
        assert isinstance(ptr, int)
        assert isinstance(readonly, bool)
        assert (strides is None) or isinstance(strides, tuple)
        assert (descr is None) or isinstance(descr, list)
        if isinstance(descr, list):
            for item in descr:
                assert isinstance(item, tuple)
        assert (stream is None) or isinstance(stream, int)

@testing.parameterize(*testing.product({
    'stream': (cupy.cuda.Stream.null, cupy.cuda.Stream()),
    'sync': (True, False),
}))
@testing.gpu
class TestCUDAArrayInterfaceStream(unittest.TestCase):
    def setUp(self):
        self.sync_config = _util.CUDA_ARRAY_INTERFACE_SYNC
        _util.CUDA_ARRAY_INTERFACE_SYNC = self.sync

    def tearDown(self):
        _util.CUDA_ARRAY_INTERFACE_SYNC = self.sync_config

    def test_stream_export(self):
        a = cupy.empty(100)

        # the stream context should export the stream
        with self.stream:
            stream_ptr = a.__cuda_array_interface__['stream']
        if self.stream is cupy.cuda.Stream.null:
            if self.sync:
                assert stream_ptr == 1
            else:
                assert stream_ptr is None
        else:
            assert stream_ptr == self.stream.ptr

        # without a stream context, it's always the default stream
        stream_ptr = a.__cuda_array_interface__['stream']
        if self.sync:
            assert stream_ptr == 1
        else:
            assert stream_ptr is None
<|endoftext|>


<commit_before>'''This package controls syndication for the :mod:papers, mainly for :class:`~papers.models.Publication` objects.

The available feeds are as follows:

+--------------------+---------------------------+---------------------------------------------+
| Feed               | Feed Location             | Feed Class                                  |
+====================+===========================+=============================================+
| Laboratory papers  | /feeds/lab-papers         | :class:`~papers.feeds.LabPapersFeed         |
+--------------------+---------------------------+---------------------------------------------+
| Interesting papers | /feeds/interesting-papers | :class:`~papers.feeds.InterestingPapersFeed |
+--------------------+---------------------------+---------------------------------------------+

A main page describing all feeds is available at **/feeds**.
'''

from django.contrib.syndication.views import Feed
from django.core.urlresolvers import reverse
from django.conf import settings

from papers.models import Publication

class PapersFeed(Feed):
    '''This is the main class for all feeds related to publications.'''

    def item_title(self, item):
        return item.title

    def item_description(self, item):
        return item.abstract

class LabPapersFeed(PapersFeed):
    title = "Papers From the %s" % settings.LAB_NAME
    link = "/feeds/lab-papers"
    description = "This feed contains papers published by members of the %s." % settings.LAB_NAME

    def items(self):
        return Publication.objects.filter(laboratory_paper=True)

class InterestingPapersFeed(PapersFeed):
    title = "Papers of Interest"
    link = "/feeds/interesting-papers"
    description = "This feed contains papers that the %s is particularly interested in." % settings.LAB_NAME

    def items(self):
        return Publication.objects.filter(interesting_paper=True)<commit_msg>Documented paper feed objects.  Part of issue #18 and issue #12.<commit_after>'''This package controls syndication for the :mod:papers, mainly for :class:`~papers.models.Publication` objects.

The available feeds are as follows:

+--------------------+---------------------------+---------------------------------------------+
| Feed               | Feed Location             | Feed Class                                  |
+====================+===========================+=============================================+
| Laboratory papers  | /feeds/lab-papers         | :class:`~papers.feeds.LabPapersFeed         |
+--------------------+---------------------------+---------------------------------------------+
| Interesting papers | /feeds/interesting-papers | :class:`~papers.feeds.InterestingPapersFeed |
+--------------------+---------------------------+---------------------------------------------+

A main page describing all feeds is available at **/feeds**.
'''

from django.contrib.syndication.views import Feed
from django.core.urlresolvers import reverse
from django.conf import settings

from papers.models import Publication

class PapersFeed(Feed):
    '''This is the main class for all feeds related to :class:`~papers.models.Publication`.'''

    def item_title(self, item):
        '''The feed item title is the title of the paper.'''
        return item.title

    def item_description(self, item):
        '''The feed description is the abstract of the paper.'''
        return item.abstract

class LabPapersFeed(PapersFeed):
    '''This generates a feed for papers from this group.'''

    title = "Papers From the %s" % settings.LAB_NAME
    link = "/feeds/lab-papers"
    description = "This feed contains papers published by members of the %s." % settings.LAB_NAME

    def items(self):
        '''The items returned by this feed are papers set as laboratory papers.'''
        return Publication.objects.filter(laboratory_paper=True)

class InterestingPapersFeed(PapersFeed):
    '''This generates a feed for papers marked as interesting.'''

    title = "Papers of Interest"
    link = "/feeds/interesting-papers"
    description = "This feed contains papers that the %s is particularly interested in." % settings.LAB_NAME

    def items(self):
        '''The items returned by this feed are papers set as interesting.'''
        return Publication.objects.filter(interesting_paper=True)<|endoftext|>