<commit_before>#!/usr/bin/env python

def fixRange(data):
    """ Reduces the minimum so that the calculations are numerically stable """

    from numpy import log10, min, max, abs

    dataFactor = abs(log10(min(data)))
    scaled = data * 10**dataFactor

    fixed = True

    return scaled, dataFactor, fixed

def calculateRegression(x, y, type):
    """ Calculate three types of regression:
        1. Linear (lin)
        2. Log (log)
        3. Exponential (exp)
        4. Linear through zero (lin0)
    """

    import numpy as np
    from scipy import stats

    # Check the smallest value in x or y isn't below ~1x10-7 otherwise
    # we hit numerical instabilities.
    xFactorFix, xFactor = False, 0
    yFactorFix, yFactor = False, 0
    if min(x) < 2e-7:
        print 'Scaling x-data to improve numerical stability'
        x, xFactor, xFactorFix = fixRange(x)
    if min(y) < 2e-7:
        print 'Scaling y-data to improve numerical stability'
        y, yFactor, yFactorFix = fixRange(y)

    if type is 'lin0':
        xf = x
        x = x[:,np.newaxis] # make a singleton extra dimension
        m, _, _, _ = np.linalg.lstsq(x, y)
        c, r, p = 0, np.nan, np.nan
    elif type is 'lin':
        print 'lin'
        m, c, r, p, std_err = stats.linregress(x, y)
        xf = x
    elif type is 'log':
        m, c, r, p, std_err = stats.linregress(np.log10(x),y)
        xf = np.log10(x)
    elif type is 'exp':
        m, c, r, p, std_err = stats.linregress(x,np.log10(y))
        xf = x
    else:
        raise ValueError('Unknown regression type')

    yf = (m * xf) + c

    if xFactorFix:
        xf = xf / 10**xFactor
    if yFactorFix:
        yf = yf / 10**yFactor

    return xf, yf, m, c, r

def calculatePolyfit(x, y):
    """ Calculate a linear regression using polyfit instead """

    # FIXME(pica) This doesn't work for large ranges (tens of orders of
    # magnitude). No idea why as the fixRange() function above should make
    # all values greater than one. The trouble mainly seems to happen when
    # log10(min(x)) negative.

    from scipy import polyfit, polyval

    # Check the smallest value in x or y isn't below 2x10-7 otherwise
    # we hit numerical instabilities.
    xFactorFix, xFactor = False, 0
    yFactorFix, yFactor = False, 0
    if min(x) < 2e-7:
        x, xFactor, xFactorFix = fixRange(x)
    if min(y) < 2e-7:
        y, yFactor, yFactorFix = fixRange(y)

    (ar, br) = polyfit(x, y, 1)
    yf = polyval([ar, br], x)
    xf = x

    if xFactorFix:
        xf = xf / 10**xFactor
    if yFactorFix:
        yf = yf / 10**yFactor

    return xf, yf

if __name__ == '__main__':

    import numpy as np
    import matplotlib.pyplot as plt
    from scipy import randn

    n = 50
    x = np.logspace(-50, 30, n)
    y = np.logspace(-30, 50, n)

    linX, linY = calculateRegression(x, y, 'lin')
    #linX2, linY2 = calculatePolyfit(x, y)
    logX, logY = calculateRegression(x, y, 'log')
    expX, expY = calculateRegression(x, y, 'exp')

    plt.figure()
    plt.loglog(x, y, 'b-s', label = 'data')
    plt.loglog(linX, linY, 'g-x', label = 'linear')
    #plt.loglog(linX2, linY2, 'y-+', label = 'linear2')
    #plt.loglog(logX, logY, 'r', label = 'log')
    #plt.loglog(expX, expY, 'k-+', label = 'exponential')
    plt.legend(loc=2)

    plt.show()

<commit_msg>Fixed some bugs in the regression when presented with an int instead of a numpy array<commit_after>#!/usr/bin/env python

def fixRange(data):
    """ Reduces the minimum so that the calculations are numerically stable """

    from numpy import log10, min, max, abs

    dataFactor = abs(log10(min(data)))
    scaled = data * 10**dataFactor

    fixed = True

    return scaled, dataFactor, fixed

def calculateRegression(x, y, type):
    """ Calculate three types of regression:
        1. Linear (lin)
        2. Log (log)
        3. Exponential (exp)
        4. Linear through zero (lin0)
    """

    import numpy as np
    from scipy import stats

    # Check the smallest value in x or y isn't below ~1x10-7 otherwise
    # we hit numerical instabilities.
    xFactorFix, xFactor = False, 0
    yFactorFix, yFactor = False, 0
    minX = min(x)
    minY = min(y)
    if minX < 2e-7:
        print 'Scaling x-data to improve numerical stability'
        x, xFactor, xFactorFix = fixRange(x)
    if minY < 2e-7:
        print 'Scaling y-data to improve numerical stability'
        y, yFactor, yFactorFix = fixRange(y)

    if type is 'lin0':
        xf = x
        x = x[:,np.newaxis] # make a singleton extra dimension
        m, _, _, _ = np.linalg.lstsq(x, y)
        c, r, p = 0, np.nan, np.nan
    elif type is 'lin':
        print 'lin'
        m, c, r, p, std_err = stats.linregress(x, y)
        xf = x
    elif type is 'log':
        m, c, r, p, std_err = stats.linregress(np.log10(x),y)
        xf = np.log10(x)
    elif type is 'exp':
        m, c, r, p, std_err = stats.linregress(x,np.log10(y))
        xf = x
    else:
        raise ValueError('Unknown regression type')

    yf = (m * xf) + c

    if xFactorFix:
        xf = xf / 10**xFactor
    if yFactorFix:
        yf = yf / 10**yFactor

    return xf, yf, m, c, r

def calculatePolyfit(x, y):
    """ Calculate a linear regression using polyfit instead """

    # FIXME(pica) This doesn't work for large ranges (tens of orders of
    # magnitude). No idea why as the fixRange() function above should make
    # all values greater than one. The trouble mainly seems to happen when
    # log10(min(x)) negative.

    from scipy import polyfit, polyval

    # Check the smallest value in x or y isn't below 2x10-7 otherwise
    # we hit numerical instabilities.
    xFactorFix, xFactor = False, 0
    yFactorFix, yFactor = False, 0
    minX = min(x)
    minY = min(y)
    if minX < 2e-7:
        x, xFactor, xFactorFix = fixRange(x)
    if minY < 2e-7:
        y, yFactor, yFactorFix = fixRange(y)

    (ar, br) = polyfit(x, y, 1)
    yf = polyval([ar, br], x)
    xf = x

    if xFactorFix:
        xf = xf / 10**xFactor
    if yFactorFix:
        yf = yf / 10**yFactor

    return xf, yf

if __name__ == '__main__':

    import numpy as np
    import matplotlib.pyplot as plt
    from scipy import randn

    n = 50
    x = np.logspace(-50, 30, n)
    y = np.logspace(-30, 50, n)

    linX, linY = calculateRegression(x, y, 'lin')
    #linX2, linY2 = calculatePolyfit(x, y)
    logX, logY = calculateRegression(x, y, 'log')
    expX, expY = calculateRegression(x, y, 'exp')

    plt.figure()
    plt.loglog(x, y, 'b-s', label = 'data')
    plt.loglog(linX, linY, 'g-x', label = 'linear')
    #plt.loglog(linX2, linY2, 'y-+', label = 'linear2')
    #plt.loglog(logX, logY, 'r', label = 'log')
    #plt.loglog(expX, expY, 'k-+', label = 'exponential')
    plt.legend(loc=2)

    plt.show()

<|endoftext|>


<commit_before>
import ast
from tokenize import *

from collections import namedtuple
from StringIO import StringIO

from .tokenise import tokenise, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK

class TemplateSyntaxError(Exception):
    pass

### Back-ported from Py3
EXACT_TOKEN_TYPES = {
    '(':   LPAR,
    ')':   RPAR,
    '[':   LSQB,
    ']':   RSQB,
    ':':   COLON,
    ',':   COMMA,
    ';':   SEMI,
    '+':   PLUS,
    '-':   MINUS,
    '*':   STAR,
    '/':   SLASH,
    '|':   VBAR,
    '&':   AMPER,
    '<':   LESS,
    '>':   GREATER,
    '=':   EQUAL,
    '.':   DOT,
    '%':   PERCENT,
    '{':   LBRACE,
    '}':   RBRACE,
    '==':  EQEQUAL,
    '!=':  NOTEQUAL,
    '<=':  LESSEQUAL,
    '>=':  GREATEREQUAL,
    '~':   TILDE,
    '^':   CIRCUMFLEX,
    '<<':  LEFTSHIFT,
    '>>':  RIGHTSHIFT,
    '**':  DOUBLESTAR,
    '+=':  PLUSEQUAL,
    '-=':  MINEQUAL,
    '*=':  STAREQUAL,
    '/=':  SLASHEQUAL,
    '%=':  PERCENTEQUAL,
    '&=':  AMPEREQUAL,
    '|=':  VBAREQUAL,
    '^=': CIRCUMFLEXEQUAL,
    '<<=': LEFTSHIFTEQUAL,
    '>>=': RIGHTSHIFTEQUAL,
    '**=': DOUBLESTAREQUAL,
    '//':  DOUBLESLASH,
    '//=': DOUBLESLASHEQUAL,
    '@':   AT
}

class TokenInfo(namedtuple('TokenInfo', 'type string start end line')):
    def __repr__(self):
        annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
        return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
                self._replace(type=annotated_type))

    @property
    def exact_type(self):
        if self.type == OP and self.string in EXACT_TOKEN_TYPES:
            return EXACT_TOKEN_TYPES[self.string]
        else:
            return self.type
###

def token_stream(content):
    '''Fix the hole in Py2, change token types to their real types'''
    for token in generate_tokens(StringIO(content).readline):
        yield TokenInfo(*token)

def _context_lookup(x):
    '''Return AST for looking up x in the Context'''
    return ast.Subscript(
        value=ast.Name(id='context', ctx=ast.Load()),
        slice=ast.Index(value=ast.Str(s=x), ctx=ast.Load()),
        ctx=ast.Load(),
    )

def _make_number(tok):
    '''Turn a token of type NUMBER into an ast.Num'''
    if '.' in tok.string:
        val = float(tok.string)
    else:
        val = int(tok.string)
    return ast.Num(n=val)

def _convert_name_or_literal(tok):
    '''Turn STRING, NUMBER or NAME tokens into equivalent AST node.'''
    if tok.exact_type == STRING:
        code = ast.Str(s=tok.string[1:-1])
    elif tok.exact_type == NUMBER:
        code = _make_number(tok)
    elif tok.exact_type == NAME:
        code = _context_lookup(tok.string)
    else:
        raise TemplateSyntaxError(content)
    return code

def parse_expr(content):
    '''Turn a content string into AST'''
    stream = token_stream(content)

    # First token MUST be either a literal, or name
    tok = next(stream)
    code = _convert_name_or_literal(tok)

    for tok in stream:
        if tok.exact_type == ENDMARKER:
            break
        if tok.exact_type == DOT:
            tok = next(stream)
            if not tok.exact_type == NAME:
                raise TemplateSyntaxError(content)
            attr = tok.string
            code = ast.Attribute(value=code, attr=attr, ctx=ast.Load())
        elif tok.exact_type == LSQB:  # [
            tok = next(stream)
            lookup = _convert_name_or_literal(tok)
            code = ast.Subscript(
                value=code,
                slice=ast.Index(value=lookup, ctx=ast.Load()),
                ctx=ast.Load()
            )
            tok = next(stream)
            if not tok.exact_type == RSQB: # ]
                raise TemplateSyntaxError('Expected ], found: %r' % tok)
        else:
            raise TemplateSyntaxError('Found unexpected token %r', tok)
    return code

def parse_block_tag(content, stream):
    parts = token_stream(content)

    tok = next(parts)
    if tok.exact_type != NAME:
        raise TemplateSyntaxError("Expected NAME: found %r" % tok)

class Template(object):
    def __init__(self, source):
        self.source = source

        code = self.parse()
        ast.fix_missing_locations(code)
        self.func = compile(code, filename="<template>", mode="exec")

    def _token_to_code(self, token):
        '''Given a Token instance, convert it to AST'''
        code = None
        if token.mode == TOKEN_TEXT:
            code = ast.Str(s=token.content)
        elif token.mode == TOKEN_VAR:
            # parse
            code = parse_expr(token.content)
        elif token.mode == TOKEN_BLOCK:
            # Parse args/kwargs
            parse_block_tag(token.content, self.stream)
            # create tag class instance
        else:
            # Must be a comment
            pass

        if code is not None:
            return token._position(code)

    def parse(self):
        '''Convert the parsed tokens into a list of expressions
        Then join them'''
        steps = []
        self.stream = tokenise(self.source)
        for token in self.stream:
            code = self._token_to_code(token)
            if code is not None:
                steps.append(code)

        # result = [str(x) for x in steps]
        return ast.Module(
            body=[ast.Assign(
                targets=[ast.Name(id='result', ctx=ast.Store())],
                value=ast.ListComp(
                    elt=ast.Call(
                        func=ast.Name(id='str', ctx=ast.Load()),
                        args=[
                            ast.Name(id='x', ctx=ast.Load()),
                        ],
                        keywords=[],
                    ),
                    generators=[
                        ast.comprehension(
                            target=ast.Name(id='x', ctx=ast.Store()),
                            iter=ast.List(elts=steps, ctx=ast.Load()),
                            ifs=[]
                        )
                    ]
                )
            )
        ])

    def render(self, context):
        ctx = {'context': context}
        exec(self.func, ctx)
        return u''.join(ctx['result'])

<commit_msg>Continue with kwargs processing<commit_after>
import ast
from tokenize import *

from collections import namedtuple
from StringIO import StringIO

from .tokenise import tokenise, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK

class TemplateSyntaxError(Exception):
    pass

### Back-ported from Py3
EXACT_TOKEN_TYPES = {
    '(':   LPAR,
    ')':   RPAR,
    '[':   LSQB,
    ']':   RSQB,
    ':':   COLON,
    ',':   COMMA,
    ';':   SEMI,
    '+':   PLUS,
    '-':   MINUS,
    '*':   STAR,
    '/':   SLASH,
    '|':   VBAR,
    '&':   AMPER,
    '<':   LESS,
    '>':   GREATER,
    '=':   EQUAL,
    '.':   DOT,
    '%':   PERCENT,
    '{':   LBRACE,
    '}':   RBRACE,
    '==':  EQEQUAL,
    '!=':  NOTEQUAL,
    '<=':  LESSEQUAL,
    '>=':  GREATEREQUAL,
    '~':   TILDE,
    '^':   CIRCUMFLEX,
    '<<':  LEFTSHIFT,
    '>>':  RIGHTSHIFT,
    '**':  DOUBLESTAR,
    '+=':  PLUSEQUAL,
    '-=':  MINEQUAL,
    '*=':  STAREQUAL,
    '/=':  SLASHEQUAL,
    '%=':  PERCENTEQUAL,
    '&=':  AMPEREQUAL,
    '|=':  VBAREQUAL,
    '^=': CIRCUMFLEXEQUAL,
    '<<=': LEFTSHIFTEQUAL,
    '>>=': RIGHTSHIFTEQUAL,
    '**=': DOUBLESTAREQUAL,
    '//':  DOUBLESLASH,
    '//=': DOUBLESLASHEQUAL,
    '@':   AT
}

class TokenInfo(namedtuple('TokenInfo', 'type string start end line')):
    def __repr__(self):
        annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
        return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
                self._replace(type=annotated_type))

    @property
    def exact_type(self):
        if self.type == OP and self.string in EXACT_TOKEN_TYPES:
            return EXACT_TOKEN_TYPES[self.string]
        else:
            return self.type
###

def token_stream(content):
    '''Fix the hole in Py2, change token types to their real types'''
    for token in generate_tokens(StringIO(content).readline):
        yield TokenInfo(*token)

def _context_lookup(x):
    '''Return AST for looking up x in the Context'''
    return ast.Subscript(
        value=ast.Name(id='context', ctx=ast.Load()),
        slice=ast.Index(value=ast.Str(s=x), ctx=ast.Load()),
        ctx=ast.Load(),
    )

def _make_number(tok):
    '''Turn a token of type NUMBER into an ast.Num'''
    if '.' in tok.string:
        val = float(tok.string)
    else:
        val = int(tok.string)
    return ast.Num(n=val)

def _convert_name_or_literal(tok):
    '''Turn STRING, NUMBER or NAME tokens into equivalent AST node.'''
    if tok.exact_type == STRING:
        code = ast.Str(s=tok.string[1:-1])
    elif tok.exact_type == NUMBER:
        code = _make_number(tok)
    elif tok.exact_type == NAME:
        code = _context_lookup(tok.string)
    else:
        raise TemplateSyntaxError(content)
    return code

def parse_expr(content):
    '''Turn a content string into AST'''
    stream = token_stream(content)

    # First token MUST be either a literal, or name
    tok = next(stream)
    code = _convert_name_or_literal(tok)

    for tok in stream:
        if tok.exact_type == ENDMARKER:
            break
        if tok.exact_type == DOT:
            tok = next(stream)
            if not tok.exact_type == NAME:
                raise TemplateSyntaxError(content)
            attr = tok.string
            code = ast.Attribute(value=code, attr=attr, ctx=ast.Load())
        elif tok.exact_type == LSQB:  # [
            tok = next(stream)
            lookup = _convert_name_or_literal(tok)
            code = ast.Subscript(
                value=code,
                slice=ast.Index(value=lookup, ctx=ast.Load()),
                ctx=ast.Load()
            )
            tok = next(stream)
            if not tok.exact_type == RSQB: # ]
                raise TemplateSyntaxError('Expected ], found: %r' % tok)
        else:
            raise TemplateSyntaxError('Found unexpected token %r', tok)
    return code

def parse_block_tag(content, stream):
    parts = token_stream(content)

    tok = next(parts)
    if tok.exact_type != NAME:
        raise TemplateSyntaxError("Expected NAME: found %r" % tok)

    # Parse args
    # Parse kwargs
    for tok in parts:
        if tok.exact_type != NAME:
            break
        key = tok.string
        tok = next(parts)
        if tok.exact_type != EQUAL:
            pass

class Template(object):
    def __init__(self, source):
        self.source = source

        code = self.parse()
        ast.fix_missing_locations(code)
        self.func = compile(code, filename="<template>", mode="exec")

    def _token_to_code(self, token):
        '''Given a Token instance, convert it to AST'''
        code = None
        if token.mode == TOKEN_TEXT:
            code = ast.Str(s=token.content)
        elif token.mode == TOKEN_VAR:
            # parse
            code = parse_expr(token.content)
        elif token.mode == TOKEN_BLOCK:
            # Parse args/kwargs
            parse_block_tag(token.content, self.stream)
            # create tag class instance
        else:
            # Must be a comment
            pass

        if code is not None:
            return token._position(code)

    def parse(self):
        '''Convert the parsed tokens into a list of expressions
        Then join them'''
        steps = []
        self.stream = tokenise(self.source)
        for token in self.stream:
            code = self._token_to_code(token)
            if code is not None:
                steps.append(code)

        # result = [str(x) for x in steps]
        return ast.Module(
            body=[ast.Assign(
                targets=[ast.Name(id='result', ctx=ast.Store())],
                value=ast.ListComp(
                    elt=ast.Call(
                        func=ast.Name(id='str', ctx=ast.Load()),
                        args=[
                            ast.Name(id='x', ctx=ast.Load()),
                        ],
                        keywords=[],
                    ),
                    generators=[
                        ast.comprehension(
                            target=ast.Name(id='x', ctx=ast.Store()),
                            iter=ast.List(elts=steps, ctx=ast.Load()),
                            ifs=[]
                        )
                    ]
                )
            )
        ])

    def render(self, context):
        ctx = {'context': context}
        exec(self.func, ctx)
        return u''.join(ctx['result'])

<|endoftext|>


<commit_before>from rdflib.Identifier import Identifier
from rdflib.URIRef import URIRef
from rdflib.Namespace import Namespace
from rdflib.exceptions import Error
from datetime import date,time,datetime
from time import strptime
import base64

try:
    from hashlib import md5
except ImportError:
    from md5 import md5

import logging

_logger = logging.getLogger(__name__)

class Literal(Identifier):
    """
    RDF Literal: http://www.w3.org/TR/rdf-concepts/#section-Graph-Literal

    >>> Literal(1).toPython()
    1L
    >>> cmp(Literal("adsf"), 1)
    1
    >>> lit2006 = Literal('2006-01-01',datatype=_XSD_NS.date)
    >>> lit2006.toPython()
    datetime.date(2006, 1, 1)
    >>> lit2006 < Literal('2007-01-01',datatype=_XSD_NS.date)
    True
    >>> Literal(datetime.utcnow()).datatype
    rdflib.URIRef('http://www.w3.org/2001/XMLSchema#dateTime')
    >>> oneInt     = Literal(1)
    >>> twoInt     = Literal(2)
    >>> twoInt < oneInt
    False
    >>> Literal('1') < Literal(1)
    False
    >>> Literal('1') < Literal('1')
    False
    >>> Literal(1) < Literal('1')
    True
    >>> Literal(1) < Literal(2.0)
    True
    >>> Literal(1) < URIRef('foo')
    True
    >>> Literal(1) < 2.0
    True
    >>> Literal(1) < object
    True
    >>> lit2006 < "2007"
    True
    >>> "2005" < lit2006
    True
    """

    __slots__ = ("language", "datatype", "_cmp_value")

    def __new__(cls, value, lang=None, datatype=None):
        if datatype:
            lang = None
        else:
            value,datatype = _castPythonToLiteral(value)
            if datatype:
                lang = None
        if datatype:
            datatype = URIRef(datatype)
        try:
            inst = unicode.__new__(cls,value)
        except UnicodeDecodeError:
            inst = unicode.__new__(cls,value,'utf-8')
        inst.language = lang
        inst.datatype = datatype
        inst._cmp_value = inst._toCompareValue()
        return inst

    def __reduce__(self):
        return (Literal, (unicode(self), self.language, self.datatype),)

    def __getstate__(self):
        return (None, dict(language=self.language, datatype=self.datatype))

    def __setstate__(self, arg):
        _, d = arg
        self.language = d["language"]
        self.datatype = d["datatype"]

    def __add__(self, val):
        """
        >>> Literal(1) + 1
        2L
        >>> Literal("1") + "1"
        rdflib.Literal('11', lang=None, datatype=None)
        """

        py = self.toPython()
        if isinstance(py, Literal):
            s = super(Literal, self).__add__(val)
            return Literal(s, self.language, self.datatype)
        else:
            return py + val

    def __lt__(self, other):
        """
        >>> Literal("YXNkZg==", datatype=_XSD_NS[u'base64Binary']) < "foo"
        True
        >>> u"\xfe" < Literal(u"foo")
        False
        >>> Literal(base64.encodestring(u"\xfe".encode("utf-8")), datatype=URIRef("http://www.w3.org/2001/XMLSchema#base64Binary")) < u"foo"
        False
        """

        if other is None:
            return False # Nothing is less than None
        try:
            return self._cmp_value < other
        except TypeError, te:
            return unicode(self._cmp_value) < other
        except UnicodeDecodeError, ue:
            if isinstance(self._cmp_value, str):
                return self._cmp_value < other.encode("utf-8")
            else:
                raise ue

    def __le__(self, other):
        if other is None:
            return False
        if self==other:
            return True
        else:
            return self < other

    def __gt__(self, other):
        if other is None:
            return True # Everything is greater than None
        try:
            return self._cmp_value > other
        except TypeError, te:
            return unicode(self._cmp_value) > other
        except UnicodeDecodeError, ue:
            if isinstance(self._cmp_value, str):
                return self._cmp_value > other.encode("utf-8")
            else:
                raise ue

    def __ge__(self, other):
        if other is None:
            return False
        if self==other:
            return True
        else:
            return self > other

    def __ne__(self, other):
        """
        Overriden to ensure property result for comparisons with None via !=.
        Routes all other such != and <> comparisons to __eq__

        >>> Literal('') != None
        True
        >>> Literal('2') <> Literal('2')
        False

        """
        return not self.__eq__(other)

    def __eq__(self, other):
        """
        >>> f = URIRef("foo")
        >>> f is None or f == ''
        False
        >>> Literal("1", datatype=URIRef("foo")) == Literal("1", datatype=URIRef("foo"))
        True
        >>> Literal("1", datatype=URIRef("foo")) == Literal("2", datatype=URIRef("foo"))
        False
        >>> Literal("1", datatype=URIRef("foo")) == "asdf"
        False
        >>> Literal('2007-01-01', datatype=_XSD_NS.date) == Literal('2007-01-01', datatype=_XSD_NS.date)
        True
        >>> Literal('2007-01-01', datatype=_XSD_NS.date) == date(2007, 1, 1)
        True
        >>> oneInt     = Literal(1)
        >>> oneNoDtype = Literal('1')
        >>> oneInt == oneNoDtype
        False
        >>> Literal("1",_XSD_NS[u'string']) == Literal("1",_XSD_NS[u'string'])
        True
        >>> Literal("one",lang="en") == Literal("one",lang="en")
        True
        >>> Literal("hast",lang='en') == Literal("hast",lang='de')
        False
        >>> oneInt == Literal(1)
        True
        >>> oneFloat   = Literal(1.0)
        >>> oneInt == oneFloat
        True
        >>> oneInt == 1
        True
        """
        if other is None:
            return False
        if isinstance(other, Literal):
            return self._cmp_value == other._cmp_value
        else:
            return self._cmp_value == other

    def n3(self):
        language = self.language
        datatype = self.datatype
        # unfortunately this doesn't work: a newline gets encoded as \\n, which is ok in sourcecode, but we want \n
        #encoded = self.encode('unicode-escape').replace('\\', '\\\\').replace('"','\\"')
        #encoded = self.replace.replace('\\', '\\\\').replace('"','\\"')

        # TODO: We could also chose quotes based on the quotes appearing in the string, i.e. '"' and "'" ...

        # which is nicer?
        #if self.find("\"")!=-1 or self.find("'")!=-1 or self.find("\n")!=-1:
        if self.find("\n")!=-1:
            # Triple quote this string.
            encoded=self.replace('\\', '\\\\')
            if self.find('"""')!=-1:
                # is this ok?
                encoded=encoded.replace('"""','\\"""')
            if encoded.endswith('"'): encoded=encoded[:-1]+"\\\""
            encoded='"""%s"""'%encoded
        else:
            encoded='"%s"'%self.replace('\n','\\n').replace('\\', '\\\\').replace('"','\\"')
        if language:
            if datatype:
                return '%s@%s^^<%s>' % (encoded, language, datatype)
            else:
                return '%s@%s' % (encoded, language)
        else:
            if datatype:
                return '%s^^<%s>' % (encoded, datatype)
            else:
                return '%s' % encoded

    def __str__(self):
        return self.encode("unicode-escape")

    def __repr__(self):
        return """rdflib.Literal('%s', lang=%s, datatype=%s)""" % (str(self), repr(self.language), repr(self.datatype))

    def toPython(self):
        """
        Returns an appropriate python datatype derived from this RDF Literal
        """
        convFunc = _toPythonMapping.get(self.datatype, None)

        if convFunc:
            rt = convFunc(self)
        else:
            rt = self
        return rt

    def _toCompareValue(self):
        try:
            rt = self.toPython()
        except Exception, e:
            _logger.warning("could not convert %s to a Python datatype" % repr(self))
            rt = self

        if rt is self:
            if self.language is None and self.datatype is None:
                return unicode(rt)
            else:
                return (unicode(rt), rt.datatype, rt.language)
        return rt

    def md5_term_hash(self):
        d = md5(str(self))
        d.update("L")
        return d.hexdigest()

_XSD_NS = Namespace(u'http://www.w3.org/2001/XMLSchema#')

#Casts a python datatype to a tuple of the lexical value and a datatype URI (or None)
def _castPythonToLiteral(obj):
    for pType,(castFunc,dType) in _PythonToXSD:
        if isinstance(obj,pType):
            if castFunc:
                return castFunc(obj),dType
            elif dType:
                return obj,dType
            else:
                return obj,None
    return obj, None # TODO: is this right for the fall through case?

# Mappings from Python types to XSD datatypes and back (burrowed from sparta)
# datetime instances are also instances of date... so we need to order these.
_PythonToXSD = [
    (basestring, (None,None)),
    (float     , (None,_XSD_NS[u'float'])),
    (int       , (None,_XSD_NS[u'int'])),
    (long      , (None,_XSD_NS[u'long'])),
    (bool      , (None,_XSD_NS[u'boolean'])),
    (datetime  , (lambda i:i.isoformat(),_XSD_NS[u'dateTime'])),
    (date      , (lambda i:i.isoformat(),_XSD_NS[u'date'])),
    (time      , (lambda i:i.isoformat(),_XSD_NS[u'time'])),
]

def _strToTime(v) :
    return strptime(v,"%H:%M:%S")

def _strToDate(v) :
    tstr = strptime(v,"%Y-%m-%d")
    return date(tstr.tm_year,tstr.tm_mon,tstr.tm_mday)

def _strToDateTime(v) :
    """
    Attempt to cast to datetime, or just return the string (otherwise)
    """
    try:
        tstr = strptime(v,"%Y-%m-%dT%H:%M:%S")
    except:
        try:
            tstr = strptime(v,"%Y-%m-%dT%H:%M:%SZ")
        except:
            try:
                tstr = strptime(v,"%Y-%m-%dT%H:%M:%S%Z")
            except:
                return v

    return datetime(tstr.tm_year,tstr.tm_mon,tstr.tm_mday,tstr.tm_hour,tstr.tm_min,tstr.tm_sec)

XSDToPython = {
    _XSD_NS[u'time']               : _strToTime,
    _XSD_NS[u'date']               : _strToDate,
    _XSD_NS[u'dateTime']           : _strToDateTime,
    _XSD_NS[u'string']             : None,
    _XSD_NS[u'normalizedString']   : None,
    _XSD_NS[u'token']              : None,
    _XSD_NS[u'language']           : None,
    _XSD_NS[u'boolean']            : lambda i:i.lower() in ['1','true'],
    _XSD_NS[u'decimal']            : float,
    _XSD_NS[u'integer']            : long,
    _XSD_NS[u'nonPositiveInteger'] : int,
    _XSD_NS[u'long']               : long,
    _XSD_NS[u'nonNegativeInteger'] : int,
    _XSD_NS[u'negativeInteger']    : int,
    _XSD_NS[u'int']                : long,
    _XSD_NS[u'unsignedLong']       : long,
    _XSD_NS[u'positiveInteger']    : int,
    _XSD_NS[u'short']              : int,
    _XSD_NS[u'unsignedInt']        : long,
    _XSD_NS[u'byte']               : int,
    _XSD_NS[u'unsignedShort']      : int,
    _XSD_NS[u'unsignedByte']       : int,
    _XSD_NS[u'float']              : float,
    _XSD_NS[u'double']             : float,
    _XSD_NS[u'base64Binary']       : base64.decodestring,
    _XSD_NS[u'anyURI']             : None,
}

_toPythonMapping = {}
_toPythonMapping.update(XSDToPython)

def bind(datatype, conversion_function):
    """bind a datatype to a function for converting it into a Python instance."""
    if datatype in _toPythonMapping:
        _logger.warning("datatype '%s' was already bound. Rebinding." % datatype)
    _toPythonMapping[datatype] = conversion_function

def test():
    import doctest
    doctest.testmod()

if __name__ == '__main__':
    test()
<commit_msg>fix for test_datatype_parsing.py.  Added __hash__ method which is used (instead of __eq__) internally to manage dictionary key comparison.<commit_after>from rdflib.Identifier import Identifier
from rdflib.URIRef import URIRef
from rdflib.Namespace import Namespace
from rdflib.exceptions import Error
from datetime import date,time,datetime
from time import strptime
import base64

try:
    from hashlib import md5
except ImportError:
    from md5 import md5

import logging

_logger = logging.getLogger(__name__)

class Literal(Identifier):
    """
    RDF Literal: http://www.w3.org/TR/rdf-concepts/#section-Graph-Literal

    >>> Literal(1).toPython()
    1L
    >>> cmp(Literal("adsf"), 1)
    1
    >>> lit2006 = Literal('2006-01-01',datatype=_XSD_NS.date)
    >>> lit2006.toPython()
    datetime.date(2006, 1, 1)
    >>> lit2006 < Literal('2007-01-01',datatype=_XSD_NS.date)
    True
    >>> Literal(datetime.utcnow()).datatype
    rdflib.URIRef('http://www.w3.org/2001/XMLSchema#dateTime')
    >>> oneInt     = Literal(1)
    >>> twoInt     = Literal(2)
    >>> twoInt < oneInt
    False
    >>> Literal('1') < Literal(1)
    False
    >>> Literal('1') < Literal('1')
    False
    >>> Literal(1) < Literal('1')
    True
    >>> Literal(1) < Literal(2.0)
    True
    >>> Literal(1) < URIRef('foo')
    True
    >>> Literal(1) < 2.0
    True
    >>> Literal(1) < object
    True
    >>> lit2006 < "2007"
    True
    >>> "2005" < lit2006
    True
    """

    __slots__ = ("language", "datatype", "_cmp_value")

    def __new__(cls, value, lang=None, datatype=None):
        if datatype:
            lang = None
        else:
            value,datatype = _castPythonToLiteral(value)
            if datatype:
                lang = None
        if datatype:
            datatype = URIRef(datatype)
        try:
            inst = unicode.__new__(cls,value)
        except UnicodeDecodeError:
            inst = unicode.__new__(cls,value,'utf-8')
        inst.language = lang
        inst.datatype = datatype
        inst._cmp_value = inst._toCompareValue()
        return inst

    def __reduce__(self):
        return (Literal, (unicode(self), self.language, self.datatype),)

    def __getstate__(self):
        return (None, dict(language=self.language, datatype=self.datatype))

    def __setstate__(self, arg):
        _, d = arg
        self.language = d["language"]
        self.datatype = d["datatype"]

    def __add__(self, val):
        """
        >>> Literal(1) + 1
        2L
        >>> Literal("1") + "1"
        rdflib.Literal('11', lang=None, datatype=None)
        """

        py = self.toPython()
        if isinstance(py, Literal):
            s = super(Literal, self).__add__(val)
            return Literal(s, self.language, self.datatype)
        else:
            return py + val

    def __lt__(self, other):
        """
        >>> Literal("YXNkZg==", datatype=_XSD_NS[u'base64Binary']) < "foo"
        True
        >>> u"\xfe" < Literal(u"foo")
        False
        >>> Literal(base64.encodestring(u"\xfe".encode("utf-8")), datatype=URIRef("http://www.w3.org/2001/XMLSchema#base64Binary")) < u"foo"
        False
        """

        if other is None:
            return False # Nothing is less than None
        try:
            return self._cmp_value < other
        except TypeError, te:
            return unicode(self._cmp_value) < other
        except UnicodeDecodeError, ue:
            if isinstance(self._cmp_value, str):
                return self._cmp_value < other.encode("utf-8")
            else:
                raise ue

    def __le__(self, other):
        if other is None:
            return False
        if self==other:
            return True
        else:
            return self < other

    def __gt__(self, other):
        if other is None:
            return True # Everything is greater than None
        try:
            return self._cmp_value > other
        except TypeError, te:
            return unicode(self._cmp_value) > other
        except UnicodeDecodeError, ue:
            if isinstance(self._cmp_value, str):
                return self._cmp_value > other.encode("utf-8")
            else:
                raise ue

    def __ge__(self, other):
        if other is None:
            return False
        if self==other:
            return True
        else:
            return self > other

    def __ne__(self, other):
        """
        Overriden to ensure property result for comparisons with None via !=.
        Routes all other such != and <> comparisons to __eq__

        >>> Literal('') != None
        True
        >>> Literal('2') <> Literal('2')
        False

        """
        return not self.__eq__(other)

    def __hash__(self):
        """
        >>> a = {Literal('1',datatype=_XSD_NS.integer):'one'}
        >>> Literal('1',datatype=_XSD_NS.double) in a
        False

        [[
        Called for the key object for dictionary operations,
        and by the built-in function hash(). Should return
        a 32-bit integer usable as a hash value for
        dictionary operations. The only required property
        is that objects which compare equal have the same
        hash value; it is advised to somehow mix together
        (e.g., using exclusive or) the hash values for the
        components of the object that also play a part in
        comparison of objects.
        ]] -- 3.4.1 Basic customization (Python)

        [[
        Two literals are equal if and only if all of the following hold:
        * The strings of the two lexical forms compare equal, character by character.
        * Either both or neither have language tags.
        * The language tags, if any, compare equal.
        * Either both or neither have datatype URIs.
        * The two datatype URIs, if any, compare equal, character by character.
        ]] -- 6.5.1 Literal Equality (RDF: Concepts and Abstract Syntax)

        """
        return hash(str(self)) ^ hash(self.language) ^ hash(self.datatype)

    def __eq__(self, other):
        """
        >>> f = URIRef("foo")
        >>> f is None or f == ''
        False
        >>> Literal("1", datatype=URIRef("foo")) == Literal("1", datatype=URIRef("foo"))
        True
        >>> Literal("1", datatype=URIRef("foo")) == Literal("2", datatype=URIRef("foo"))
        False
        >>> Literal("1", datatype=URIRef("foo")) == "asdf"
        False
        >>> Literal('2007-01-01', datatype=_XSD_NS.date) == Literal('2007-01-01', datatype=_XSD_NS.date)
        True
        >>> Literal('2007-01-01', datatype=_XSD_NS.date) == date(2007, 1, 1)
        True
        >>> oneInt     = Literal(1)
        >>> oneNoDtype = Literal('1')
        >>> oneInt == oneNoDtype
        False
        >>> Literal("1",_XSD_NS[u'string']) == Literal("1",_XSD_NS[u'string'])
        True
        >>> Literal("one",lang="en") == Literal("one",lang="en")
        True
        >>> Literal("hast",lang='en') == Literal("hast",lang='de')
        False
        >>> oneInt == Literal(1)
        True
        >>> oneFloat   = Literal(1.0)
        >>> oneInt == oneFloat
        True
        >>> oneInt == 1
        True
        """
        if other is None:
            return False
        if isinstance(other, Literal):
            return self._cmp_value == other._cmp_value
        else:
            return self._cmp_value == other

    def n3(self):
        language = self.language
        datatype = self.datatype
        # unfortunately this doesn't work: a newline gets encoded as \\n, which is ok in sourcecode, but we want \n
        #encoded = self.encode('unicode-escape').replace('\\', '\\\\').replace('"','\\"')
        #encoded = self.replace.replace('\\', '\\\\').replace('"','\\"')

        # TODO: We could also chose quotes based on the quotes appearing in the string, i.e. '"' and "'" ...

        # which is nicer?
        #if self.find("\"")!=-1 or self.find("'")!=-1 or self.find("\n")!=-1:
        if self.find("\n")!=-1:
            # Triple quote this string.
            encoded=self.replace('\\', '\\\\')
            if self.find('"""')!=-1:
                # is this ok?
                encoded=encoded.replace('"""','\\"""')
            if encoded.endswith('"'): encoded=encoded[:-1]+"\\\""
            encoded='"""%s"""'%encoded
        else:
            encoded='"%s"'%self.replace('\n','\\n').replace('\\', '\\\\').replace('"','\\"')
        if language:
            if datatype:
                return '%s@%s^^<%s>' % (encoded, language, datatype)
            else:
                return '%s@%s' % (encoded, language)
        else:
            if datatype:
                return '%s^^<%s>' % (encoded, datatype)
            else:
                return '%s' % encoded

    def __str__(self):
        return self.encode("unicode-escape")

    def __repr__(self):
        return """rdflib.Literal('%s', lang=%s, datatype=%s)""" % (str(self), repr(self.language), repr(self.datatype))

    def toPython(self):
        """
        Returns an appropriate python datatype derived from this RDF Literal
        """
        convFunc = _toPythonMapping.get(self.datatype, None)

        if convFunc:
            rt = convFunc(self)
        else:
            rt = self
        return rt

    def _toCompareValue(self):
        try:
            rt = self.toPython()
        except Exception, e:
            _logger.warning("could not convert %s to a Python datatype" % repr(self))
            rt = self

        if rt is self:
            if self.language is None and self.datatype is None:
                return unicode(rt)
            else:
                return (unicode(rt), rt.datatype, rt.language)
        return rt

    def md5_term_hash(self):
        d = md5(str(self))
        d.update("L")
        return d.hexdigest()

_XSD_NS = Namespace(u'http://www.w3.org/2001/XMLSchema#')

#Casts a python datatype to a tuple of the lexical value and a datatype URI (or None)
def _castPythonToLiteral(obj):
    for pType,(castFunc,dType) in _PythonToXSD:
        if isinstance(obj,pType):
            if castFunc:
                return castFunc(obj),dType
            elif dType:
                return obj,dType
            else:
                return obj,None
    return obj, None # TODO: is this right for the fall through case?

# Mappings from Python types to XSD datatypes and back (burrowed from sparta)
# datetime instances are also instances of date... so we need to order these.
_PythonToXSD = [
    (basestring, (None,None)),
    (float     , (None,_XSD_NS[u'float'])),
    (int       , (None,_XSD_NS[u'int'])),
    (long      , (None,_XSD_NS[u'long'])),
    (bool      , (None,_XSD_NS[u'boolean'])),
    (datetime  , (lambda i:i.isoformat(),_XSD_NS[u'dateTime'])),
    (date      , (lambda i:i.isoformat(),_XSD_NS[u'date'])),
    (time      , (lambda i:i.isoformat(),_XSD_NS[u'time'])),
]

def _strToTime(v) :
    return strptime(v,"%H:%M:%S")

def _strToDate(v) :
    tstr = strptime(v,"%Y-%m-%d")
    return date(tstr.tm_year,tstr.tm_mon,tstr.tm_mday)

def _strToDateTime(v) :
    """
    Attempt to cast to datetime, or just return the string (otherwise)
    """
    try:
        tstr = strptime(v,"%Y-%m-%dT%H:%M:%S")
    except:
        try:
            tstr = strptime(v,"%Y-%m-%dT%H:%M:%SZ")
        except:
            try:
                tstr = strptime(v,"%Y-%m-%dT%H:%M:%S%Z")
            except:
                return v

    return datetime(tstr.tm_year,tstr.tm_mon,tstr.tm_mday,tstr.tm_hour,tstr.tm_min,tstr.tm_sec)

XSDToPython = {
    _XSD_NS[u'time']               : _strToTime,
    _XSD_NS[u'date']               : _strToDate,
    _XSD_NS[u'dateTime']           : _strToDateTime,
    _XSD_NS[u'string']             : None,
    _XSD_NS[u'normalizedString']   : None,
    _XSD_NS[u'token']              : None,
    _XSD_NS[u'language']           : None,
    _XSD_NS[u'boolean']            : lambda i:i.lower() in ['1','true'],
    _XSD_NS[u'decimal']            : float,
    _XSD_NS[u'integer']            : long,
    _XSD_NS[u'nonPositiveInteger'] : int,
    _XSD_NS[u'long']               : long,
    _XSD_NS[u'nonNegativeInteger'] : int,
    _XSD_NS[u'negativeInteger']    : int,
    _XSD_NS[u'int']                : long,
    _XSD_NS[u'unsignedLong']       : long,
    _XSD_NS[u'positiveInteger']    : int,
    _XSD_NS[u'short']              : int,
    _XSD_NS[u'unsignedInt']        : long,
    _XSD_NS[u'byte']               : int,
    _XSD_NS[u'unsignedShort']      : int,
    _XSD_NS[u'unsignedByte']       : int,
    _XSD_NS[u'float']              : float,
    _XSD_NS[u'double']             : float,
    _XSD_NS[u'base64Binary']       : base64.decodestring,
    _XSD_NS[u'anyURI']             : None,
}

_toPythonMapping = {}
_toPythonMapping.update(XSDToPython)

def bind(datatype, conversion_function):
    """bind a datatype to a function for converting it into a Python instance."""
    if datatype in _toPythonMapping:
        _logger.warning("datatype '%s' was already bound. Rebinding." % datatype)
    _toPythonMapping[datatype] = conversion_function

def test():
    import doctest
    doctest.testmod()

if __name__ == '__main__':
    test()
<|endoftext|>


<commit_before>import time
import logging
import threading
from hashlib import sha1
from normality import stringify
from PIL import Image
from io import BytesIO
from languagecodes import list_to_alpha3 as alpha3

from ingestors import settings
from ingestors.support.cache import CacheSupport
from ingestors.util import temp_locale

log = logging.getLogger(__name__)
TESSERACT_LOCALE = 'C'

class OCRSupport(CacheSupport):
    MIN_SIZE = 1024 * 2
    MAX_SIZE = (1024 * 1024 * 30) - 1024

    def extract_ocr_text(self, data, languages=None):
        if not self.MIN_SIZE < len(data) < self.MAX_SIZE:
            log.info('OCR: file size out of range (%d)', len(data))
            return None

        languages = sorted(set(languages or []))
        data_key = sha1(data).hexdigest()
        key = self.cache_key('ocr', data_key, *languages)
        text = self.get_cache_value(key)
        if text is not None:
            log.info('OCR: %s chars cached', len(text))
            return stringify(text)

        if not hasattr(settings, '_ocr_service'):
            if settings.OCR_VISION_API:
                settings._ocr_service = GoogleOCRService()
            else:
                settings._ocr_service = LocalOCRService()

        text = settings._ocr_service.extract_text(data, languages=languages)
        self.set_cache_value(key, text)
        if text is not None:
            log.info('OCR: %s chars (from %s bytes)',
                     len(text), len(data))
        return stringify(text)

class LocalOCRService(object):
    """Perform OCR using an RPC-based service."""
    MAX_MODELS = 4

    def __init__(self):
        self.tl = threading.local()

    def language_list(self, languages):
        if not hasattr(settings, 'ocr_supported'):
            with temp_locale(TESSERACT_LOCALE):
                # Tesseract language types:
                from tesserocr import get_languages
                _, settings.ocr_supported = get_languages()
                # log.info("OCR languages: %r", settings.ocr_supported)
        models = [c for c in alpha3(languages) if c in settings.ocr_supported]
        if len(models) > self.MAX_MODELS:
            log.warning("Too many models, limit: %s", self.MAX_MODELS)
            models = models[:self.MAX_MODELS]
        models.append('eng')
        return '+'.join(sorted(set(models)))

    def configure_engine(self, languages):
        if not hasattr(self.tl, 'api') or self.tl.api is None:
            from tesserocr import PyTessBaseAPI, PSM, OEM
            log.info("Configuring OCR engine (%s)", languages)
            self.tl.api = PyTessBaseAPI(lang=languages,
                                        oem=OEM.LSTM_ONLY,
                                        psm=PSM.AUTO_OSD)
        if languages != self.tl.api.GetInitLanguagesAsString():
            log.info("Re-initialising OCR engine (%s)", languages)
            self.tl.api.Init(lang=languages, oem=OEM.LSTM_ONLY)
        return self.tl.api

    def extract_text(self, data, languages=None):
        """Extract text from a binary string of data."""
        try:
            image = Image.open(BytesIO(data))
            image.load()
        except Exception:
            log.exception("Cannot open image data using Pillow")
            return ''

        with temp_locale(TESSERACT_LOCALE):
            api = self.configure_engine(languages)
            try:
                languages = self.language_list(languages)
                # TODO: play with contrast and sharpening the images.
                start_time = time.time()
                api.SetImage(image)
                text = api.GetUTF8Text()
                confidence = api.MeanTextConf()
                end_time = time.time()
                duration = end_time - start_time
                log.info("w: %s, h: %s, l: %s, c: %s, took: %.5f",
                         image.width, image.height, languages,
                         confidence, duration)
                return text
            except Exception as exc:
                log.warning("OCR error: %s", exc)
                return ''
            finally:
                api.Clear()

class GoogleOCRService(object):
    """Use Google's Vision API to perform OCR. This has very good quality
    but is quite expensive. For this reason, its use is controlled via a
    separate configuration variable, OCR_VISION_API, which must be set to
    'true'. To use the API, you must also have a service account JSON file
    under GOOGLE_APPLICATION_CREDENTIALS."""

    def __init__(self):
        import google.auth
        from google.cloud.vision import ImageAnnotatorClient
        credentials, project_id = google.auth.default()
        self.client = ImageAnnotatorClient(credentials=credentials)
        log.info("Using Google Vision API. Charges apply.")

    def extract_text(self, data, languages=None):
        from google.cloud.vision import types
        image = types.Image(content=data)
        res = self.client.document_text_detection(image)
        return res.full_text_annotation.text or ''
<commit_msg>re-init<commit_after>import time
import logging
import threading
from hashlib import sha1
from normality import stringify
from PIL import Image
from io import BytesIO
from languagecodes import list_to_alpha3 as alpha3

from ingestors import settings
from ingestors.support.cache import CacheSupport
from ingestors.util import temp_locale

log = logging.getLogger(__name__)
TESSERACT_LOCALE = 'C'

class OCRSupport(CacheSupport):
    MIN_SIZE = 1024 * 2
    MAX_SIZE = (1024 * 1024 * 30) - 1024

    def extract_ocr_text(self, data, languages=None):
        if not self.MIN_SIZE < len(data) < self.MAX_SIZE:
            log.info('OCR: file size out of range (%d)', len(data))
            return None

        languages = sorted(set(languages or []))
        data_key = sha1(data).hexdigest()
        key = self.cache_key('ocr', data_key, *languages)
        text = self.get_cache_value(key)
        if text is not None:
            log.info('OCR: %s chars cached', len(text))
            return stringify(text)

        if not hasattr(settings, '_ocr_service'):
            if settings.OCR_VISION_API:
                settings._ocr_service = GoogleOCRService()
            else:
                settings._ocr_service = LocalOCRService()

        text = settings._ocr_service.extract_text(data, languages=languages)
        self.set_cache_value(key, text)
        if text is not None:
            log.info('OCR: %s chars (from %s bytes)',
                     len(text), len(data))
        return stringify(text)

class LocalOCRService(object):
    """Perform OCR using an RPC-based service."""
    MAX_MODELS = 4

    def __init__(self):
        self.tl = threading.local()

    def language_list(self, languages):
        if not hasattr(settings, 'ocr_supported'):
            with temp_locale(TESSERACT_LOCALE):
                # Tesseract language types:
                from tesserocr import get_languages
                _, settings.ocr_supported = get_languages()
                # log.info("OCR languages: %r", settings.ocr_supported)
        models = [c for c in alpha3(languages) if c in settings.ocr_supported]
        if len(models) > self.MAX_MODELS:
            log.warning("Too many models, limit: %s", self.MAX_MODELS)
            models = models[:self.MAX_MODELS]
        models.append('eng')
        return '+'.join(sorted(set(models)))

    def configure_engine(self, languages):
        from tesserocr import PyTessBaseAPI, PSM, OEM
        if not hasattr(self.tl, 'api') or self.tl.api is None:
            log.info("Configuring OCR engine (%s)", languages)
            self.tl.api = PyTessBaseAPI(lang=languages,
                                        oem=OEM.LSTM_ONLY,
                                        psm=PSM.AUTO_OSD)
        if languages != self.tl.api.GetInitLanguagesAsString():
            log.info("Re-initialising OCR engine (%s)", languages)
            self.tl.api.Init(lang=languages, oem=OEM.LSTM_ONLY)
        return self.tl.api

    def extract_text(self, data, languages=None):
        """Extract text from a binary string of data."""
        try:
            image = Image.open(BytesIO(data))
            image.load()
        except Exception:
            log.exception("Cannot open image data using Pillow")
            return ''

        with temp_locale(TESSERACT_LOCALE):
            api = self.configure_engine(languages)
            try:
                languages = self.language_list(languages)
                # TODO: play with contrast and sharpening the images.
                start_time = time.time()
                api.SetImage(image)
                text = api.GetUTF8Text()
                confidence = api.MeanTextConf()
                end_time = time.time()
                duration = end_time - start_time
                log.info("w: %s, h: %s, l: %s, c: %s, took: %.5f",
                         image.width, image.height, languages,
                         confidence, duration)
                return text
            except Exception as exc:
                log.warning("OCR error: %s", exc)
                return ''
            finally:
                api.Clear()

class GoogleOCRService(object):
    """Use Google's Vision API to perform OCR. This has very good quality
    but is quite expensive. For this reason, its use is controlled via a
    separate configuration variable, OCR_VISION_API, which must be set to
    'true'. To use the API, you must also have a service account JSON file
    under GOOGLE_APPLICATION_CREDENTIALS."""

    def __init__(self):
        import google.auth
        from google.cloud.vision import ImageAnnotatorClient
        credentials, project_id = google.auth.default()
        self.client = ImageAnnotatorClient(credentials=credentials)
        log.info("Using Google Vision API. Charges apply.")

    def extract_text(self, data, languages=None):
        from google.cloud.vision import types
        image = types.Image(content=data)
        res = self.client.document_text_detection(image)
        return res.full_text_annotation.text or ''
<|endoftext|>


<commit_before>from rdflib.Identifier import Identifier
from rdflib.URIRef import URIRef
from rdflib.Namespace import Namespace
from rdflib.exceptions import Error
from datetime import date,time,datetime
from time import strptime
import base64

class Literal(Identifier):
    """

    http://www.w3.org/TR/rdf-concepts/#section-Graph-Literal

    >>> Literal(1)._toPython()
    1L
    >>> cmp(Literal("adsf"), 1)
    1
    >>> lit2006 = Literal('2006-01-01',datatype=_XSD_NS.date)
    >>> lit2006._toPython()
    datetime.date(2006, 1, 1)
    >>> lit2006 < Literal('2007-01-01',datatype=_XSD_NS.date)
    True
    >>> oneInt     = Literal(1)
    >>> twoInt     = Literal(2)
    >>> twoInt < oneInt
    False
    >>> Literal('1') < Literal(1)
    False
    >>> Literal('1') < Literal('1')
    False
    >>> Literal(1) < Literal('1')
    True
    >>> Literal(1) < Literal(2.0)
    True
    >>> Literal(1) < URIRef('foo')
    True
    >>> Literal(1) < 2.0
    True
    >>> Literal(1) < object
    True
    >>> lit2006 < "2007"
    True
    >>> "2005" < lit2006
    True
    """

    __slots__ = ("language", "datatype", "_cmp_value")

    def __new__(cls, value, lang=None, datatype=None):
        if datatype:
            lang = None
        else:
            value,datatype = _castPythonToLiteral(value)
            if datatype:
                lang = None
        if datatype:
            datatype = URIRef(datatype)
        try:
            inst = unicode.__new__(cls,value)
        except UnicodeDecodeError:
            inst = unicode.__new__(cls,value,'utf-8')
        inst.language = lang
        inst.datatype = datatype
        inst._cmp_value = inst._toPython()
        return inst

    def __reduce__(self):
        return (Literal, (unicode(self), self.language, self.datatype),)

    def __getstate__(self):
        return (None, dict(language=self.language, datatype=self.datatype))

    def __setstate__(self, arg):
        _, d = arg
        self.language = d["language"]
        self.datatype = d["datatype"]

    def __add__(self, val):
        s = super(Literal, self).__add__(val)
        return Literal(s, self.language, self.datatype)

    def __lt__(self, other):
        if other is None:
            return False # Nothing is less than None
        try:
            return self._cmp_value < other
        except TypeError, te:
            return unicode(self._cmp_value) < other

    def __le__(self, other):
        if other is None:
            return False
        if self==other:
            return True
        else:
            return self < other

    def __gt__(self, other):
        if other is None:
            return True # Everything is greater than None
        try:
            return self._cmp_value > other
        except TypeError, te:
            return unicode(self._cmp_value) > other

    def __ge__(self, other):
        if other is None:
            return False
        if self==other:
            return True
        else:
            return self > other

    def __ne__(self, other):
        """
        Overriden to ensure property result for comparisons with None via !=.
        Routes all other such != and <> comparisons to __eq__

        >>> Literal('') != None
        True
        >>> Literal('2') <> Literal('2')
        False

        """
        if other is None:
            return True
        else:
            return not self.__eq__(other)

    def __eq__(self, other):
        """
        >>> f = URIRef("foo")
        >>> f is None or f == ''
        False
        >>> Literal("1", datatype=URIRef("foo")) == Literal("1", datatype=URIRef("foo"))
        True
        >>> Literal("1", datatype=URIRef("foo")) == Literal("2", datatype=URIRef("foo"))
        False
        >>> Literal("1", datatype=URIRef("foo")) == "asdf"
        False
        >>> oneInt     = Literal(1)
        >>> oneNoDtype = Literal('1')
        >>> oneInt == oneNoDtype
        False
        >>> Literal("1",_XSD_NS[u'string']) == Literal("1",_XSD_NS[u'string'])
        True
        >>> Literal("one",lang="en") == Literal("one",lang="en")
        True
        >>> Literal("hast",lang='en') == Literal("hast",lang='de')
        False
        >>> oneInt == Literal(1)
        True
        >>> oneFloat   = Literal(1.0)
        >>> oneInt == oneFloat
        True
        >>> oneInt == 1
        True
        """
        if other is None:
            return False
        else:
            return self._cmp_value==other

    def n3(self):
        language = self.language
        datatype = self.datatype
        # unfortunately this doesn't work: a newline gets encoded as \\n, which is ok in sourcecode, but we want \n
        #encoded = self.encode('unicode-escape').replace('\\', '\\\\').replace('"','\\"')
        #encoded = self.replace.replace('\\', '\\\\').replace('"','\\"')

        # TODO: We could also chose quotes based on the quotes appearing in the string, i.e. '"' and "'" ...

        # which is nicer?
        #if self.find("\"")!=-1 or self.find("'")!=-1 or self.find("\n")!=-1:
        if self.find("\n")!=-1:
            # Triple quote this string.
            encoded=self.replace('\\', '\\\\')
            if self.find('"""')!=-1:
                # is this ok?
                encoded=encoded.replace('"""','\\"""')
            if encoded.endswith('"'): encoded=encoded[:-1]+"\\\""
            encoded='"""%s"""'%encoded
        else:
            encoded='"%s"'%self.replace('\n','\\n').replace('\\', '\\\\').replace('"','\\"')
        if language:
            if datatype:
                return '%s@%s^^<%s>' % (encoded, language, datatype)
            else:
                return '%s@%s' % (encoded, language)
        else:
            if datatype:
                return '%s^^<%s>' % (encoded, datatype)
            else:
                return '%s' % encoded

    def __str__(self):
        return self.encode("unicode-escape")

    def __repr__(self):
        klass, convFunc = _XSDToPython.get(self.datatype, (None, None))
        if klass:
            return "%s(%s)"%(klass.__name__, str(self))
        else:
            return """rdflib.Literal('%s', language=%s, datatype=%s)""" % (str(self),repr(self.language), repr(self.datatype))

    def _toPython(self):
        """
        Returns an appropriate python datatype derived from this RDF Literal
        """
        klass,convFunc = _XSDToPython.get(self.datatype,(None,None))
        rt = self
        if convFunc:
            rt = convFunc(rt)
        if klass:
            rt = klass(rt)
        if rt is self:
            if self.language is None and self.datatype is None:
                return unicode(rt)
            else:
                return (unicode(rt), rt.datatype, rt.language)
        return rt

_XSD_NS = Namespace(u'http://www.w3.org/2001/XMLSchema#')

#Casts a python datatype to a tuple of the lexical value and a datatype URI (or None)
def _castPythonToLiteral(obj):
    for pType,(castFunc,dType) in _PythonToXSD.items():
        if isinstance(obj,pType):
            if castFunc:
                return castFunc(obj),dType
            elif dType:
                return obj,dType
            else:
                return obj,None
    return obj, None # TODO: is this right for the fall through case?

#Mappings from Python types to XSD datatypes and back (burrowed from sparta)
_PythonToXSD = {
    basestring : (None,None),
    float      : (None,_XSD_NS[u'float']),
    int        : (None,_XSD_NS[u'int']),
    long       : (None,_XSD_NS[u'long']),
    bool       : (None,_XSD_NS[u'boolean']),
    date       : (lambda i:i.isoformat(),_XSD_NS[u'date']),
    time       : (lambda i:i.isoformat(),_XSD_NS[u'time']),
    datetime   : (lambda i:i.isoformat(),_XSD_NS[u'dateTime']),
}

def _strToTime(v) :
    return strptime(v,"%H:%M:%S")

def _strToDate(v) :
    tstr = strptime(v,"%Y-%m-%d")
    return date(tstr.tm_year,tstr.tm_mon,tstr.tm_mday)

def _strToDateTime(v) :
    """
    Attempt to cast to datetime, or just return the string (otherwise)
    """
    try:
        tstr = strptime(v,"%Y-%m-%dT%H:%M:%S")
    except:
        try:
            tstr = strptime(v,"%Y-%m-%dT%H:%M:%SZ")
        except:
            try:
                tstr = strptime(v,"%Y-%m-%dT%H:%M:%S%Z")
            except:
                return v

    return datetime(tstr.tm_year,tstr.tm_mon,tstr.tm_mday,tstr.tm_hour,tstr.tm_min,tstr.tm_sec)

_XSDToPython = {
    _XSD_NS[u'time']               : (None,_strToTime),
    _XSD_NS[u'date']               : (None,_strToDate),
    _XSD_NS[u'dateTime']           : (None,_strToDateTime),
    _XSD_NS[u'string']             : (None,None),
    _XSD_NS[u'normalizedString']   : (None,None),
    _XSD_NS[u'token']              : (None,None),
    _XSD_NS[u'language']           : (None,None),
    _XSD_NS[u'boolean']            : (None, lambda i:i.lower() in ['1','true']),
    _XSD_NS[u'decimal']            : (float,None),
    _XSD_NS[u'integer']            : (long ,None),
    _XSD_NS[u'nonPositiveInteger'] : (int,None),
    _XSD_NS[u'long']               : (long,None),
    _XSD_NS[u'nonNegativeInteger'] : (int, None),
    _XSD_NS[u'negativeInteger']    : (int, None),
    _XSD_NS[u'int']                : (long, None),
    _XSD_NS[u'unsignedLong']       : (long, None),
    _XSD_NS[u'positiveInteger']    : (int, None),
    _XSD_NS[u'short']              : (int, None),
    _XSD_NS[u'unsignedInt']        : (long, None),
    _XSD_NS[u'byte']               : (int, None),
    _XSD_NS[u'unsignedShort']      : (int, None),
    _XSD_NS[u'unsignedByte']       : (int, None),
    _XSD_NS[u'float']              : (float, None),
    _XSD_NS[u'double']             : (float, None),
    _XSD_NS[u'base64Binary']       : (base64.decodestring, None),
    _XSD_NS[u'anyURI']             : (None,None),
}

def test():
    import doctest
    doctest.testmod()

if __name__ == '__main__':
    test()
<commit_msg>changed __repr__ to always reflect that it's an rdf.Literal -- as this is the case even though we now have it acting like the corresponding type in some casses<commit_after>from rdflib.Identifier import Identifier
from rdflib.URIRef import URIRef
from rdflib.Namespace import Namespace
from rdflib.exceptions import Error
from datetime import date,time,datetime
from time import strptime
import base64

class Literal(Identifier):
    """

    http://www.w3.org/TR/rdf-concepts/#section-Graph-Literal

    >>> Literal(1)._toPython()
    1L
    >>> cmp(Literal("adsf"), 1)
    1
    >>> lit2006 = Literal('2006-01-01',datatype=_XSD_NS.date)
    >>> lit2006._toPython()
    datetime.date(2006, 1, 1)
    >>> lit2006 < Literal('2007-01-01',datatype=_XSD_NS.date)
    True
    >>> oneInt     = Literal(1)
    >>> twoInt     = Literal(2)
    >>> twoInt < oneInt
    False
    >>> Literal('1') < Literal(1)
    False
    >>> Literal('1') < Literal('1')
    False
    >>> Literal(1) < Literal('1')
    True
    >>> Literal(1) < Literal(2.0)
    True
    >>> Literal(1) < URIRef('foo')
    True
    >>> Literal(1) < 2.0
    True
    >>> Literal(1) < object
    True
    >>> lit2006 < "2007"
    True
    >>> "2005" < lit2006
    True
    """

    __slots__ = ("language", "datatype", "_cmp_value")

    def __new__(cls, value, lang=None, datatype=None):
        if datatype:
            lang = None
        else:
            value,datatype = _castPythonToLiteral(value)
            if datatype:
                lang = None
        if datatype:
            datatype = URIRef(datatype)
        try:
            inst = unicode.__new__(cls,value)
        except UnicodeDecodeError:
            inst = unicode.__new__(cls,value,'utf-8')
        inst.language = lang
        inst.datatype = datatype
        inst._cmp_value = inst._toPython()
        return inst

    def __reduce__(self):
        return (Literal, (unicode(self), self.language, self.datatype),)

    def __getstate__(self):
        return (None, dict(language=self.language, datatype=self.datatype))

    def __setstate__(self, arg):
        _, d = arg
        self.language = d["language"]
        self.datatype = d["datatype"]

    def __add__(self, val):
        s = super(Literal, self).__add__(val)
        return Literal(s, self.language, self.datatype)

    def __lt__(self, other):
        if other is None:
            return False # Nothing is less than None
        try:
            return self._cmp_value < other
        except TypeError, te:
            return unicode(self._cmp_value) < other

    def __le__(self, other):
        if other is None:
            return False
        if self==other:
            return True
        else:
            return self < other

    def __gt__(self, other):
        if other is None:
            return True # Everything is greater than None
        try:
            return self._cmp_value > other
        except TypeError, te:
            return unicode(self._cmp_value) > other

    def __ge__(self, other):
        if other is None:
            return False
        if self==other:
            return True
        else:
            return self > other

    def __ne__(self, other):
        """
        Overriden to ensure property result for comparisons with None via !=.
        Routes all other such != and <> comparisons to __eq__

        >>> Literal('') != None
        True
        >>> Literal('2') <> Literal('2')
        False

        """
        if other is None:
            return True
        else:
            return not self.__eq__(other)

    def __eq__(self, other):
        """
        >>> f = URIRef("foo")
        >>> f is None or f == ''
        False
        >>> Literal("1", datatype=URIRef("foo")) == Literal("1", datatype=URIRef("foo"))
        True
        >>> Literal("1", datatype=URIRef("foo")) == Literal("2", datatype=URIRef("foo"))
        False
        >>> Literal("1", datatype=URIRef("foo")) == "asdf"
        False
        >>> oneInt     = Literal(1)
        >>> oneNoDtype = Literal('1')
        >>> oneInt == oneNoDtype
        False
        >>> Literal("1",_XSD_NS[u'string']) == Literal("1",_XSD_NS[u'string'])
        True
        >>> Literal("one",lang="en") == Literal("one",lang="en")
        True
        >>> Literal("hast",lang='en') == Literal("hast",lang='de')
        False
        >>> oneInt == Literal(1)
        True
        >>> oneFloat   = Literal(1.0)
        >>> oneInt == oneFloat
        True
        >>> oneInt == 1
        True
        """
        if other is None:
            return False
        else:
            return self._cmp_value==other

    def n3(self):
        language = self.language
        datatype = self.datatype
        # unfortunately this doesn't work: a newline gets encoded as \\n, which is ok in sourcecode, but we want \n
        #encoded = self.encode('unicode-escape').replace('\\', '\\\\').replace('"','\\"')
        #encoded = self.replace.replace('\\', '\\\\').replace('"','\\"')

        # TODO: We could also chose quotes based on the quotes appearing in the string, i.e. '"' and "'" ...

        # which is nicer?
        #if self.find("\"")!=-1 or self.find("'")!=-1 or self.find("\n")!=-1:
        if self.find("\n")!=-1:
            # Triple quote this string.
            encoded=self.replace('\\', '\\\\')
            if self.find('"""')!=-1:
                # is this ok?
                encoded=encoded.replace('"""','\\"""')
            if encoded.endswith('"'): encoded=encoded[:-1]+"\\\""
            encoded='"""%s"""'%encoded
        else:
            encoded='"%s"'%self.replace('\n','\\n').replace('\\', '\\\\').replace('"','\\"')
        if language:
            if datatype:
                return '%s@%s^^<%s>' % (encoded, language, datatype)
            else:
                return '%s@%s' % (encoded, language)
        else:
            if datatype:
                return '%s^^<%s>' % (encoded, datatype)
            else:
                return '%s' % encoded

    def __str__(self):
        return self.encode("unicode-escape")

    def __repr__(self):
        return """rdflib.Literal('%s', language=%s, datatype=%s)""" % (str(self), repr(self.language), repr(self.datatype))

    def _toPython(self):
        """
        Returns an appropriate python datatype derived from this RDF Literal
        """
        klass,convFunc = _XSDToPython.get(self.datatype,(None,None))
        rt = self
        if convFunc:
            rt = convFunc(rt)
        if klass:
            rt = klass(rt)
        if rt is self:
            if self.language is None and self.datatype is None:
                return unicode(rt)
            else:
                return (unicode(rt), rt.datatype, rt.language)
        return rt

_XSD_NS = Namespace(u'http://www.w3.org/2001/XMLSchema#')

#Casts a python datatype to a tuple of the lexical value and a datatype URI (or None)
def _castPythonToLiteral(obj):
    for pType,(castFunc,dType) in _PythonToXSD.items():
        if isinstance(obj,pType):
            if castFunc:
                return castFunc(obj),dType
            elif dType:
                return obj,dType
            else:
                return obj,None
    return obj, None # TODO: is this right for the fall through case?

#Mappings from Python types to XSD datatypes and back (burrowed from sparta)
_PythonToXSD = {
    basestring : (None,None),
    float      : (None,_XSD_NS[u'float']),
    int        : (None,_XSD_NS[u'int']),
    long       : (None,_XSD_NS[u'long']),
    bool       : (None,_XSD_NS[u'boolean']),
    date       : (lambda i:i.isoformat(),_XSD_NS[u'date']),
    time       : (lambda i:i.isoformat(),_XSD_NS[u'time']),
    datetime   : (lambda i:i.isoformat(),_XSD_NS[u'dateTime']),
}

def _strToTime(v) :
    return strptime(v,"%H:%M:%S")

def _strToDate(v) :
    tstr = strptime(v,"%Y-%m-%d")
    return date(tstr.tm_year,tstr.tm_mon,tstr.tm_mday)

def _strToDateTime(v) :
    """
    Attempt to cast to datetime, or just return the string (otherwise)
    """
    try:
        tstr = strptime(v,"%Y-%m-%dT%H:%M:%S")
    except:
        try:
            tstr = strptime(v,"%Y-%m-%dT%H:%M:%SZ")
        except:
            try:
                tstr = strptime(v,"%Y-%m-%dT%H:%M:%S%Z")
            except:
                return v

    return datetime(tstr.tm_year,tstr.tm_mon,tstr.tm_mday,tstr.tm_hour,tstr.tm_min,tstr.tm_sec)

_XSDToPython = {
    _XSD_NS[u'time']               : (None,_strToTime),
    _XSD_NS[u'date']               : (None,_strToDate),
    _XSD_NS[u'dateTime']           : (None,_strToDateTime),
    _XSD_NS[u'string']             : (None,None),
    _XSD_NS[u'normalizedString']   : (None,None),
    _XSD_NS[u'token']              : (None,None),
    _XSD_NS[u'language']           : (None,None),
    _XSD_NS[u'boolean']            : (None, lambda i:i.lower() in ['1','true']),
    _XSD_NS[u'decimal']            : (float,None),
    _XSD_NS[u'integer']            : (long ,None),
    _XSD_NS[u'nonPositiveInteger'] : (int,None),
    _XSD_NS[u'long']               : (long,None),
    _XSD_NS[u'nonNegativeInteger'] : (int, None),
    _XSD_NS[u'negativeInteger']    : (int, None),
    _XSD_NS[u'int']                : (long, None),
    _XSD_NS[u'unsignedLong']       : (long, None),
    _XSD_NS[u'positiveInteger']    : (int, None),
    _XSD_NS[u'short']              : (int, None),
    _XSD_NS[u'unsignedInt']        : (long, None),
    _XSD_NS[u'byte']               : (int, None),
    _XSD_NS[u'unsignedShort']      : (int, None),
    _XSD_NS[u'unsignedByte']       : (int, None),
    _XSD_NS[u'float']              : (float, None),
    _XSD_NS[u'double']             : (float, None),
    _XSD_NS[u'base64Binary']       : (base64.decodestring, None),
    _XSD_NS[u'anyURI']             : (None,None),
}

def test():
    import doctest
    doctest.testmod()

if __name__ == '__main__':
    test()
<|endoftext|>


<commit_before>from pyramid.response import Response
from pyramid.view import view_config
from contentbase import Item
from collections import OrderedDict
import cgi
from urllib.parse import (
    parse_qs,
    urlencode,
)

def includeme(config):
    config.add_route('batch_hub', '/batch_hub/{search_params}/{txt}')
    config.add_route('batch_hub:trackdb', '/batch_hub/{search_params}/{assembly}/{txt}')
    config.scan(__name__)

TAB = '\t'
NEWLINE = '\n'
HUB_TXT = 'hub.txt'
GENOMES_TXT = 'genomes.txt'
TRACKDB_TXT = 'trackDb.txt'
BIGWIG_FILE_TYPES = ['bigWig']
BIGBED_FILE_TYPES = ['bigBed']
FILE_QUERY = {
    'files.file_format': BIGBED_FILE_TYPES + BIGWIG_FILE_TYPES,
    'limit': ['all'],
    'frame': ['embedded']
}

def render(data):
    arr = []
    for i in range(len(data)):
        temp = list(data.popitem())
        str1 = ' '.join(temp)
        arr.append(str1)
    return arr

def get_parent_track(accession, label, visibility):
    parent = OrderedDict([
        ('sortOrder', 'view=+'),
        ('type', 'bed 3'),
        ('subGroup1', 'view Views PK=Peaks SIG=Signals'),
        ('dragAndDrop', 'subTracks'),
        ('visibility', visibility),
        ('compositeTrack', 'on'),
        ('longLabel', label),
        ('shortLabel', label),
        ('track', accession)
    ])
    parent_array = render(parent)
    return NEWLINE.join(parent_array)

def get_track(f, label, parent):
    '''Returns tracks for each file'''

    file_format = 'bigWig'
    sub_group = 'view=SIG'

    if f['file_format'] in BIGBED_FILE_TYPES:
        sub_group = 'view=PK'
        file_format = 'bigBed 6 +'
        if f.get('file_format_type') == 'gappedPeak':
            file_format = 'bigBed 12 +'

    label = label + ' - {title} {format} {output}'.format(
        title=f['title'],
        format=f.get('file_format_type', ''),
        output=f['output_type']
    )

    replicate_number = 'pooled'
    if 'replicate' in f:
        replicate_number = 'rep {rep}'.format(
            rep=str(f['replicate']['biological_replicate_number'])
        )

    track = OrderedDict([
        ('subGroups', sub_group),
        ('visibility', 'dense'),
        ('longLabel', label + ' ' + replicate_number),
        ('shortLabel', f['title']),
        ('parent', parent + ' on'),
        ('bigDataUrl', '{href}?proxy=true'.format(**f)),
        ('type', file_format),
        ('track', f['title']),
    ])

    if parent == '':
        del(track['parent'])
        del(track['subGroups'])
        track_array = render(track)
        return NEWLINE.join(track_array)
    track_array = render(track)
    return (NEWLINE + (2 * TAB)).join(track_array)

def get_peak_view(accession, view):
    s_label = view + 's'
    track_name = view + 'View'
    view_data = OrderedDict([
        ('autoScale', 'on'),
        ('type', 'bigBed 6 +'),
        ('viewUi', 'on'),
        ('visibility', 'dense'),
        ('view', 'PK'),
        ('shortLabel', s_label),
        ('parent', accession),
        ('track', track_name)
    ])
    view_array = render(view_data)
    return (NEWLINE + TAB).join(view_array)

def get_signal_view(accession, view):
    s_label = view + 's'
    track_name = view + 'View'
    view_data = OrderedDict([
        ('autoScale', 'on'),
        ('maxHeightPixels', '100:32:8'),
        ('type', 'bigWig'),
        ('viewUi', 'on'),
        ('visibility', 'dense'),
        ('view', 'SIG'),
        ('shortLabel', s_label),
        ('parent', accession),
        ('track', track_name)
    ])
    view_array = render(view_data)
    return (NEWLINE + TAB).join(view_array)

def get_genomes_txt(assembly):
    genome = OrderedDict([
        ('trackDb', assembly + '/trackDb.txt'),
        ('genome', assembly)
    ])
    return render(genome)

def get_hub(label):
    hub = OrderedDict([
        ('email', 'encode-help@lists.stanford.edu'),
        ('genomesFile', 'genomes.txt'),
        ('longLabel', 'ENCODE Data Coordination Center Data Hub'),
        ('shortLabel', 'Hub (' + label + ')'),
        ('hub', 'ENCODE_DCC_' + label)
    ])
    return render(hub)

def generate_trackDb(embedded, visibility, assembly=None):

    files = embedded.get('files', None)

    # checks if there is assembly specified for each experiment
    new_files = []
    if assembly is not None:
        for f in files:
            if 'assembly' in f and f['assembly'] == assembly:
                new_files.append(f)
    if len(new_files):
        files = new_files

    long_label = '{assay_term_name} of {biosample_term_name} - {accession}'.format(
        assay_term_name=embedded['assay_term_name'],
        biosample_term_name=embedded['biosample_term_name'],
        accession=embedded['accession']
    )
    if 'target' in embedded:
        long_label = long_label + '(Target - {label})'.format(
            label=embedded['target']['label']
        )
    parent = get_parent_track(embedded['accession'], long_label, visibility)
    track_label = '{assay} of {biosample} - {accession}'.format(
        assay=embedded['assay_term_name'],
        biosample=embedded['biosample_term_name'],
        accession=embedded['accession']
    )
    peak_view = ''
    signal_view = ''
    signal_count = 0
    call_count = 0
    for f in files:
        if f['file_format'] in BIGBED_FILE_TYPES:
            if call_count == 0:
                peak_view = get_peak_view(
                    embedded['accession'],
                    embedded['accession'] + 'PK'
                ) + NEWLINE + (2 * TAB)
            else:
                peak_view = peak_view + NEWLINE
            peak_view = peak_view + NEWLINE + (2 * TAB) + get_track(
                f, track_label,
                embedded['accession'] + 'PKView'
            )
            call_count = call_count + 1
        elif f['file_format'] == 'bigWig':
            if signal_count == 0:
                signal_view = get_signal_view(
                    embedded['accession'],
                    embedded['accession'] + 'SIG'
                ) + NEWLINE + (2 * TAB)
            else:
                signal_view = signal_view + NEWLINE
            signal_view = signal_view + NEWLINE + (2 * TAB) + get_track(
                f, track_label,
                embedded['accession'] + 'SIGView'
            )
            signal_count = signal_count + 1
    if signal_view == '':
        parent = parent + (NEWLINE * 2) + TAB + peak_view
    elif peak_view == '':
        parent = parent + (NEWLINE * 2) + TAB + signal_view
    else:
        parent = parent + (NEWLINE * 2) + TAB + peak_view + (NEWLINE * 2) + TAB + signal_view
    if not parent.endswith('\n'):
        parent = parent + '\n'
    return parent

def generate_html(context, request):
    ''' Generates and returns HTML for the track hub'''

    url_ret = (request.url).split('@@hub')
    embedded = {}
    if url_ret[0] == request.url:
        item = request.root.__getitem__((request.url.split('/')[-1])[:-5])
        embedded = request.embed(request.resource_path(item))
    else:
        embedded = request.embed(request.resource_path(context))
    link = request.host_url + '/experiments/' + embedded['accession']
    files_json = embedded.get('files', None)
    data_accession = '{accession}
' \
        .format(link=link, accession=embedded['accession'])
    data_description = '
{description}
' \
        .format(description=cgi.escape(embedded['description']))
    data_files = ''
    for f in files_json:
        if f['file_format'] in BIGBED_FILE_TYPES + BIGWIG_FILE_TYPES:
            replicate_number = 'pooled'
            if 'replicate' in f:
                replicate_number = str(f['replicate']['biological_replicate_number'])
            data_files = data_files + \
                '
{title}	{file_format_type}	{output_type}	{replicate_number}	Click here
'\
                .format(replicate_number=replicate_number, request=request, **f)

    file_table = '

Accession	File format	Output type	Biological replicate	Download link
{files}
' \
        .format(files=data_files)
    header = '
This trackhub was automatically generated from the files and metadata for the experiment - ' + \
        data_accession
    return data_description + header + file_table

def generate_batch_hubs(context, request):
    '''search for the input params and return the trackhub'''

    results = {}
    txt = request.matchdict['txt']
    param_list = parse_qs(request.matchdict['search_params'].replace(',,', '&'))

    if len(request.matchdict) == 3:

        # Should generate a HTML page for requests other than trackDb.txt
        if txt != TRACKDB_TXT:
            data_policy = '
ENCODE data use policy
'
            return generate_html(context, request) + data_policy

        assembly = str(request.matchdict['assembly'])
        params = dict(param_list, **FILE_QUERY)
        results = []
        params['assembly'] = [assembly]

        # if files.file_format is a input param
        if 'files.file_format' in param_list:
            params['files.file_format'] = param_list['files.file_format']
            path = '/search/?%s' % urlencode(params, True)
            for result in request.embed(path, as_user=True)['@graph']:
                if 'files' in result:
                    for f in result['files']:
                        if f['file_format'] in BIGWIG_FILE_TYPES + BIGBED_FILE_TYPES:
                            results.append(result)
                        break
        else:
            path = '/search/?%s' % urlencode(params, True)
            results = request.embed(path, as_user=True)['@graph']
        trackdb = ''
        for i, experiment in enumerate(results):
            if i < 5:
                if i == 0:
                    trackdb = generate_trackDb(experiment, 'full', None)
                else:
                    trackdb = trackdb + NEWLINE + generate_trackDb(experiment, 'full', None)
            else:
                trackdb = trackdb + NEWLINE + generate_trackDb(experiment, 'hide', None)
        return trackdb
    elif txt == HUB_TXT:
        return NEWLINE.join(get_hub('search'))
    elif txt == GENOMES_TXT:
        path = '/search/?%s' % urlencode(param_list, True)
        results = request.embed(path, as_user=True)
        g_text = ''
        if 'assembly' in param_list:
            for assembly in param_list.get('assembly'):
                if g_text == '':
                    g_text = NEWLINE.join(get_genomes_txt(assembly))
                else:
                    g_text = g_text + 2 * NEWLINE + NEWLINE.join(get_genomes_txt(assembly))
        else:
            for facet in results['facets']:
                if facet['field'] == 'assembly':
                    for term in facet['terms']:
                        if term['doc_count'] != 0:
                            if g_text == '':
                                g_text = NEWLINE.join(get_genomes_txt(term['key']))
                            else:
                                g_text = g_text + 2 * NEWLINE + NEWLINE.join(get_genomes_txt(term['key']))
        return g_text

@view_config(name='hub', context=Item, request_method='GET', permission='view')
def hub(context, request):
    ''' Creates trackhub on fly for a given experiment '''

    url_ret = (request.url).split('@@hub')
    embedded = request.embed(request.resource_path(context))

    assemblies = ''
    if 'assembly' in embedded:
        assemblies = embedded['assembly']

    if url_ret[1][1:] == HUB_TXT:
        return Response(
            NEWLINE.join(get_hub(embedded['accession'])),
            content_type='text/plain'
        )
    elif url_ret[1][1:] == GENOMES_TXT:
        g_text = ''
        for assembly in assemblies:
            if g_text == '':
                g_text = NEWLINE.join(get_genomes_txt(assembly))
            else:
                g_text = g_text + 2 * NEWLINE + NEWLINE.join(get_genomes_txt(assembly))
        return Response(g_text, content_type='text/plain')
    elif url_ret[1][1:].endswith(TRACKDB_TXT):
        parent_track = generate_trackDb(embedded, 'full', url_ret[1][1:].split('/')[0])
        return Response(parent_track, content_type='text/plain')
    else:
        data_policy = '
ENCODE data use policy
'
        return Response(generate_html(context, request) + data_policy, content_type='text/html')

@view_config(route_name='batch_hub')
@view_config(route_name='batch_hub:trackdb')
def batch_hub(context, request):
    ''' View for batch track hubs '''
    return Response(generate_batch_hubs(context, request), content_type='text/plain')
<commit_msg>restricting track hubs to only released experiments<commit_after>from pyramid.response import Response
from pyramid.view import view_config
from contentbase import Item
from collections import OrderedDict
import cgi
from urllib.parse import (
    parse_qs,
    urlencode,
)

def includeme(config):
    config.add_route('batch_hub', '/batch_hub/{search_params}/{txt}')
    config.add_route('batch_hub:trackdb', '/batch_hub/{search_params}/{assembly}/{txt}')
    config.scan(__name__)

TAB = '\t'
NEWLINE = '\n'
HUB_TXT = 'hub.txt'
GENOMES_TXT = 'genomes.txt'
TRACKDB_TXT = 'trackDb.txt'
BIGWIG_FILE_TYPES = ['bigWig']
BIGBED_FILE_TYPES = ['bigBed']
FILE_QUERY = {
    'files.file_format': BIGBED_FILE_TYPES + BIGWIG_FILE_TYPES,
    'limit': ['all'],
    'frame': ['embedded'],
    'status': ['released']
}

def render(data):
    arr = []
    for i in range(len(data)):
        temp = list(data.popitem())
        str1 = ' '.join(temp)
        arr.append(str1)
    return arr

def get_parent_track(accession, label, visibility):
    parent = OrderedDict([
        ('sortOrder', 'view=+'),
        ('type', 'bed 3'),
        ('subGroup1', 'view Views PK=Peaks SIG=Signals'),
        ('dragAndDrop', 'subTracks'),
        ('visibility', visibility),
        ('compositeTrack', 'on'),
        ('longLabel', label),
        ('shortLabel', label),
        ('track', accession)
    ])
    parent_array = render(parent)
    return NEWLINE.join(parent_array)

def get_track(f, label, parent):
    '''Returns tracks for each file'''

    file_format = 'bigWig'
    sub_group = 'view=SIG'

    if f['file_format'] in BIGBED_FILE_TYPES:
        sub_group = 'view=PK'
        file_format = 'bigBed 6 +'
        if f.get('file_format_type') == 'gappedPeak':
            file_format = 'bigBed 12 +'

    label = label + ' - {title} {format} {output}'.format(
        title=f['title'],
        format=f.get('file_format_type', ''),
        output=f['output_type']
    )

    replicate_number = 'pooled'
    if 'replicate' in f:
        replicate_number = 'rep {rep}'.format(
            rep=str(f['replicate']['biological_replicate_number'])
        )

    track = OrderedDict([
        ('subGroups', sub_group),
        ('visibility', 'dense'),
        ('longLabel', label + ' ' + replicate_number),
        ('shortLabel', f['title']),
        ('parent', parent + ' on'),
        ('bigDataUrl', '{href}?proxy=true'.format(**f)),
        ('type', file_format),
        ('track', f['title']),
    ])

    if parent == '':
        del(track['parent'])
        del(track['subGroups'])
        track_array = render(track)
        return NEWLINE.join(track_array)
    track_array = render(track)
    return (NEWLINE + (2 * TAB)).join(track_array)

def get_peak_view(accession, view):
    s_label = view + 's'
    track_name = view + 'View'
    view_data = OrderedDict([
        ('autoScale', 'on'),
        ('type', 'bigBed 6 +'),
        ('viewUi', 'on'),
        ('visibility', 'dense'),
        ('view', 'PK'),
        ('shortLabel', s_label),
        ('parent', accession),
        ('track', track_name)
    ])
    view_array = render(view_data)
    return (NEWLINE + TAB).join(view_array)

def get_signal_view(accession, view):
    s_label = view + 's'
    track_name = view + 'View'
    view_data = OrderedDict([
        ('autoScale', 'on'),
        ('maxHeightPixels', '100:32:8'),
        ('type', 'bigWig'),
        ('viewUi', 'on'),
        ('visibility', 'dense'),
        ('view', 'SIG'),
        ('shortLabel', s_label),
        ('parent', accession),
        ('track', track_name)
    ])
    view_array = render(view_data)
    return (NEWLINE + TAB).join(view_array)

def get_genomes_txt(assembly):
    genome = OrderedDict([
        ('trackDb', assembly + '/trackDb.txt'),
        ('genome', assembly)
    ])
    return render(genome)

def get_hub(label):
    hub = OrderedDict([
        ('email', 'encode-help@lists.stanford.edu'),
        ('genomesFile', 'genomes.txt'),
        ('longLabel', 'ENCODE Data Coordination Center Data Hub'),
        ('shortLabel', 'Hub (' + label + ')'),
        ('hub', 'ENCODE_DCC_' + label)
    ])
    return render(hub)

def generate_trackDb(embedded, visibility, assembly=None):

    files = embedded.get('files', None)

    # checks if there is assembly specified for each experiment
    new_files = []
    if assembly is not None:
        for f in files:
            if 'assembly' in f and f['assembly'] == assembly:
                new_files.append(f)
    if len(new_files):
        files = new_files

    long_label = '{assay_term_name} of {biosample_term_name} - {accession}'.format(
        assay_term_name=embedded['assay_term_name'],
        biosample_term_name=embedded['biosample_term_name'],
        accession=embedded['accession']
    )
    if 'target' in embedded:
        long_label = long_label + '(Target - {label})'.format(
            label=embedded['target']['label']
        )
    parent = get_parent_track(embedded['accession'], long_label, visibility)
    track_label = '{assay} of {biosample} - {accession}'.format(
        assay=embedded['assay_term_name'],
        biosample=embedded['biosample_term_name'],
        accession=embedded['accession']
    )
    peak_view = ''
    signal_view = ''
    signal_count = 0
    call_count = 0
    for f in files:
        if f['file_format'] in BIGBED_FILE_TYPES:
            if call_count == 0:
                peak_view = get_peak_view(
                    embedded['accession'],
                    embedded['accession'] + 'PK'
                ) + NEWLINE + (2 * TAB)
            else:
                peak_view = peak_view + NEWLINE
            peak_view = peak_view + NEWLINE + (2 * TAB) + get_track(
                f, track_label,
                embedded['accession'] + 'PKView'
            )
            call_count = call_count + 1
        elif f['file_format'] == 'bigWig':
            if signal_count == 0:
                signal_view = get_signal_view(
                    embedded['accession'],
                    embedded['accession'] + 'SIG'
                ) + NEWLINE + (2 * TAB)
            else:
                signal_view = signal_view + NEWLINE
            signal_view = signal_view + NEWLINE + (2 * TAB) + get_track(
                f, track_label,
                embedded['accession'] + 'SIGView'
            )
            signal_count = signal_count + 1
    if signal_view == '':
        parent = parent + (NEWLINE * 2) + TAB + peak_view
    elif peak_view == '':
        parent = parent + (NEWLINE * 2) + TAB + signal_view
    else:
        parent = parent + (NEWLINE * 2) + TAB + peak_view + (NEWLINE * 2) + TAB + signal_view
    if not parent.endswith('\n'):
        parent = parent + '\n'
    return parent

def generate_html(context, request):
    ''' Generates and returns HTML for the track hub'''

    url_ret = (request.url).split('@@hub')
    embedded = {}
    if url_ret[0] == request.url:
        item = request.root.__getitem__((request.url.split('/')[-1])[:-5])
        embedded = request.embed(request.resource_path(item))
    else:
        embedded = request.embed(request.resource_path(context))
    link = request.host_url + '/experiments/' + embedded['accession']
    files_json = embedded.get('files', None)
    data_accession = '{accession}
' \
        .format(link=link, accession=embedded['accession'])
    data_description = '
{description}
' \
        .format(description=cgi.escape(embedded['description']))
    data_files = ''
    for f in files_json:
        if f['file_format'] in BIGBED_FILE_TYPES + BIGWIG_FILE_TYPES:
            replicate_number = 'pooled'
            if 'replicate' in f:
                replicate_number = str(f['replicate']['biological_replicate_number'])
            data_files = data_files + \
                '
{title}	{file_format_type}	{output_type}	{replicate_number}	Click here
'\
                .format(replicate_number=replicate_number, request=request, **f)

    file_table = '

Accession	File format	Output type	Biological replicate	Download link
{files}
' \
        .format(files=data_files)
    header = '
This trackhub was automatically generated from the files and metadata for the experiment - ' + \
        data_accession
    return data_description + header + file_table

def generate_batch_hubs(context, request):
    '''search for the input params and return the trackhub'''

    results = {}
    txt = request.matchdict['txt']
    param_list = parse_qs(request.matchdict['search_params'].replace(',,', '&'))

    if len(request.matchdict) == 3:

        # Should generate a HTML page for requests other than trackDb.txt
        if txt != TRACKDB_TXT:
            data_policy = '
ENCODE data use policy
'
            return generate_html(context, request) + data_policy

        assembly = str(request.matchdict['assembly'])
        params = dict(param_list, **FILE_QUERY)
        results = []
        params['assembly'] = [assembly]

        # if files.file_format is a input param
        if 'files.file_format' in param_list:
            params['files.file_format'] = param_list['files.file_format']
            path = '/search/?%s' % urlencode(params, True)
            for result in request.embed(path, as_user=True)['@graph']:
                if 'files' in result:
                    for f in result['files']:
                        if f['file_format'] in BIGWIG_FILE_TYPES + BIGBED_FILE_TYPES:
                            results.append(result)
                        break
        else:
            path = '/search/?%s' % urlencode(params, True)
            results = request.embed(path, as_user=True)['@graph']
        trackdb = ''
        for i, experiment in enumerate(results):
            if i < 5:
                if i == 0:
                    trackdb = generate_trackDb(experiment, 'full', None)
                else:
                    trackdb = trackdb + NEWLINE + generate_trackDb(experiment, 'full', None)
            else:
                trackdb = trackdb + NEWLINE + generate_trackDb(experiment, 'hide', None)
        return trackdb
    elif txt == HUB_TXT:
        return NEWLINE.join(get_hub('search'))
    elif txt == GENOMES_TXT:
        path = '/search/?%s' % urlencode(param_list, True)
        results = request.embed(path, as_user=True)
        g_text = ''
        if 'assembly' in param_list:
            for assembly in param_list.get('assembly'):
                if g_text == '':
                    g_text = NEWLINE.join(get_genomes_txt(assembly))
                else:
                    g_text = g_text + 2 * NEWLINE + NEWLINE.join(get_genomes_txt(assembly))
        else:
            for facet in results['facets']:
                if facet['field'] == 'assembly':
                    for term in facet['terms']:
                        if term['doc_count'] != 0:
                            if g_text == '':
                                g_text = NEWLINE.join(get_genomes_txt(term['key']))
                            else:
                                g_text = g_text + 2 * NEWLINE + NEWLINE.join(get_genomes_txt(term['key']))
        return g_text

@view_config(name='hub', context=Item, request_method='GET', permission='view')
def hub(context, request):
    ''' Creates trackhub on fly for a given experiment '''

    url_ret = (request.url).split('@@hub')
    embedded = request.embed(request.resource_path(context))

    assemblies = ''
    if 'assembly' in embedded:
        assemblies = embedded['assembly']

    if url_ret[1][1:] == HUB_TXT:
        return Response(
            NEWLINE.join(get_hub(embedded['accession'])),
            content_type='text/plain'
        )
    elif url_ret[1][1:] == GENOMES_TXT:
        g_text = ''
        for assembly in assemblies:
            if g_text == '':
                g_text = NEWLINE.join(get_genomes_txt(assembly))
            else:
                g_text = g_text + 2 * NEWLINE + NEWLINE.join(get_genomes_txt(assembly))
        return Response(g_text, content_type='text/plain')
    elif url_ret[1][1:].endswith(TRACKDB_TXT):
        parent_track = generate_trackDb(embedded, 'full', url_ret[1][1:].split('/')[0])
        return Response(parent_track, content_type='text/plain')
    else:
        data_policy = '
ENCODE data use policy
'
        return Response(generate_html(context, request) + data_policy, content_type='text/html')

@view_config(route_name='batch_hub')
@view_config(route_name='batch_hub:trackdb')
def batch_hub(context, request):
    ''' View for batch track hubs '''
    return Response(generate_batch_hubs(context, request), content_type='text/plain')
<|endoftext|>