<commit_before>################################################################################
#
#   MRC FGU Computational Genomics Group
#
#   $Id$
#
#   Copyright (C) 2009 Andreas Heger
#
#   This program is free software; you can redistribute it and/or
#   modify it under the terms of the GNU General Public License
#   as published by the Free Software Foundation; either version 2
#   of the License, or (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#################################################################################
'''
Bed.py - Tools for working with bed files
=========================================

:Author: Andreas Heger
:Release: $Id$
:Date: |today|
:Tags: Python

Code
----

'''
import re
import ncl
import numpy
import bisect
import itertools

class Bed(object):
    """an interval in bed format."""

    map_key2field = { 'name' : 0,
                      'score' : 1,
                      'strand' : 2,
                      'thickStart' : 3,
                      'thickEnd' : 4,
                      'itemRGB' : 5,
                      'blockCount': 6,
                      'blockSizes': 7,
                      'blockStarts': 8 }

    def __init__(self):
        '''empty constructor.'''
        self.contig = None
        self.start = 0
        self.end = 0
        self.mFields = []
        self.mTrack = None

    def __str__(self):
        return "\t".join( (self.contig, str(self.start), str(self.end) ) + tuple(map(str, self.mFields)))

    def fromGFF( self, gff, is_gtf = False, name=None ):
        """fill from gff formatted entry."""
        self.contig, self.start, self.end = gff.contig, gff.start, gff.end
        try:
            self.mFields = [getattr( gff, name),
                            [gff.score,0][gff.score == None],
                            gff.strand ]
        except AttributeError:
            self.mFields = [gff[name],
                            [gff.score,0][gff.score == None],
                            gff.strand ]

    def __contains__(self, key ):
        return self.map_key2field[key] < len(self.mFields)

    def __getitem__(self, key):
        return self.mFields[self.map_key2field[key]]

    def __getattr__(self, key ):
        return self.mFields[self.map_key2field[key]]

    @property
    def columns(self):
        '''return number of columns in bed-entry.'''
        return 3 + len(self.mFields)

class Track(object):
    '''bed track information.'''
    def __init__(self, line ):
        r= re.compile('([^ =]+) *= *("[^"]*"|[^ ]*)')

        self._d = {}
        for k, v in r.findall(line[:-1]):
            if v[:1]=='"':
                self._d[k] = v[1:-1]
            else:
                self._d[k] = v

        self._line = line[:-1]
    def __str__(self):
        return self._line

    def __getitem__(self, key): return self._d[key]
    def __setitem__(self, key,val): self._d[key] = val

def iterator( infile ):
    """iterate over a bed formatted file.

    The iterator is :term:`track` aware.

    This iterator yields :class:`Bed` objects.
    """

    track = None
    for line in infile:
        if line.startswith("track"):
            track = Track( line )
            continue
        # ignore comments
        if line.startswith("#"): continue
        # ignore empty lines (in order to parse pseudo bed files)
        if line.strip() == "": continue

        b = Bed()
        data = line[:-1].split()
        try:
            b.contig, b.start, b.end = data[0], int(data[1]), int(data[2])
        except IndexError:
            raise ValueError("parsing error in line '%s'" % line[:-1])
        b.mFields = data[3:]
        b.mTrack = track
        yield b

# for compatibility, remove
def bed_iterator( infile ):
    return iterator( infile )

def setName( iterator ):
    '''yield bed entries in which name is set if unset.
    '''
    for i, bed in enumerate(iterator):
        if "name" not in bed:
            bed.name = str(i)
        yield bed

def grouped_iterator( iterator ):
    '''yield bed results grouped by track.'''
    return itertools.groupby( iterator, lambda x: x.mTrack )

def blocked_iterator( iterator ):
    '''yield blocked bed results.'''

    last_id = None
    blocks = []

    def _update( bed, blocks ):
        blocks.sort()
        bed.start, bed.end = blocks[0][0], blocks[-1][1]
        s = bed.start
        # hacky - needs be abstracted into Bed object
        bed.mFields.extend( [""] * (9 - len(bed.mFields)))
        bed.mFields[3] = str(bed.start)
        bed.mFields[4] = str(bed.end)
        bed.mFields[5] = 0
        bed.mFields[6] = len(blocks)
        bed.mFields[7] = ",".join( [str(y-x) for x,y in blocks ])
        bed.mFields[8] = ",".join( [str(x-s) for x,y in blocks])
        return bed

    last_bed = None
    for bed in iterator:
        if last_id != bed.name:
            if last_id: yield _update(last_bed, blocks)
            blocks = []
            last_id = bed.name
        last_bed = bed
        blocks.append( (bed.start, bed.end) )

    yield _update(bed, blocks)

def readAndIndex( infile, with_values = False, per_track = False ):
    """read and index a bed formatted file in ``infile``.

    If ``with_values`` is set, the original bed entry will be kept for
    further reference. Otherwise only the intervals will be indexed and
    any additional fields in the bed entry will be ignored.

    The default is to use all intervals. If per_track is set,
    separate indices will be build for each track.
    """

    if with_values:
        idx_factory = ncl.NCL
    else:
        idx_factory = ncl.NCLSimple

    def _build( iter ):
        idx = {}
        for e in iter:
            if e.contig not in idx: idx[e.contig] = idx_factory()
            try:
                if with_values:
                    idx[e.contig].add( e.start,e.end,e )
                else:
                    idx[e.contig].add( e.start,e.end )
            except ValueError:
                # ignore zero-length intervals
                pass
        return idx

    if per_track:
        indices = {}
        for track, beds in grouped_iterator( bed_iterator( infile ) ):
            if track == None:
                return _build(beds)
            else:
                indices[track["name"]] = _build( beds )
        return indices
    else:
        return _build( bed_iterator( infile ) )

def binIntervals( iterator, num_bins = 5, method = "equal-bases", bin_edges = None ):
    '''merge adjacent bins by score.

    Several merging methods are possible:

    equal-bases
       merge intervals such that each bin contains the equal number of bases

    equal-intervals
       merge intervals such that each bin contains the equal number intervals

    This options requires the fifth field (score) of the bed input
    file to be present.

    bins should be non-overlapping.

    returns a list of intervals (:class:`Bed`) and the bin_edges
    '''

    data = []
    beds = list(iterator)

    for bed in beds: bed.mFields[1] = float( bed.mFields[1])

    if bin_edges == None:
        if method == "equal-bases":
            data = numpy.array( sorted( [(x.mFields[1], x.end-x.start) for x in beds ] ) )
        elif method == "equal-intervals":
            data = numpy.array( sorted( [(x.mFields[1], 1) for x in beds ] ) )
        elif method == "equal-range":
            vals = [x.mFields[1] for x in beds ]
            mi, ma = min( vals ), max( vals )
            increment = float(ma - mi) / num_bins
            bin_edges = numpy.arange( mi, ma, increment)
        else:
            raise ValueError("unknown method %s to compute bins, supply bin_edges" % method )

    if bin_edges == None:
        sums = data[:,1].cumsum( axis=0 )
        total = float(sums[-1])
        increment = float( total / num_bins )
        bin_edges = [data[0][0]]
        occupancies = []
        threshold = increment
        occ = 0
        for v, s in zip( data, sums ):
            occ += v[1]
            if s > threshold:
                bin_edges.append( v[0] )
                threshold += increment
                occupancies.append( occ )
                occ = 0

        bin_edges.append( data[-1][0] + 1 )

    beds.sort( key=lambda x: (x.contig,x.start) )
    last_contig, start, end, last_name = None, None, None, None
    new_beds = []
    for bed in beds:
        name = bisect.bisect_right(bin_edges, bed.mFields[1])-1
        contig = bed.contig
        if name != last_name or last_contig != contig:
            if last_name != None:
                b = Bed()
                b.contig, b.start, b.end, b.mFields = last_contig, start, end, [last_name]
                new_beds.append(b)
            start = bed.start
            last_name = name
            last_contig = contig

        end = bed.end

    if last_name != None:
        b = Bed()
        b.contig, b.start, b.end, b.mFields = contig, start, end, [last_name]
        new_beds.append(b)

    return new_beds, bin_edges

def merge( iterator ):
    '''merge overlapping intervals.

    returns a list of merged intervals.
    '''

    beds = list(iterator)
    if len(beds) == 0: return []

    beds.sort( key = lambda x: (x.contig, x.start) )

    def iterate_chunks( beds ):

        last = beds[0]
        to_join = [last]
        end = last.end
        contig = last.contig

        for this in beds[1:]:
            d = this.start - end
            if this.contig != contig or d >= 0:
                yield to_join
                contig = this.contig
                to_join = []
                end = this.end

            end = max(end, this.end)
            to_join.append( this )

        yield to_join
        raise StopIteration

    n = []
    for to_join in iterate_chunks(beds):

        y = Bed()
        y.contig = to_join[0].contig
        y.start = to_join[0].start
        y.end = to_join[-1].end
        n.append( y )

    return n
<commit_msg>Added __setattr__ method to Bed class in Bed.py<commit_after>################################################################################
#
#   MRC FGU Computational Genomics Group
#
#   $Id$
#
#   Copyright (C) 2009 Andreas Heger
#
#   This program is free software; you can redistribute it and/or
#   modify it under the terms of the GNU General Public License
#   as published by the Free Software Foundation; either version 2
#   of the License, or (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#################################################################################
'''
Bed.py - Tools for working with bed files
=========================================

:Author: Andreas Heger
:Release: $Id$
:Date: |today|
:Tags: Python

Code
----

'''
import re
import ncl
import numpy
import bisect
import itertools

class Bed(object):
    """an interval in bed format."""

    map_key2field = { 'name' : 0,
                      'score' : 1,
                      'strand' : 2,
                      'thickStart' : 3,
                      'thickEnd' : 4,
                      'itemRGB' : 5,
                      'blockCount': 6,
                      'blockSizes': 7,
                      'blockStarts': 8 }

    def __init__(self):
        '''empty constructor.'''
        self.contig = None
        self.start = 0
        self.end = 0
        self.mFields = []
        self.mTrack = None

    def __str__(self):
        return "\t".join( (self.contig, str(self.start), str(self.end) ) + tuple(map(str, self.mFields)))

    def fromGFF( self, gff, is_gtf = False, name=None ):
        """fill from gff formatted entry."""
        self.contig, self.start, self.end = gff.contig, gff.start, gff.end
        try:
            self.mFields = [getattr( gff, name),
                            [gff.score,0][gff.score == None],
                            gff.strand ]
        except AttributeError:
            self.mFields = [gff[name],
                            [gff.score,0][gff.score == None],
                            gff.strand ]

    def __contains__(self, key ):
        return self.map_key2field[key] < len(self.mFields)

    def __getitem__(self, key):
        return self.mFields[self.map_key2field[key]]

    def __getattr__(self, key ):
        try:
            return self.mFields[self.map_key2field[key]]
        except IndexError:
            print "Key = %s" % key
            print "mFields = " + str(self.mFields)
            print "Bed = " + str(self)
            raise

    def __setattr__self, key, value ):
        #IMS overwrite the standard set attr so that if we add a name, strand or score
        #to a Bed object without one, it gets stored in the right place.

        #first check if the key is a standard one
        if key in self.map_key2field:
            mFields[self.map_key2field[key]] = value
        else:
            self.__dict__[key] = value

    @property
    def columns(self):
        '''return number of columns in bed-entry.'''
        return 3 + len(self.mFields)

class Track(object):
    '''bed track information.'''
    def __init__(self, line ):
        r= re.compile('([^ =]+) *= *("[^"]*"|[^ ]*)')

        self._d = {}
        for k, v in r.findall(line[:-1]):
            if v[:1]=='"':
                self._d[k] = v[1:-1]
            else:
                self._d[k] = v

        self._line = line[:-1]
    def __str__(self):
        return self._line

    def __getitem__(self, key): return self._d[key]
    def __setitem__(self, key,val): self._d[key] = val

def iterator( infile ):
    """iterate over a bed formatted file.

    The iterator is :term:`track` aware.

    This iterator yields :class:`Bed` objects.
    """

    track = None
    for line in infile:
        if line.startswith("track"):
            track = Track( line )
            continue
        # ignore comments
        if line.startswith("#"): continue
        # ignore empty lines (in order to parse pseudo bed files)
        if line.strip() == "": continue

        b = Bed()
        data = line[:-1].split()
        try:
            b.contig, b.start, b.end = data[0], int(data[1]), int(data[2])
        except IndexError:
            raise ValueError("parsing error in line '%s'" % line[:-1])
        b.mFields = data[3:]
        b.mTrack = track
        yield b

# for compatibility, remove
def bed_iterator( infile ):
    return iterator( infile )

def setName( iterator ):
    '''yield bed entries in which name is set if unset.
    '''
    for i, bed in enumerate(iterator):
        if "name" not in bed:
            bed.name = str(i)
        yield bed

def grouped_iterator( iterator ):
    '''yield bed results grouped by track.'''
    return itertools.groupby( iterator, lambda x: x.mTrack )

def blocked_iterator( iterator ):
    '''yield blocked bed results.'''

    last_id = None
    blocks = []

    def _update( bed, blocks ):
        blocks.sort()
        bed.start, bed.end = blocks[0][0], blocks[-1][1]
        s = bed.start
        # hacky - needs be abstracted into Bed object
        bed.mFields.extend( [""] * (9 - len(bed.mFields)))
        bed.mFields[3] = str(bed.start)
        bed.mFields[4] = str(bed.end)
        bed.mFields[5] = 0
        bed.mFields[6] = len(blocks)
        bed.mFields[7] = ",".join( [str(y-x) for x,y in blocks ])
        bed.mFields[8] = ",".join( [str(x-s) for x,y in blocks])
        return bed

    last_bed = None
    for bed in iterator:
        if last_id != bed.name:
            if last_id: yield _update(last_bed, blocks)
            blocks = []
            last_id = bed.name
        last_bed = bed
        blocks.append( (bed.start, bed.end) )

    yield _update(bed, blocks)

def readAndIndex( infile, with_values = False, per_track = False ):
    """read and index a bed formatted file in ``infile``.

    If ``with_values`` is set, the original bed entry will be kept for
    further reference. Otherwise only the intervals will be indexed and
    any additional fields in the bed entry will be ignored.

    The default is to use all intervals. If per_track is set,
    separate indices will be build for each track.
    """

    if with_values:
        idx_factory = ncl.NCL
    else:
        idx_factory = ncl.NCLSimple

    def _build( iter ):
        idx = {}
        for e in iter:
            if e.contig not in idx: idx[e.contig] = idx_factory()
            try:
                if with_values:
                    idx[e.contig].add( e.start,e.end,e )
                else:
                    idx[e.contig].add( e.start,e.end )
            except ValueError:
                # ignore zero-length intervals
                pass
        return idx

    if per_track:
        indices = {}
        for track, beds in grouped_iterator( bed_iterator( infile ) ):
            if track == None:
                return _build(beds)
            else:
                indices[track["name"]] = _build( beds )
        return indices
    else:
        return _build( bed_iterator( infile ) )

def binIntervals( iterator, num_bins = 5, method = "equal-bases", bin_edges = None ):
    '''merge adjacent bins by score.

    Several merging methods are possible:

    equal-bases
       merge intervals such that each bin contains the equal number of bases

    equal-intervals
       merge intervals such that each bin contains the equal number intervals

    This options requires the fifth field (score) of the bed input
    file to be present.

    bins should be non-overlapping.

    returns a list of intervals (:class:`Bed`) and the bin_edges
    '''

    data = []
    beds = list(iterator)

    for bed in beds: bed.mFields[1] = float( bed.mFields[1])

    if bin_edges == None:
        if method == "equal-bases":
            data = numpy.array( sorted( [(x.mFields[1], x.end-x.start) for x in beds ] ) )
        elif method == "equal-intervals":
            data = numpy.array( sorted( [(x.mFields[1], 1) for x in beds ] ) )
        elif method == "equal-range":
            vals = [x.mFields[1] for x in beds ]
            mi, ma = min( vals ), max( vals )
            increment = float(ma - mi) / num_bins
            bin_edges = numpy.arange( mi, ma, increment)
        else:
            raise ValueError("unknown method %s to compute bins, supply bin_edges" % method )

    if bin_edges == None:
        sums = data[:,1].cumsum( axis=0 )
        total = float(sums[-1])
        increment = float( total / num_bins )
        bin_edges = [data[0][0]]
        occupancies = []
        threshold = increment
        occ = 0
        for v, s in zip( data, sums ):
            occ += v[1]
            if s > threshold:
                bin_edges.append( v[0] )
                threshold += increment
                occupancies.append( occ )
                occ = 0

        bin_edges.append( data[-1][0] + 1 )

    beds.sort( key=lambda x: (x.contig,x.start) )
    last_contig, start, end, last_name = None, None, None, None
    new_beds = []
    for bed in beds:
        name = bisect.bisect_right(bin_edges, bed.mFields[1])-1
        contig = bed.contig
        if name != last_name or last_contig != contig:
            if last_name != None:
                b = Bed()
                b.contig, b.start, b.end, b.mFields = last_contig, start, end, [last_name]
                new_beds.append(b)
            start = bed.start
            last_name = name
            last_contig = contig

        end = bed.end

    if last_name != None:
        b = Bed()
        b.contig, b.start, b.end, b.mFields = contig, start, end, [last_name]
        new_beds.append(b)

    return new_beds, bin_edges

def merge( iterator ):
    '''merge overlapping intervals.

    returns a list of merged intervals.
    '''

    beds = list(iterator)
    if len(beds) == 0: return []

    beds.sort( key = lambda x: (x.contig, x.start) )

    def iterate_chunks( beds ):

        last = beds[0]
        to_join = [last]
        end = last.end
        contig = last.contig

        for this in beds[1:]:
            d = this.start - end
            if this.contig != contig or d >= 0:
                yield to_join
                contig = this.contig
                to_join = []
                end = this.end

            end = max(end, this.end)
            to_join.append( this )

        yield to_join
        raise StopIteration

    n = []
    for to_join in iterate_chunks(beds):

        y = Bed()
        y.contig = to_join[0].contig
        y.start = to_join[0].start
        y.end = to_join[-1].end
        n.append( y )

    return n
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2012 Lucien Loiseau
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
"backend for http://translate.google.com"

from weboob.capabilities.translate import CapTranslate, Translation, TranslationFail, LanguageNotSupported
from weboob.tools.backend import Module

from .browser import GoogleTranslateBrowser

__all__ = ['GoogleTranslateModule']

class GoogleTranslateModule(Module, CapTranslate):
    MAINTAINER = u'Lucien Loiseau'
    EMAIL = 'loiseau.lucien@gmail.com'
    VERSION = '1.1'
    LICENSE = 'AGPLv3+'
    NAME = 'googletranslate'
    DESCRIPTION = u'Google translation web service'
    BROWSER = GoogleTranslateBrowser
    GOOGLELANGUAGE = {
        'Arabic':'ar', 'Afrikaans':'af', 'Albanian':'sq', 'Armenian':'hy', 'Azerbaijani':'az', 'Basque':'eu', 'Belarusian':'be',
        'Bengali':'bn', 'Bulgarian':'bg', 'Catalan':'ca', 'Chinese':'zh-CN', 'Croatian':'hr', 'Czech':'cs', 'Danish':'da',
        'Dutch':'nl', 'English':'en', 'Esperanto':'eo', 'Estonian':'et', 'Filipino':'tl', 'Finnish':'fi', 'French':'fr',
        'Galician':'gl', 'Georgian':'ka', 'German':'de', 'Greek':'gr', 'Gujarati':'gu', 'Haitian':'ht', 'Hebrew':'iw',
        'Hindi':'hi', 'Hungaric':'hu', 'Icelandic':'is', 'Indonesian':'id', 'Irish':'ga', 'Italian':'it', 'Japanese':'ja',
        'Kannada':'kn', 'Korean':'ko', 'Latin':'la', 'Latvian':'lv', 'Lithuanian':'lt', 'Macedonian':'mk', 'Malay':'ms',
        'Maltese':'mt', 'Norwegian':'no', 'Persian':'fa', 'Polish':'pl', 'Portuguese':'pt', 'Romanian':'ro', 'Russian':'ru',
        'Serbian':'sr', 'Slovak':'sk', 'Slovenian':'sl', 'Spanish':'es', 'Swahili':'sw', 'Swedish':'sv', 'Tamil':'ta',
        'Telugu':'te', 'Thai':'th', 'Turkish':'tr', 'Ukrainian':'uk', 'Urdu':'ur', 'Vietnamese':'vi', 'Welsh':'cy', 'Yiddish':'yi',
        }

    def translate(self, lan_from, lan_to, text):
        if lan_from not in self.GOOGLELANGUAGE.keys():
            raise LanguageNotSupported()

        if lan_to not in self.GOOGLELANGUAGE.keys():
            raise LanguageNotSupported()

        translation = Translation(0)
        translation.lang_src = unicode(self.GOOGLELANGUAGE[lan_from])
        translation.lang_dst = unicode(self.GOOGLELANGUAGE[lan_to])
        translation.text = self.browser.translate(self.GOOGLELANGUAGE[lan_from], self.GOOGLELANGUAGE[lan_to], text)

        if translation.text is None:
            raise TranslationFail()

        return translation
<commit_msg>fix code used by Google Translate for Greek<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2012 Lucien Loiseau
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
"backend for http://translate.google.com"

from weboob.capabilities.translate import CapTranslate, Translation, TranslationFail, LanguageNotSupported
from weboob.tools.backend import Module

from .browser import GoogleTranslateBrowser

__all__ = ['GoogleTranslateModule']

class GoogleTranslateModule(Module, CapTranslate):
    MAINTAINER = u'Lucien Loiseau'
    EMAIL = 'loiseau.lucien@gmail.com'
    VERSION = '1.1'
    LICENSE = 'AGPLv3+'
    NAME = 'googletranslate'
    DESCRIPTION = u'Google translation web service'
    BROWSER = GoogleTranslateBrowser
    GOOGLELANGUAGE = {
        'Arabic':'ar', 'Afrikaans':'af', 'Albanian':'sq', 'Armenian':'hy', 'Azerbaijani':'az', 'Basque':'eu', 'Belarusian':'be',
        'Bengali':'bn', 'Bulgarian':'bg', 'Catalan':'ca', 'Chinese':'zh-CN', 'Croatian':'hr', 'Czech':'cs', 'Danish':'da',
        'Dutch':'nl', 'English':'en', 'Esperanto':'eo', 'Estonian':'et', 'Filipino':'tl', 'Finnish':'fi', 'French':'fr',
        'Galician':'gl', 'Georgian':'ka', 'German':'de', 'Greek':'el', 'Gujarati':'gu', 'Haitian':'ht', 'Hebrew':'iw',
        'Hindi':'hi', 'Hungaric':'hu', 'Icelandic':'is', 'Indonesian':'id', 'Irish':'ga', 'Italian':'it', 'Japanese':'ja',
        'Kannada':'kn', 'Korean':'ko', 'Latin':'la', 'Latvian':'lv', 'Lithuanian':'lt', 'Macedonian':'mk', 'Malay':'ms',
        'Maltese':'mt', 'Norwegian':'no', 'Persian':'fa', 'Polish':'pl', 'Portuguese':'pt', 'Romanian':'ro', 'Russian':'ru',
        'Serbian':'sr', 'Slovak':'sk', 'Slovenian':'sl', 'Spanish':'es', 'Swahili':'sw', 'Swedish':'sv', 'Tamil':'ta',
        'Telugu':'te', 'Thai':'th', 'Turkish':'tr', 'Ukrainian':'uk', 'Urdu':'ur', 'Vietnamese':'vi', 'Welsh':'cy', 'Yiddish':'yi',
        }

    def translate(self, lan_from, lan_to, text):
        if lan_from not in self.GOOGLELANGUAGE.keys():
            raise LanguageNotSupported()

        if lan_to not in self.GOOGLELANGUAGE.keys():
            raise LanguageNotSupported()

        translation = Translation(0)
        translation.lang_src = unicode(self.GOOGLELANGUAGE[lan_from])
        translation.lang_dst = unicode(self.GOOGLELANGUAGE[lan_to])
        translation.text = self.browser.translate(self.GOOGLELANGUAGE[lan_from], self.GOOGLELANGUAGE[lan_to], text)

        if translation.text is None:
            raise TranslationFail()

        return translation
<|endoftext|>


<commit_before>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2014, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import numpy
import logging
import operator
import collections
from functools import partial

from openquake.hazardlib.site import SiteCollection
from openquake.hazardlib.calc.hazard_curve import (
    hazard_curves_per_trt, zero_curves, zero_maps, agg_curves)
from openquake.hazardlib.calc.filters import source_site_distance_filter, \
    rupture_site_distance_filter
from openquake.risklib import scientific
from openquake.commonlib import parallel, source, datastore
from openquake.baselib.general import AccumDict, split_in_blocks

from openquake.commonlib.calculators import base, calc

HazardCurve = collections.namedtuple('HazardCurve', 'location poes')

@parallel.litetask
def classical(sources, sitecol, gsims_assoc, monitor):
    """
    :param sources:
        a non-empty sequence of sources of homogeneous tectonic region type
    :param sitecol:
        a SiteCollection instance
    :param gsims_assoc:
        associations trt_model_id -> gsims
    :param monitor:
        a monitor instance
    :returns:
        an AccumDict rlz -> curves
    """
    max_dist = monitor.oqparam.maximum_distance
    truncation_level = monitor.oqparam.truncation_level
    imtls = monitor.oqparam.imtls
    trt_model_id = sources[0].trt_model_id
    gsims = gsims_assoc[trt_model_id]
    curves_by_gsim = hazard_curves_per_trt(
        sources, sitecol, imtls, gsims, truncation_level,
        source_site_filter=source_site_distance_filter(max_dist),
        rupture_site_filter=rupture_site_distance_filter(max_dist),
        monitor=monitor)
    dic = dict(calc_times=monitor.calc_times)
    for gsim, curves in zip(gsims, curves_by_gsim):
        dic[trt_model_id, str(gsim)] = curves
    return dic

def agg_dicts(acc, val):
    """
    Aggregate dictionaries of hazard curves by updating the accumulator
    """
    for key in val:
        if key == 'calc_times':
            acc[key].extend(val[key])
        else:  # aggregate curves
            acc[key] = agg_curves(acc[key], val[key])
    return acc

source_info_dt = numpy.dtype(
    [('trt_model_id', numpy.uint32),
     ('source_id', (bytes, 20)),
     ('calc_time', numpy.float32)])

@base.calculators.add('classical')
class ClassicalCalculator(base.HazardCalculator):
    """
    Classical PSHA calculator
    """
    core_func = classical
    source_info = datastore.persistent_attribute('source_info')

    def execute(self):
        """
        Run in parallel `core_func(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor(self.core_func.__name__)
        monitor.oqparam = self.oqparam
        sources = self.csm.get_sources()
        zc = zero_curves(len(self.sitecol.complete), self.oqparam.imtls)
        zerodict = AccumDict((key, zc) for key in self.rlzs_assoc)
        zerodict['calc_times'] = []
        gsims_assoc = self.rlzs_assoc.gsims_by_trt_id
        curves_by_trt_gsim = parallel.apply_reduce(
            self.core_func.__func__,
            (sources, self.sitecol, gsims_assoc, monitor),
            agg=agg_dicts, acc=zerodict,
            concurrent_tasks=self.oqparam.concurrent_tasks,
            weight=operator.attrgetter('weight'),
            key=operator.attrgetter('trt_model_id'))
        return curves_by_trt_gsim

    def post_execute(self, curves_by_trt_gsim):
        """
        Collect the hazard curves by realization and export them.

        :param curves_by_trt_gsim:
            a dictionary (trt_id, gsim) -> hazard curves
        """
        # save calculation time per source
        try:
            calc_times = curves_by_trt_gsim.pop('calc_times')
        except KeyError:
            pass
        else:
            sources = self.csm.get_sources()
            info = []
            for i, dt in calc_times:
                src = sources[i]
                info.append((src.trt_model_id, src.source_id, dt))
            info.sort(key=operator.itemgetter(2), reverse=True)
            self.source_info = numpy.array(info, source_info_dt)

        # save curves_by_trt_gsim
        for sm in self.rlzs_assoc.csm_info.source_models:
            group = self.datastore.hdf5.create_group(
                'curves_by_sm/' + '_'.join(sm.path))
            group.attrs['source_model'] = sm.name
            for tm in sm.trt_models:
                for gsim in tm.gsims:
                    try:
                        curves = curves_by_trt_gsim[tm.id, gsim]
                    except KeyError:  # no data for the trt_model
                        pass
                    else:
                        ts = '%03d-%s' % (tm.id, gsim)
                        group[ts] = curves
                        group[ts].attrs['trt'] = tm.trt
        oq = self.oqparam
        zc = zero_curves(len(self.sitecol.complete), oq.imtls)
        curves_by_rlz = self.rlzs_assoc.combine_curves(
            curves_by_trt_gsim, agg_curves, zc)
        rlzs = self.rlzs_assoc.realizations
        nsites = len(self.sitecol)
        if oq.individual_curves:
            for rlz, curves in curves_by_rlz.iteritems():
                self.store_curves('rlz-%03d' % rlz.ordinal, curves, rlz)

        if len(rlzs) == 1:  # cannot compute statistics
            [self.mean_curves] = curves_by_rlz.values()
            return

        weights = (None if oq.number_of_logic_tree_samples
                   else [rlz.weight for rlz in rlzs])
        mean = oq.mean_hazard_curves
        if mean:
            self.mean_curves = numpy.array(zc)
            for imt in oq.imtls:
                self.mean_curves[imt] = scientific.mean_curve(
                    [curves_by_rlz[rlz][imt] for rlz in rlzs], weights)

        self.quantile = {}
        for q in oq.quantile_hazard_curves:
            self.quantile[q] = qc = numpy.array(zc)
            for imt in oq.imtls:
                curves = [curves_by_rlz[rlz][imt] for rlz in rlzs]
                qc[imt] = scientific.quantile_curve(
                    curves, q, weights).reshape((nsites, -1))

        if mean:
            self.store_curves('mean', self.mean_curves)
        for q in self.quantile:
            self.store_curves('quantile-%s' % q, self.quantile[q])

    def hazard_maps(self, curves):
        """
        Compute the hazard maps associated to the curves
        """
        n, p = len(self.sitecol), len(self.oqparam.poes)
        maps = zero_maps((n, p), self.oqparam.imtls)
        for imt in curves.dtype.fields:
            maps[imt] = calc.compute_hazard_maps(
                curves[imt], self.oqparam.imtls[imt], self.oqparam.poes)
        return maps

    def store_curves(self, kind, curves, rlz=None):
        """
        Store all kind of curves, optionally computing maps and uhs curves.

        :param kind: the kind of curves to store
        :param curves: an array of N curves to store
        :param rlz: hazard realization, if any
        """
        if not self.persistent:  # do nothing
            return
        oq = self.oqparam
        self._store('hcurves/' + kind, curves, rlz)
        if oq.hazard_maps or oq.uniform_hazard_spectra:
            # hmaps is a composite array of shape (N, P)
            hmaps = self.hazard_maps(curves)
            if oq.hazard_maps:
                self._store('hmaps/' + kind, hmaps, rlz, poes=oq.poes)
            if oq.uniform_hazard_spectra:
                # uhs is an array of shape (N, I, P)
                self._store('uhs/' + kind, calc.make_uhs(hmaps), rlz,
                            poes=oq.poes)

    def _store(self, name, curves, rlz, **kw):
        self.datastore.hdf5[name] = curves
        dset = self.datastore.hdf5[name]
        if rlz is not None:
            dset.attrs['uid'] = rlz.uid
        for k, v in kw.items():
            dset.attrs[k] = v

def is_effective_trt_model(result_dict, trt_model):
    """
    Returns True on tectonic region types
    which ID in contained in the result_dict.

    :param result_dict: a dictionary with keys (trt_id, gsim)
    """
    return any(trt_model.id == trt_id for trt_id, _gsim in result_dict)

@parallel.litetask
def classical_tiling(calculator, sitecol, position, tileno, monitor):
    """
    :param calculator:
        a ClassicalCalculator instance
    :param sitecol:
        the site collection of the current tile
    :param position:
        position of the current tile in the full site collection
    :param tileno:
        the tile ordinal
    :param monitor:
        a monitor instance
    :returns:
        a dictionary file name -> full path for each exported file
    """
    calculator.sitecol = sitecol
    calculator.tileno = '.%04d' % tileno
    curves_by_trt_gsim = calculator.execute()
    curves_by_trt_gsim.indices = range(position, position + len(sitecol))
    # build the correct realizations from the (reduced) logic tree
    calculator.rlzs_assoc = calculator.csm.get_rlzs_assoc(
        partial(is_effective_trt_model, curves_by_trt_gsim))
    n_levels = sum(len(imls) for imls in calculator.oqparam.imtls.itervalues())
    tup = (len(calculator.sitecol), n_levels, len(calculator.rlzs_assoc),
           len(calculator.rlzs_assoc.realizations))
    logging.info('Processed tile %d, (sites, levels, keys, rlzs)=%s',
                 tileno, tup)
    return curves_by_trt_gsim

def agg_curves_by_trt_gsim(acc, curves_by_trt_gsim):
    """
    :param acc: AccumDict (trt_id, gsim) -> N curves
    :param curves_by_trt_gsim: AccumDict (trt_id, gsim) -> T curves

    where N is the total number of sites and T the number of sites
    in the current tile. Works by side effect, by updating the accumulator.
    """
    for k in curves_by_trt_gsim:
        if k == 'calc_times':
            acc[k].extend(curves_by_trt_gsim[k])
        else:
            acc[k][curves_by_trt_gsim.indices] = curves_by_trt_gsim[k]
    return acc

@base.calculators.add('classical_tiling')
class ClassicalTilingCalculator(ClassicalCalculator):
    """
    Classical Tiling calculator
    """
    SourceProcessor = source.SourceFilter

    def execute(self):
        """
        Split the computation by tiles which are run in parallel.
        """
        monitor = self.monitor(self.core_func.__name__)
        monitor.oqparam = oq = self.oqparam
        self.tiles = split_in_blocks(
            self.sitecol, self.oqparam.concurrent_tasks or 1)
        oq.concurrent_tasks = 0
        calculator = ClassicalCalculator(
            self.oqparam, monitor, persistent=False)
        calculator.csm = self.csm
        rlzs_assoc = self.csm.get_rlzs_assoc()
        self.rlzs_assoc = calculator.rlzs_assoc = rlzs_assoc

        # parallelization
        all_args = []
        position = 0
        for (i, tile) in enumerate(self.tiles):
            all_args.append((calculator, SiteCollection(tile),
                             position, i, monitor))
            position += len(tile)
        acc = {trt_gsim: zero_curves(len(self.sitecol), oq.imtls)
               for trt_gsim in calculator.rlzs_assoc}
        acc['calc_times'] = []
        return parallel.starmap(classical_tiling, all_args).reduce(
            agg_curves_by_trt_gsim, acc)
<commit_msg>Small improvement<commit_after>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2014, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import numpy
import logging
import operator
import collections
from functools import partial

from openquake.hazardlib.site import SiteCollection
from openquake.hazardlib.calc.hazard_curve import (
    hazard_curves_per_trt, zero_curves, zero_maps, agg_curves)
from openquake.hazardlib.calc.filters import source_site_distance_filter, \
    rupture_site_distance_filter
from openquake.risklib import scientific
from openquake.commonlib import parallel, source, datastore
from openquake.baselib.general import AccumDict, split_in_blocks

from openquake.commonlib.calculators import base, calc

HazardCurve = collections.namedtuple('HazardCurve', 'location poes')

@parallel.litetask
def classical(sources, sitecol, gsims_assoc, monitor):
    """
    :param sources:
        a non-empty sequence of sources of homogeneous tectonic region type
    :param sitecol:
        a SiteCollection instance
    :param gsims_assoc:
        associations trt_model_id -> gsims
    :param monitor:
        a monitor instance
    :returns:
        an AccumDict rlz -> curves
    """
    max_dist = monitor.oqparam.maximum_distance
    truncation_level = monitor.oqparam.truncation_level
    imtls = monitor.oqparam.imtls
    trt_model_id = sources[0].trt_model_id
    gsims = gsims_assoc[trt_model_id]
    curves_by_gsim = hazard_curves_per_trt(
        sources, sitecol, imtls, gsims, truncation_level,
        source_site_filter=source_site_distance_filter(max_dist),
        rupture_site_filter=rupture_site_distance_filter(max_dist),
        monitor=monitor)
    dic = dict(monitor=monitor)
    for gsim, curves in zip(gsims, curves_by_gsim):
        dic[trt_model_id, str(gsim)] = curves
    return dic

def agg_dicts(acc, val):
    """
    Aggregate dictionaries of hazard curves by updating the accumulator
    """
    for key in val:
        if key == 'monitor':
            acc['calc_times'].extend(val[key].calc_times)
        else:  # aggregate curves
            acc[key] = agg_curves(acc[key], val[key])
    return acc

source_info_dt = numpy.dtype(
    [('trt_model_id', numpy.uint32),
     ('source_id', (bytes, 20)),
     ('calc_time', numpy.float32)])

@base.calculators.add('classical')
class ClassicalCalculator(base.HazardCalculator):
    """
    Classical PSHA calculator
    """
    core_func = classical
    source_info = datastore.persistent_attribute('source_info')

    def execute(self):
        """
        Run in parallel `core_func(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor(self.core_func.__name__)
        monitor.oqparam = self.oqparam
        sources = self.csm.get_sources()
        zc = zero_curves(len(self.sitecol.complete), self.oqparam.imtls)
        zerodict = AccumDict((key, zc) for key in self.rlzs_assoc)
        zerodict['calc_times'] = []
        gsims_assoc = self.rlzs_assoc.gsims_by_trt_id
        curves_by_trt_gsim = parallel.apply_reduce(
            self.core_func.__func__,
            (sources, self.sitecol, gsims_assoc, monitor),
            agg=agg_dicts, acc=zerodict,
            concurrent_tasks=self.oqparam.concurrent_tasks,
            weight=operator.attrgetter('weight'),
            key=operator.attrgetter('trt_model_id'))
        return curves_by_trt_gsim

    def post_execute(self, curves_by_trt_gsim):
        """
        Collect the hazard curves by realization and export them.

        :param curves_by_trt_gsim:
            a dictionary (trt_id, gsim) -> hazard curves
        """
        # save calculation time per source
        try:
            calc_times = curves_by_trt_gsim.pop('calc_times')
        except KeyError:
            pass
        else:
            sources = self.csm.get_sources()
            info = []
            for i, dt in calc_times:
                src = sources[i]
                info.append((src.trt_model_id, src.source_id, dt))
            info.sort(key=operator.itemgetter(2), reverse=True)
            self.source_info = numpy.array(info, source_info_dt)

        # save curves_by_trt_gsim
        for sm in self.rlzs_assoc.csm_info.source_models:
            group = self.datastore.hdf5.create_group(
                'curves_by_sm/' + '_'.join(sm.path))
            group.attrs['source_model'] = sm.name
            for tm in sm.trt_models:
                for gsim in tm.gsims:
                    try:
                        curves = curves_by_trt_gsim[tm.id, gsim]
                    except KeyError:  # no data for the trt_model
                        pass
                    else:
                        ts = '%03d-%s' % (tm.id, gsim)
                        group[ts] = curves
                        group[ts].attrs['trt'] = tm.trt
        oq = self.oqparam
        zc = zero_curves(len(self.sitecol.complete), oq.imtls)
        curves_by_rlz = self.rlzs_assoc.combine_curves(
            curves_by_trt_gsim, agg_curves, zc)
        rlzs = self.rlzs_assoc.realizations
        nsites = len(self.sitecol)
        if oq.individual_curves:
            for rlz, curves in curves_by_rlz.iteritems():
                self.store_curves('rlz-%03d' % rlz.ordinal, curves, rlz)

        if len(rlzs) == 1:  # cannot compute statistics
            [self.mean_curves] = curves_by_rlz.values()
            return

        weights = (None if oq.number_of_logic_tree_samples
                   else [rlz.weight for rlz in rlzs])
        mean = oq.mean_hazard_curves
        if mean:
            self.mean_curves = numpy.array(zc)
            for imt in oq.imtls:
                self.mean_curves[imt] = scientific.mean_curve(
                    [curves_by_rlz[rlz][imt] for rlz in rlzs], weights)

        self.quantile = {}
        for q in oq.quantile_hazard_curves:
            self.quantile[q] = qc = numpy.array(zc)
            for imt in oq.imtls:
                curves = [curves_by_rlz[rlz][imt] for rlz in rlzs]
                qc[imt] = scientific.quantile_curve(
                    curves, q, weights).reshape((nsites, -1))

        if mean:
            self.store_curves('mean', self.mean_curves)
        for q in self.quantile:
            self.store_curves('quantile-%s' % q, self.quantile[q])

    def hazard_maps(self, curves):
        """
        Compute the hazard maps associated to the curves
        """
        n, p = len(self.sitecol), len(self.oqparam.poes)
        maps = zero_maps((n, p), self.oqparam.imtls)
        for imt in curves.dtype.fields:
            maps[imt] = calc.compute_hazard_maps(
                curves[imt], self.oqparam.imtls[imt], self.oqparam.poes)
        return maps

    def store_curves(self, kind, curves, rlz=None):
        """
        Store all kind of curves, optionally computing maps and uhs curves.

        :param kind: the kind of curves to store
        :param curves: an array of N curves to store
        :param rlz: hazard realization, if any
        """
        if not self.persistent:  # do nothing
            return
        oq = self.oqparam
        self._store('hcurves/' + kind, curves, rlz)
        if oq.hazard_maps or oq.uniform_hazard_spectra:
            # hmaps is a composite array of shape (N, P)
            hmaps = self.hazard_maps(curves)
            if oq.hazard_maps:
                self._store('hmaps/' + kind, hmaps, rlz, poes=oq.poes)
            if oq.uniform_hazard_spectra:
                # uhs is an array of shape (N, I, P)
                self._store('uhs/' + kind, calc.make_uhs(hmaps), rlz,
                            poes=oq.poes)

    def _store(self, name, curves, rlz, **kw):
        self.datastore.hdf5[name] = curves
        dset = self.datastore.hdf5[name]
        if rlz is not None:
            dset.attrs['uid'] = rlz.uid
        for k, v in kw.items():
            dset.attrs[k] = v

def is_effective_trt_model(result_dict, trt_model):
    """
    Returns True on tectonic region types
    which ID in contained in the result_dict.

    :param result_dict: a dictionary with keys (trt_id, gsim)
    """
    return any(trt_model.id == trt_id for trt_id, _gsim in result_dict)

@parallel.litetask
def classical_tiling(calculator, sitecol, position, tileno, monitor):
    """
    :param calculator:
        a ClassicalCalculator instance
    :param sitecol:
        the site collection of the current tile
    :param position:
        position of the current tile in the full site collection
    :param tileno:
        the tile ordinal
    :param monitor:
        a monitor instance
    :returns:
        a dictionary file name -> full path for each exported file
    """
    calculator.sitecol = sitecol
    calculator.tileno = '.%04d' % tileno
    curves_by_trt_gsim = calculator.execute()
    curves_by_trt_gsim.indices = range(position, position + len(sitecol))
    # build the correct realizations from the (reduced) logic tree
    calculator.rlzs_assoc = calculator.csm.get_rlzs_assoc(
        partial(is_effective_trt_model, curves_by_trt_gsim))
    n_levels = sum(len(imls) for imls in calculator.oqparam.imtls.itervalues())
    tup = (len(calculator.sitecol), n_levels, len(calculator.rlzs_assoc),
           len(calculator.rlzs_assoc.realizations))
    logging.info('Processed tile %d, (sites, levels, keys, rlzs)=%s',
                 tileno, tup)
    return curves_by_trt_gsim

def agg_curves_by_trt_gsim(acc, curves_by_trt_gsim):
    """
    :param acc: AccumDict (trt_id, gsim) -> N curves
    :param curves_by_trt_gsim: AccumDict (trt_id, gsim) -> T curves

    where N is the total number of sites and T the number of sites
    in the current tile. Works by side effect, by updating the accumulator.
    """
    for k in curves_by_trt_gsim:
        if k == 'calc_times':
            acc['calc_times'].extend(curves_by_trt_gsim[k])
        else:
            acc[k][curves_by_trt_gsim.indices] = curves_by_trt_gsim[k]
    return acc

@base.calculators.add('classical_tiling')
class ClassicalTilingCalculator(ClassicalCalculator):
    """
    Classical Tiling calculator
    """
    SourceProcessor = source.SourceFilter

    def execute(self):
        """
        Split the computation by tiles which are run in parallel.
        """
        monitor = self.monitor(self.core_func.__name__)
        monitor.oqparam = oq = self.oqparam
        self.tiles = split_in_blocks(
            self.sitecol, self.oqparam.concurrent_tasks or 1)
        oq.concurrent_tasks = 0
        calculator = ClassicalCalculator(
            self.oqparam, monitor, persistent=False)
        calculator.csm = self.csm
        rlzs_assoc = self.csm.get_rlzs_assoc()
        self.rlzs_assoc = calculator.rlzs_assoc = rlzs_assoc

        # parallelization
        all_args = []
        position = 0
        for (i, tile) in enumerate(self.tiles):
            all_args.append((calculator, SiteCollection(tile),
                             position, i, monitor))
            position += len(tile)
        acc = {trt_gsim: zero_curves(len(self.sitecol), oq.imtls)
               for trt_gsim in calculator.rlzs_assoc}
        acc['calc_times'] = []
        return parallel.starmap(classical_tiling, all_args).reduce(
            agg_curves_by_trt_gsim, acc)
<|endoftext|>


<commit_before>from __future__ import annotations

from typing import TYPE_CHECKING

from dask.base import tokenize
from dask.highlevelgraph import HighLevelGraph
from dask.layers import SimpleShuffleLayer

from distributed.shuffle.shuffle_extension import ShuffleId, ShuffleWorkerExtension

if TYPE_CHECKING:
    import pandas as pd

    from dask.dataframe import DataFrame

def get_ext() -> ShuffleWorkerExtension:
    from distributed import get_worker

    try:
        worker = get_worker()
    except ValueError as e:
        raise RuntimeError(
            "`shuffle='p2p'` requires Dask's distributed scheduler. This task is not running on a Worker; "
            "please confirm that you've created a distributed Client and are submitting this computation through it."
        ) from e
    extension: ShuffleWorkerExtension | None = worker.extensions.get("shuffle")
    if not extension:
        raise RuntimeError(
            f"The worker {worker.address} does not have a ShuffleExtension. "
            "Is pandas installed on the worker?"
        )
    return extension

def shuffle_transfer(
    input: pd.DataFrame,
    id: ShuffleId,
    npartitions: int,
    column: str,
) -> None:
    get_ext().add_partition(input, id, npartitions=npartitions, column=column)

def shuffle_unpack(
    id: ShuffleId, output_partition: int, barrier: object
) -> pd.DataFrame:
    return get_ext().get_output_partition(id, output_partition)

def shuffle_barrier(id: ShuffleId, transfers: list[None]) -> None:
    get_ext().barrier(id)

def rearrange_by_column_p2p(
    df: DataFrame,
    column: str,
    npartitions: int | None = None,
) -> DataFrame:
    from dask.dataframe import DataFrame

    npartitions = npartitions or df.npartitions
    token = tokenize(df, column, npartitions)

    empty = df._meta.copy()
    for c, dt in empty.dtypes.items():
        if dt == object:
            empty[c] = empty[c].astype(
                "string"
            )  # TODO: we fail at non-string object dtypes
    empty[column] = empty[column].astype("int64")  # TODO: this shouldn't be necesssary

    name = f"shuffle-p2p-{token}"
    layer = P2PShuffleLayer(
        name,
        column,
        npartitions,
        npartitions_input=df.npartitions,
        ignore_index=True,
        name_input=df._name,
        meta_input=empty,
    )
    return DataFrame(
        HighLevelGraph.from_collections(name, layer, [df]),
        name,
        empty,
        [None] * (npartitions + 1),
    )

class P2PShuffleLayer(SimpleShuffleLayer):
    def __init__(
        self,
        name,
        column,
        npartitions,
        npartitions_input,
        ignore_index,
        name_input,
        meta_input,
        parts_out=None,
        annotations=None,
    ):
        annotations = annotations or {}
        annotations.update({"shuffle": lambda key: key[1]})
        super().__init__(
            name,
            column,
            npartitions,
            npartitions_input,
            ignore_index,
            name_input,
            meta_input,
            parts_out,
            annotations=annotations,
        )

    def get_split_keys(self):
        # TODO: This is doing some funky stuff to set priorities but we don't need this
        return []

    def __repr__(self):
        return (
            f"{type(self).__name__}<name='{self.name}', npartitions={self.npartitions}>"
        )

    def _cull(self, parts_out):
        return P2PShuffleLayer(
            self.name,
            self.column,
            self.npartitions,
            self.npartitions_input,
            self.ignore_index,
            self.name_input,
            self.meta_input,
            parts_out=parts_out,
        )

    def _construct_graph(self, deserializing=None):
        token = tokenize(self.name_input, self.column, self.npartitions, self.parts_out)
        dsk = {}
        barrier_key = "shuffle-barrier-" + token
        name = "shuffle-transfer-" + token
        tranfer_keys = list()
        for i in range(self.npartitions_input):
            tranfer_keys.append((name, i))
            dsk[(name, i)] = (
                shuffle_transfer,
                (self.name_input, i),
                token,
                self.npartitions,
                self.column,
            )

        dsk[barrier_key] = (shuffle_barrier, token, tranfer_keys)

        name = self.name
        for part_out in self.parts_out:
            dsk[(name, part_out)] = (shuffle_unpack, token, part_out, barrier_key)
        return dsk
<commit_msg>Fix mypy failure on CI (#7198)<commit_after>from __future__ import annotations

from typing import TYPE_CHECKING, Any

from dask.base import tokenize
from dask.highlevelgraph import HighLevelGraph
from dask.layers import SimpleShuffleLayer

from distributed.shuffle.shuffle_extension import ShuffleId, ShuffleWorkerExtension

if TYPE_CHECKING:
    import pandas as pd

    from dask.dataframe import DataFrame

def get_ext() -> ShuffleWorkerExtension:
    from distributed import get_worker

    try:
        worker = get_worker()
    except ValueError as e:
        raise RuntimeError(
            "`shuffle='p2p'` requires Dask's distributed scheduler. This task is not running on a Worker; "
            "please confirm that you've created a distributed Client and are submitting this computation through it."
        ) from e
    extension: ShuffleWorkerExtension | None = worker.extensions.get("shuffle")
    if not extension:
        raise RuntimeError(
            f"The worker {worker.address} does not have a ShuffleExtension. "
            "Is pandas installed on the worker?"
        )
    return extension

def shuffle_transfer(
    input: pd.DataFrame,
    id: ShuffleId,
    npartitions: int,
    column: str,
) -> None:
    get_ext().add_partition(input, id, npartitions=npartitions, column=column)

def shuffle_unpack(
    id: ShuffleId, output_partition: int, barrier: object
) -> pd.DataFrame:
    return get_ext().get_output_partition(id, output_partition)

def shuffle_barrier(id: ShuffleId, transfers: list[None]) -> None:
    get_ext().barrier(id)

def rearrange_by_column_p2p(
    df: DataFrame,
    column: str,
    npartitions: int | None = None,
) -> DataFrame:
    from dask.dataframe import DataFrame

    npartitions = npartitions or df.npartitions
    token = tokenize(df, column, npartitions)

    empty = df._meta.copy()
    for c, dt in empty.dtypes.items():
        if dt == object:
            empty[c] = empty[c].astype(
                "string"
            )  # TODO: we fail at non-string object dtypes
    empty[column] = empty[column].astype("int64")  # TODO: this shouldn't be necesssary

    name = f"shuffle-p2p-{token}"
    layer = P2PShuffleLayer(
        name,
        column,
        npartitions,
        npartitions_input=df.npartitions,
        ignore_index=True,
        name_input=df._name,
        meta_input=empty,
    )
    return DataFrame(
        HighLevelGraph.from_collections(name, layer, [df]),
        name,
        empty,
        [None] * (npartitions + 1),
    )

class P2PShuffleLayer(SimpleShuffleLayer):
    def __init__(
        self,
        name: str,
        column: str,
        npartitions: int,
        npartitions_input: int,
        ignore_index: bool,
        name_input: str,
        meta_input: pd.DataFrame,
        parts_out: list | None = None,
        annotations: dict | None = None,
    ):
        annotations = annotations or {}
        annotations.update({"shuffle": lambda key: key[1]})
        super().__init__(
            name,
            column,
            npartitions,
            npartitions_input,
            ignore_index,
            name_input,
            meta_input,
            parts_out,
            annotations=annotations,
        )

    def get_split_keys(self) -> list:
        # TODO: This is doing some funky stuff to set priorities but we don't need this
        return []

    def __repr__(self) -> str:
        return (
            f"{type(self).__name__}<name='{self.name}', npartitions={self.npartitions}>"
        )

    def _cull(self, parts_out: list) -> P2PShuffleLayer:
        return P2PShuffleLayer(
            self.name,
            self.column,
            self.npartitions,
            self.npartitions_input,
            self.ignore_index,
            self.name_input,
            self.meta_input,
            parts_out=parts_out,
        )

    def _construct_graph(self, deserializing: Any = None) -> dict[tuple | str, tuple]:
        token = tokenize(self.name_input, self.column, self.npartitions, self.parts_out)
        dsk: dict[tuple | str, tuple] = {}
        barrier_key = "shuffle-barrier-" + token
        name = "shuffle-transfer-" + token
        tranfer_keys = list()
        for i in range(self.npartitions_input):
            tranfer_keys.append((name, i))
            dsk[(name, i)] = (
                shuffle_transfer,
                (self.name_input, i),
                token,
                self.npartitions,
                self.column,
            )

        dsk[barrier_key] = (shuffle_barrier, token, tranfer_keys)

        name = self.name
        for part_out in self.parts_out:
            dsk[(name, part_out)] = (shuffle_unpack, token, part_out, barrier_key)
        return dsk
<|endoftext|>


<commit_before># Default Django settings. Override these with settings in the module
# pointed-to by the DJANGO_SETTINGS_MODULE environment variable.

# This is defined here as a do-nothing function because we can't import
# django.utils.translation -- that module depends on the settings.
gettext_noop = lambda s: s

####################
# CORE             #
####################

DEBUG = False
TEMPLATE_DEBUG = False

# Whether to use the "Etag" header. This saves bandwidth but slows down performance.
USE_ETAGS = False

# People who get code error notifications.
# In the format (('Full Name', 'email@domain.com'), ('Full Name', 'anotheremail@domain.com'))
ADMINS = ()

# Tuple of IP addresses, as strings, that:
#   * See debug comments, when DEBUG is true
#   * Receive x-headers
INTERNAL_IPS = ()

# Local time zone for this installation. All choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name (although not all
# systems may support all possibilities).
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes
LANGUAGE_CODE = 'en-us'

# Languages we provide translations for, out of the box. The language name
# should be the utf-8 encoded local name for the language.
LANGUAGES = (
    ('ar', gettext_noop('Arabic')),
    ('bn', gettext_noop('Bengali')),
    ('bg', gettext_noop('Bulgarian')),
    ('ca', gettext_noop('Catalan')),
    ('cs', gettext_noop('Czech')),
    ('cy', gettext_noop('Welsh')),
    ('da', gettext_noop('Danish')),
    ('de', gettext_noop('German')),
    ('el', gettext_noop('Greek')),
    ('en', gettext_noop('English')),
    ('es', gettext_noop('Spanish')),
    ('es_AR', gettext_noop('Argentinean Spanish')),
    ('fa', gettext_noop('Persian')),
    ('fi', gettext_noop('Finnish')),
    ('fr', gettext_noop('French')),
    ('ga', gettext_noop('Gaeilge')),
    ('gl', gettext_noop('Galician')),
    ('hu', gettext_noop('Hungarian')),
    ('he', gettext_noop('Hebrew')),
    ('hr', gettext_noop('Croatian')),
    ('is', gettext_noop('Icelandic')),
    ('it', gettext_noop('Italian')),
    ('ja', gettext_noop('Japanese')),
    ('ko', gettext_noop('Korean')),
    ('km', gettext_noop('Khmer')),
    ('kn', gettext_noop('Kannada')),
    ('lv', gettext_noop('Latvian')),
    ('mk', gettext_noop('Macedonian')),
    ('nl', gettext_noop('Dutch')),
    ('no', gettext_noop('Norwegian')),
    ('pl', gettext_noop('Polish')),
    ('pt', gettext_noop('Portugese')),
    ('pt-br', gettext_noop('Brazilian')),
    ('ro', gettext_noop('Romanian')),
    ('ru', gettext_noop('Russian')),
    ('sk', gettext_noop('Slovak')),
    ('sl', gettext_noop('Slovenian')),
    ('sr', gettext_noop('Serbian')),
    ('sv', gettext_noop('Swedish')),
    ('ta', gettext_noop('Tamil')),
    ('te', gettext_noop('Telugu')),
    ('tr', gettext_noop('Turkish')),
    ('uk', gettext_noop('Ukrainian')),
    ('zh-cn', gettext_noop('Simplified Chinese')),
    ('zh-tw', gettext_noop('Traditional Chinese')),
)

# Languages using BiDi (right-to-left) layout
LANGUAGES_BIDI = ("he", "ar", "fa")

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# Not-necessarily-technical managers of the site. They get broken link
# notifications and other various e-mails.
MANAGERS = ADMINS

# Default content type and charset to use for all HttpResponse objects, if a
# MIME type isn't manually specified. These are used to construct the
# Content-Type header.
DEFAULT_CONTENT_TYPE = 'text/html'
DEFAULT_CHARSET = 'utf-8'

# Encoding of files read from disk (template and initial SQL files).
FILE_CHARSET = 'utf-8'

# E-mail address that error messages come from.
SERVER_EMAIL = 'root@localhost'

# Whether to send broken-link e-mails.
SEND_BROKEN_LINK_EMAILS = False

# Database connection info.
DATABASE_ENGINE = ''           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'.
DATABASE_NAME = ''             # Or path to database file if using sqlite3.
DATABASE_USER = ''             # Not used with sqlite3.
DATABASE_PASSWORD = ''         # Not used with sqlite3.
DATABASE_HOST = ''             # Set to empty string for localhost. Not used with sqlite3.
DATABASE_PORT = ''             # Set to empty string for default. Not used with sqlite3.
DATABASE_OPTIONS = {}          # Set to empty dictionary for default.

# Host for sending e-mail.
EMAIL_HOST = 'localhost'

# Port for sending e-mail.
EMAIL_PORT = 25

# Optional SMTP authentication information for EMAIL_HOST.
EMAIL_HOST_USER = ''
EMAIL_HOST_PASSWORD = ''
EMAIL_USE_TLS = False

# List of strings representing installed apps.
INSTALLED_APPS = ()

# List of locations of the template source files, in search order.
TEMPLATE_DIRS = ()

# List of callables that know how to import templates from various sources.
# See the comments in django/core/template/loader.py for interface
# documentation.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
#     'django.template.loaders.eggs.load_template_source',
)

# List of processors used by RequestContext to populate the context.
# Each one should be a callable that takes the request object as its
# only parameter and returns a dictionary to add to the context.
TEMPLATE_CONTEXT_PROCESSORS = (
    'django.core.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
#    'django.core.context_processors.request',
)

# Output to use in template system for invalid (e.g. misspelled) variables.
TEMPLATE_STRING_IF_INVALID = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'

# Default e-mail address to use for various automated correspondence from
# the site managers.
DEFAULT_FROM_EMAIL = 'webmaster@localhost'

# Subject-line prefix for email messages send with django.core.mail.mail_admins
# or ...mail_managers.  Make sure to include the trailing space.
EMAIL_SUBJECT_PREFIX = '[Django] '

# Whether to append trailing slashes to URLs.
APPEND_SLASH = True

# Whether to prepend the "www." subdomain to URLs that don't have it.
PREPEND_WWW = False

# List of compiled regular expression objects representing User-Agent strings
# that are not allowed to visit any page, systemwide. Use this for bad
# robots/crawlers. Here are a few examples:
#     import re
#     DISALLOWED_USER_AGENTS = (
#         re.compile(r'^NaverBot.*'),
#         re.compile(r'^EmailSiphon.*'),
#         re.compile(r'^SiteSucker.*'),
#         re.compile(r'^sohu-search')
#     )
DISALLOWED_USER_AGENTS = ()

ABSOLUTE_URL_OVERRIDES = {}

# Tuple of strings representing allowed prefixes for the {% ssi %} tag.
# Example: ('/home/html', '/var/www')
ALLOWED_INCLUDE_ROOTS = ()

# If this is a admin settings module, this should be a list of
# settings modules (in the format 'foo.bar.baz') for which this admin
# is an admin.
ADMIN_FOR = ()

# 404s that may be ignored.
IGNORABLE_404_STARTS = ('/cgi-bin/', '/_vti_bin', '/_vti_inf')
IGNORABLE_404_ENDS = ('mail.pl', 'mailform.pl', 'mail.cgi', 'mailform.cgi', 'favicon.ico', '.php')

# A secret key for this particular Django installation. Used in secret-key
# hashing algorithms. Set this in your settings, or Django will complain
# loudly.
SECRET_KEY = ''

# Path to the "jing" executable -- needed to validate XMLFields
JING_PATH = "/usr/bin/jing"

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT.
# Example: "http://media.lawrence.com"
MEDIA_URL = ''

# Default formatting for date objects. See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
DATE_FORMAT = 'N j, Y'

# Default formatting for datetime objects. See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
DATETIME_FORMAT = 'N j, Y, P'

# Default formatting for time objects. See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
TIME_FORMAT = 'P'

# Default formatting for date objects when only the year and month are relevant.
# See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
YEAR_MONTH_FORMAT = 'F Y'

# Default formatting for date objects when only the month and day are relevant.
# See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
MONTH_DAY_FORMAT = 'F j'

# Do you want to manage transactions manually?
# Hint: you really don't!
TRANSACTIONS_MANAGED = False

# The User-Agent string to use when checking for URL validity through the
# isExistingURL validator.
from django import get_version
URL_VALIDATOR_USER_AGENT = "Django/%s (http://www.djangoproject.com)" % get_version()

##############
# MIDDLEWARE #
##############

# List of middleware classes to use.  Order is important; in the request phase,
# this middleware classes will be applied in the order given, and in the
# response phase the middleware will be applied in reverse order.
MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
#     'django.middleware.http.ConditionalGetMiddleware',
#     'django.middleware.gzip.GZipMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.doc.XViewMiddleware',
)

############
# SESSIONS #
############

SESSION_COOKIE_NAME = 'sessionid'                       # Cookie name. This can be whatever you want.
SESSION_COOKIE_AGE = 60 * 60 * 24 * 7 * 2               # Age of cookie, in seconds (default: 2 weeks).
SESSION_COOKIE_DOMAIN = None                            # A string like ".lawrence.com", or None for standard domain cookie.
SESSION_COOKIE_SECURE = False                           # Whether the session cookie should be secure (https:// only).
SESSION_COOKIE_PATH = '/'                               # The path of the session cookie.
SESSION_SAVE_EVERY_REQUEST = False                      # Whether to save the session data on every request.
SESSION_EXPIRE_AT_BROWSER_CLOSE = False                 # Whether sessions expire when a user closes his browser.
SESSION_ENGINE = 'django.contrib.sessions.backends.db'  # The module to store session data
SESSION_FILE_PATH = '/tmp/'                             # Directory to store session files if using the file session module

#########
# CACHE #
#########

# The cache backend to use.  See the docstring in django.core.cache for the
# possible values.
CACHE_BACKEND = 'simple://'
CACHE_MIDDLEWARE_KEY_PREFIX = ''
CACHE_MIDDLEWARE_SECONDS = 600

####################
# COMMENTS         #
####################

COMMENTS_ALLOW_PROFANITIES = False

# The profanities that will trigger a validation error in the
# 'hasNoProfanities' validator. All of these should be in lowercase.
PROFANITIES_LIST = ('asshat', 'asshead', 'asshole', 'cunt', 'fuck', 'gook', 'nigger', 'shit')

# The group ID that designates which users are banned.
# Set to None if you're not using it.
COMMENTS_BANNED_USERS_GROUP = None

# The group ID that designates which users can moderate comments.
# Set to None if you're not using it.
COMMENTS_MODERATORS_GROUP = None

# The group ID that designates the users whose comments should be e-mailed to MANAGERS.
# Set to None if you're not using it.
COMMENTS_SKETCHY_USERS_GROUP = None

# The system will e-mail MANAGERS the first COMMENTS_FIRST_FEW comments by each
# user. Set this to 0 if you want to disable it.
COMMENTS_FIRST_FEW = 0

# A tuple of IP addresses that have been banned from participating in various
# Django-powered features.
BANNED_IPS = ()

##################
# AUTHENTICATION #
##################

AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend',)

LOGIN_URL = '/accounts/login/'

LOGOUT_URL = '/accounts/logout/'

LOGIN_REDIRECT_URL = '/accounts/profile/'

###########
# TESTING #
###########

# The name of the method to use to invoke the test suite
TEST_RUNNER = 'django.test.simple.run_tests'

# The name of the database to use for testing purposes.
# If None, a name of 'test_' + DATABASE_NAME will be assumed
TEST_DATABASE_NAME = None

# Strings used to set the character set and collation order for the test
# database. These values are passed literally to the server, so they are
# backend-dependent. If None, no special settings are sent (system defaults are
# used).
TEST_DATABASE_CHARSET = None
TEST_DATABASE_COLLATION = None

############
# FIXTURES #
############

# The list of directories to search for fixtures
FIXTURE_DIRS = ()
<commit_msg>Changed the location of language codes to match the project settings file from [6123].<commit_after># Default Django settings. Override these with settings in the module
# pointed-to by the DJANGO_SETTINGS_MODULE environment variable.

# This is defined here as a do-nothing function because we can't import
# django.utils.translation -- that module depends on the settings.
gettext_noop = lambda s: s

####################
# CORE             #
####################

DEBUG = False
TEMPLATE_DEBUG = False

# Whether to use the "Etag" header. This saves bandwidth but slows down performance.
USE_ETAGS = False

# People who get code error notifications.
# In the format (('Full Name', 'email@domain.com'), ('Full Name', 'anotheremail@domain.com'))
ADMINS = ()

# Tuple of IP addresses, as strings, that:
#   * See debug comments, when DEBUG is true
#   * Receive x-headers
INTERNAL_IPS = ()

# Local time zone for this installation. All choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name (although not all
# systems may support all possibilities).
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

# Languages we provide translations for, out of the box. The language name
# should be the utf-8 encoded local name for the language.
LANGUAGES = (
    ('ar', gettext_noop('Arabic')),
    ('bn', gettext_noop('Bengali')),
    ('bg', gettext_noop('Bulgarian')),
    ('ca', gettext_noop('Catalan')),
    ('cs', gettext_noop('Czech')),
    ('cy', gettext_noop('Welsh')),
    ('da', gettext_noop('Danish')),
    ('de', gettext_noop('German')),
    ('el', gettext_noop('Greek')),
    ('en', gettext_noop('English')),
    ('es', gettext_noop('Spanish')),
    ('es_AR', gettext_noop('Argentinean Spanish')),
    ('fa', gettext_noop('Persian')),
    ('fi', gettext_noop('Finnish')),
    ('fr', gettext_noop('French')),
    ('ga', gettext_noop('Gaeilge')),
    ('gl', gettext_noop('Galician')),
    ('hu', gettext_noop('Hungarian')),
    ('he', gettext_noop('Hebrew')),
    ('hr', gettext_noop('Croatian')),
    ('is', gettext_noop('Icelandic')),
    ('it', gettext_noop('Italian')),
    ('ja', gettext_noop('Japanese')),
    ('ko', gettext_noop('Korean')),
    ('km', gettext_noop('Khmer')),
    ('kn', gettext_noop('Kannada')),
    ('lv', gettext_noop('Latvian')),
    ('mk', gettext_noop('Macedonian')),
    ('nl', gettext_noop('Dutch')),
    ('no', gettext_noop('Norwegian')),
    ('pl', gettext_noop('Polish')),
    ('pt', gettext_noop('Portugese')),
    ('pt-br', gettext_noop('Brazilian')),
    ('ro', gettext_noop('Romanian')),
    ('ru', gettext_noop('Russian')),
    ('sk', gettext_noop('Slovak')),
    ('sl', gettext_noop('Slovenian')),
    ('sr', gettext_noop('Serbian')),
    ('sv', gettext_noop('Swedish')),
    ('ta', gettext_noop('Tamil')),
    ('te', gettext_noop('Telugu')),
    ('tr', gettext_noop('Turkish')),
    ('uk', gettext_noop('Ukrainian')),
    ('zh-cn', gettext_noop('Simplified Chinese')),
    ('zh-tw', gettext_noop('Traditional Chinese')),
)

# Languages using BiDi (right-to-left) layout
LANGUAGES_BIDI = ("he", "ar", "fa")

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# Not-necessarily-technical managers of the site. They get broken link
# notifications and other various e-mails.
MANAGERS = ADMINS

# Default content type and charset to use for all HttpResponse objects, if a
# MIME type isn't manually specified. These are used to construct the
# Content-Type header.
DEFAULT_CONTENT_TYPE = 'text/html'
DEFAULT_CHARSET = 'utf-8'

# Encoding of files read from disk (template and initial SQL files).
FILE_CHARSET = 'utf-8'

# E-mail address that error messages come from.
SERVER_EMAIL = 'root@localhost'

# Whether to send broken-link e-mails.
SEND_BROKEN_LINK_EMAILS = False

# Database connection info.
DATABASE_ENGINE = ''           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'.
DATABASE_NAME = ''             # Or path to database file if using sqlite3.
DATABASE_USER = ''             # Not used with sqlite3.
DATABASE_PASSWORD = ''         # Not used with sqlite3.
DATABASE_HOST = ''             # Set to empty string for localhost. Not used with sqlite3.
DATABASE_PORT = ''             # Set to empty string for default. Not used with sqlite3.
DATABASE_OPTIONS = {}          # Set to empty dictionary for default.

# Host for sending e-mail.
EMAIL_HOST = 'localhost'

# Port for sending e-mail.
EMAIL_PORT = 25

# Optional SMTP authentication information for EMAIL_HOST.
EMAIL_HOST_USER = ''
EMAIL_HOST_PASSWORD = ''
EMAIL_USE_TLS = False

# List of strings representing installed apps.
INSTALLED_APPS = ()

# List of locations of the template source files, in search order.
TEMPLATE_DIRS = ()

# List of callables that know how to import templates from various sources.
# See the comments in django/core/template/loader.py for interface
# documentation.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
#     'django.template.loaders.eggs.load_template_source',
)

# List of processors used by RequestContext to populate the context.
# Each one should be a callable that takes the request object as its
# only parameter and returns a dictionary to add to the context.
TEMPLATE_CONTEXT_PROCESSORS = (
    'django.core.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
#    'django.core.context_processors.request',
)

# Output to use in template system for invalid (e.g. misspelled) variables.
TEMPLATE_STRING_IF_INVALID = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'

# Default e-mail address to use for various automated correspondence from
# the site managers.
DEFAULT_FROM_EMAIL = 'webmaster@localhost'

# Subject-line prefix for email messages send with django.core.mail.mail_admins
# or ...mail_managers.  Make sure to include the trailing space.
EMAIL_SUBJECT_PREFIX = '[Django] '

# Whether to append trailing slashes to URLs.
APPEND_SLASH = True

# Whether to prepend the "www." subdomain to URLs that don't have it.
PREPEND_WWW = False

# List of compiled regular expression objects representing User-Agent strings
# that are not allowed to visit any page, systemwide. Use this for bad
# robots/crawlers. Here are a few examples:
#     import re
#     DISALLOWED_USER_AGENTS = (
#         re.compile(r'^NaverBot.*'),
#         re.compile(r'^EmailSiphon.*'),
#         re.compile(r'^SiteSucker.*'),
#         re.compile(r'^sohu-search')
#     )
DISALLOWED_USER_AGENTS = ()

ABSOLUTE_URL_OVERRIDES = {}

# Tuple of strings representing allowed prefixes for the {% ssi %} tag.
# Example: ('/home/html', '/var/www')
ALLOWED_INCLUDE_ROOTS = ()

# If this is a admin settings module, this should be a list of
# settings modules (in the format 'foo.bar.baz') for which this admin
# is an admin.
ADMIN_FOR = ()

# 404s that may be ignored.
IGNORABLE_404_STARTS = ('/cgi-bin/', '/_vti_bin', '/_vti_inf')
IGNORABLE_404_ENDS = ('mail.pl', 'mailform.pl', 'mail.cgi', 'mailform.cgi', 'favicon.ico', '.php')

# A secret key for this particular Django installation. Used in secret-key
# hashing algorithms. Set this in your settings, or Django will complain
# loudly.
SECRET_KEY = ''

# Path to the "jing" executable -- needed to validate XMLFields
JING_PATH = "/usr/bin/jing"

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT.
# Example: "http://media.lawrence.com"
MEDIA_URL = ''

# Default formatting for date objects. See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
DATE_FORMAT = 'N j, Y'

# Default formatting for datetime objects. See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
DATETIME_FORMAT = 'N j, Y, P'

# Default formatting for time objects. See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
TIME_FORMAT = 'P'

# Default formatting for date objects when only the year and month are relevant.
# See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
YEAR_MONTH_FORMAT = 'F Y'

# Default formatting for date objects when only the month and day are relevant.
# See all available format strings here:
# http://www.djangoproject.com/documentation/templates/#now
MONTH_DAY_FORMAT = 'F j'

# Do you want to manage transactions manually?
# Hint: you really don't!
TRANSACTIONS_MANAGED = False

# The User-Agent string to use when checking for URL validity through the
# isExistingURL validator.
from django import get_version
URL_VALIDATOR_USER_AGENT = "Django/%s (http://www.djangoproject.com)" % get_version()

##############
# MIDDLEWARE #
##############

# List of middleware classes to use.  Order is important; in the request phase,
# this middleware classes will be applied in the order given, and in the
# response phase the middleware will be applied in reverse order.
MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
#     'django.middleware.http.ConditionalGetMiddleware',
#     'django.middleware.gzip.GZipMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.doc.XViewMiddleware',
)

############
# SESSIONS #
############

SESSION_COOKIE_NAME = 'sessionid'                       # Cookie name. This can be whatever you want.
SESSION_COOKIE_AGE = 60 * 60 * 24 * 7 * 2               # Age of cookie, in seconds (default: 2 weeks).
SESSION_COOKIE_DOMAIN = None                            # A string like ".lawrence.com", or None for standard domain cookie.
SESSION_COOKIE_SECURE = False                           # Whether the session cookie should be secure (https:// only).
SESSION_COOKIE_PATH = '/'                               # The path of the session cookie.
SESSION_SAVE_EVERY_REQUEST = False                      # Whether to save the session data on every request.
SESSION_EXPIRE_AT_BROWSER_CLOSE = False                 # Whether sessions expire when a user closes his browser.
SESSION_ENGINE = 'django.contrib.sessions.backends.db'  # The module to store session data
SESSION_FILE_PATH = '/tmp/'                             # Directory to store session files if using the file session module

#########
# CACHE #
#########

# The cache backend to use.  See the docstring in django.core.cache for the
# possible values.
CACHE_BACKEND = 'simple://'
CACHE_MIDDLEWARE_KEY_PREFIX = ''
CACHE_MIDDLEWARE_SECONDS = 600

####################
# COMMENTS         #
####################

COMMENTS_ALLOW_PROFANITIES = False

# The profanities that will trigger a validation error in the
# 'hasNoProfanities' validator. All of these should be in lowercase.
PROFANITIES_LIST = ('asshat', 'asshead', 'asshole', 'cunt', 'fuck', 'gook', 'nigger', 'shit')

# The group ID that designates which users are banned.
# Set to None if you're not using it.
COMMENTS_BANNED_USERS_GROUP = None

# The group ID that designates which users can moderate comments.
# Set to None if you're not using it.
COMMENTS_MODERATORS_GROUP = None

# The group ID that designates the users whose comments should be e-mailed to MANAGERS.
# Set to None if you're not using it.
COMMENTS_SKETCHY_USERS_GROUP = None

# The system will e-mail MANAGERS the first COMMENTS_FIRST_FEW comments by each
# user. Set this to 0 if you want to disable it.
COMMENTS_FIRST_FEW = 0

# A tuple of IP addresses that have been banned from participating in various
# Django-powered features.
BANNED_IPS = ()

##################
# AUTHENTICATION #
##################

AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend',)

LOGIN_URL = '/accounts/login/'

LOGOUT_URL = '/accounts/logout/'

LOGIN_REDIRECT_URL = '/accounts/profile/'

###########
# TESTING #
###########

# The name of the method to use to invoke the test suite
TEST_RUNNER = 'django.test.simple.run_tests'

# The name of the database to use for testing purposes.
# If None, a name of 'test_' + DATABASE_NAME will be assumed
TEST_DATABASE_NAME = None

# Strings used to set the character set and collation order for the test
# database. These values are passed literally to the server, so they are
# backend-dependent. If None, no special settings are sent (system defaults are
# used).
TEST_DATABASE_CHARSET = None
TEST_DATABASE_COLLATION = None

############
# FIXTURES #
############

# The list of directories to search for fixtures
FIXTURE_DIRS = ()
<|endoftext|>


<commit_before># Quick tests for the markup templatetags (django.contrib.markup)

import re
import unittest

from django.template import Template, Context, add_to_builtins
from django.utils.html import escape

add_to_builtins('django.contrib.markup.templatetags.markup')

class Templates(unittest.TestCase):
    def test_textile(self):
        try:
            import textile
        except ImportError:
            textile = None

        textile_content = """Paragraph 1

Paragraph 2 with "quotes" and @code@"""

        t = Template("{{ textile_content|textile }}")
        rendered = t.render(Context(locals())).strip()
        if textile:
            self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with “quotes” and code
""")
        else:
            self.assertEqual(rendered, escape(textile_content))

    def test_markdown(self):
        try:
            import markdown
        except ImportError:
            markdown = None

        markdown_content = """Paragraph 1

## An h2"""

        t = Template("{{ markdown_content|markdown }}")
        rendered = t.render(Context(locals())).strip()
        if markdown:
            pattern = re.compile("""
Paragraph 1\s*
\s*
\s*An h2
""")
            self.assert_(pattern.match(rendered))
        else:
            self.assertEqual(rendered, markdown_content)

    def test_docutils(self):
        try:
            import docutils
        except ImportError:
            docutils = None

        rest_content = """Paragraph 1

Paragraph 2 with a link_

.. _link: http://www.example.com/"""

        t = Template("{{ rest_content|restructuredtext }}")
        rendered = t.render(Context(locals())).strip()
        if docutils:
            # Different versions of docutils return slightly different HTML
            try:
                # Docutils v0.4 and earlier
                self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with a link
""")
            except AssertionError, e:
                # Docutils from SVN (which will become 0.5)
                self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with a link
""")
        else:
            self.assertEqual(rendered, rest_content)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fixed #10843: the textile tests now pass against the latest textile library.<commit_after># Quick tests for the markup templatetags (django.contrib.markup)

import re
import unittest

from django.template import Template, Context, add_to_builtins
from django.utils.html import escape

add_to_builtins('django.contrib.markup.templatetags.markup')

class Templates(unittest.TestCase):
    def test_textile(self):
        try:
            import textile
        except ImportError:
            textile = None

        textile_content = """Paragraph 1

Paragraph 2 with "quotes" and @code@"""

        t = Template("{{ textile_content|textile }}")
        rendered = t.render(Context(locals())).strip()
        if textile:
            self.assertEqual(rendered.replace('\t', ''), """
Paragraph 1

Paragraph 2 with “quotes” and code
""")
        else:
            self.assertEqual(rendered, escape(textile_content))

    def test_markdown(self):
        try:
            import markdown
        except ImportError:
            markdown = None

        markdown_content = """Paragraph 1

## An h2"""

        t = Template("{{ markdown_content|markdown }}")
        rendered = t.render(Context(locals())).strip()
        if markdown:
            pattern = re.compile("""
Paragraph 1\s*
\s*
\s*An h2
""")
            self.assert_(pattern.match(rendered))
        else:
            self.assertEqual(rendered, markdown_content)

    def test_docutils(self):
        try:
            import docutils
        except ImportError:
            docutils = None

        rest_content = """Paragraph 1

Paragraph 2 with a link_

.. _link: http://www.example.com/"""

        t = Template("{{ rest_content|restructuredtext }}")
        rendered = t.render(Context(locals())).strip()
        if docutils:
            # Different versions of docutils return slightly different HTML
            try:
                # Docutils v0.4 and earlier
                self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with a link
""")
            except AssertionError, e:
                # Docutils from SVN (which will become 0.5)
                self.assertEqual(rendered, """
Paragraph 1

Paragraph 2 with a link
""")
        else:
            self.assertEqual(rendered, rest_content)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>