<commit_before>"""
Convert XPath selectors into CSS selectors
"""

import re

_sub_regexes = {
    "tag": r"([a-zA-Z][a-zA-Z0-9]{0,10}|\*)",
    "attribute": r"[.a-zA-Z_:][-\w:.]*(\(\))?)",
    "value": r"\s*[\w/:][-/\w\s,:;.\S]*"
}

_validation_re = (
    r"(?P<node>"
    r"("
    r"^id\([\"\']?(?P<idvalue>%(value)s)[\"\']?\)"
    r"|"
    r"(?P<nav>//?)(?P<tag>%(tag)s)"
    r"(\[("
    r"(?P<matched>(?P<mattr>@?%(attribute)s=[\"\']"
    r"(?P<mvalue>%(value)s))[\"\']"
    r"|"
    r"(?P<contained>contains\((?P<cattr>@?%(attribute)s,\s*[\"\']"
    r"(?P<cvalue>%(value)s)[\"\']\))"
    r")\])?"
    r"(\[(?P<nth>\d+)\])?"
    r")"
    r")" % _sub_regexes
)

prog = re.compile(_validation_re)

class XpathException(Exception):
    pass

def _handle_brackets_in_strings(xpath):
    # Edge Case: Brackets in strings.
    # Example from GitHub.com -
    # '<input type="text" id="user[login]">' => '//*[@id="user[login]"]'
    # Need to tell apart string-brackets from regular brackets
    new_xpath = ""
    chunks = xpath.split('"')
    len_chunks = len(chunks)
    for chunk_num in range(len_chunks):
        if chunk_num % 2 != 0:
            chunks[chunk_num] = chunks[chunk_num].replace(
                '[', '_STR_L_bracket_')
            chunks[chunk_num] = chunks[chunk_num].replace(
                ']', '_STR_R_bracket_')
        new_xpath += chunks[chunk_num]
        if chunk_num != len_chunks - 1:
            new_xpath += '"'
    return new_xpath

def _filter_xpath_grouping(xpath):
    """
    This method removes the outer parentheses for xpath grouping.
    The xpath converter will break otherwise.
    Example:
    "(//button[@type='submit'])[1]" becomes "//button[@type='submit'][1]"
    """

    # First remove the first open parentheses
    xpath = xpath[1:]

    # Next remove the last closed parentheses
    index = xpath.rfind(')')
    if index == -1:
        raise XpathException("Invalid or unsupported Xpath: %s" % xpath)
    xpath = xpath[:index] + xpath[index + 1:]
    return xpath

def _get_raw_css_from_xpath(xpath):
    css = ""
    attr = ""
    position = 0

    while position < len(xpath):
        node = prog.match(xpath[position:])
        if node is None:
            raise XpathException("Invalid or unsupported Xpath: %s" % xpath)
        match = node.groupdict()

        if position != 0:
            nav = " " if match['nav'] == "//" else " > "
        else:
            nav = ""

        tag = "" if match['tag'] == "*" else match['tag'] or ""

        if match['idvalue']:
            attr = "#%s" % match['idvalue'].replace(" ", "#")
        elif match['matched']:
            if match['mattr'] == "@id":
                attr = "#%s" % match['mvalue'].replace(" ", "#")
            elif match['mattr'] == "@class":
                attr = ".%s" % match['mvalue'].replace(" ", ".")
            elif match['mattr'] in ["text()", "."]:
                attr = ":contains(^%s$)" % match['mvalue']
            elif match['mattr']:
                attr = '[%s="%s"]' % (match['mattr'].replace("@", ""),
                                      match['mvalue'])
        elif match['contained']:
            if match['cattr'].startswith("@"):
                attr = '[%s*="%s"]' % (match['cattr'].replace("@", ""),
                                       match['cvalue'])
            elif match['cattr'] == "text()":
                attr = ':contains("%s")' % match['cvalue']
            elif match['cattr'] == ".":
                attr = ':contains("%s")' % match['cvalue']
        else:
            attr = ""

        if match['nth']:
            nth = ":nth-of-type(%s)" % match['nth']
        else:
            nth = ""

        node_css = nav + tag + attr + nth
        css += node_css
        position += node.end()
    else:
        css = css.strip()
        return css

def convert_xpath_to_css(xpath):
    xpath = xpath.replace(" = '", "='")

    # **** Start of handling special xpath edge cases instantly ****

    # Handle a special edge case that converts to: 'tag.class:contains("TEXT")'
    c3 = "@class and contains(concat(' ', normalize-space(@class), ' '), ' "
    if c3 in xpath and xpath.count(c3) == 1 and xpath.count("[@") == 1:
        p2 = " ') and (contains(., '"
        if xpath.count(p2) == 1 and xpath.endswith("'))]") and (
                xpath.count("//") == 1 and (xpath.count(" ') and (") == 1)):
            s_contains = xpath.split(p2)[1].split("'))]")[0]
            s_tag = xpath.split("//")[1].split("[@class")[0]
            s_class = xpath.split(c3)[1].split(" ') and (")[0]
            return '%s.%s:contains("%s")' % (s_tag, s_class, s_contains)

    # Find instance of: //tag[@attribute='value' and (contains(., 'TEXT'))]
    data = re.match(
        r'''^\s*//(\S+)\[@(\S+)='(\S+)'\s+and\s+'''
        r'''\(contains\(\.,\s'(\S+)'\)\)\]''', xpath)
    if data:
        s_tag = data.group(1)
        s_atr = data.group(2)
        s_val = data.group(3)
        s_contains = data.group(4)
        return '%s[%s="%s"]:contains("%s")' % (s_tag, s_atr, s_val, s_contains)

    # Find instance of: //tag[@attribute1='value1' and (@attribute2='value2')]
    data = re.match(
        r'''^\s*//(\S+)\[@(\S+)='(\S+)'\s+and\s+'''
        r'''\(@(\S+)='(\S+)'\)\]''', xpath)
    if data:
        s_tag = data.group(1)
        s_atr1 = data.group(2)
        s_val1 = data.group(3)
        s_atr2 = data.group(4)
        s_val2 = data.group(5)
        return '%s[%s="%s"][%s="%s"]' % (s_tag, s_atr1, s_val1, s_atr2, s_val2)

    # **** End of handling special xpath edge cases instantly ****

    if xpath[0] != '"' and xpath[-1] != '"' and xpath.count('"') % 2 == 0:
        xpath = _handle_brackets_in_strings(xpath)
    xpath = xpath.replace("descendant-or-self::*/", "descORself/")

    if " and contains(@" in xpath and xpath.count(" and contains(@") == 1:
        spot1 = xpath.find(" and contains(@")
        spot1 = spot1 + len(" and contains(@")
        spot2 = xpath.find(",", spot1)
        attr = xpath[spot1:spot2]
        swap = " and contains(@%s, " % attr
        if swap in xpath:
            swap_spot = xpath.find(swap)
            close_paren = xpath.find(']', swap_spot) - 1
            if close_paren > 1:
                xpath = xpath[:close_paren] + xpath[close_paren+1:]
                xpath = xpath.replace(swap, "_STAR_=")

    if xpath.startswith('('):
        xpath = _filter_xpath_grouping(xpath)

    css = _get_raw_css_from_xpath(xpath)

    attribute_defs = re.findall(r'(\[\w+\=\S+\])', css)
    for attr_def in attribute_defs:
        if (attr_def.count('[') == 1 and attr_def.count(']') == 1 and (
                attr_def.count('=') == 1) and attr_def.count('"') == 0 and (
                attr_def.count("'") == 0)) and attr_def.count(' ') == 0:
            # Now safe to manipulate
            q1 = attr_def.find('=') + 1
            q2 = attr_def.find(']')
            new_attr_def = attr_def[:q1] + "'" + attr_def[q1:q2] + "']"
            css = css.replace(attr_def, new_attr_def)

    # Replace the string-brackets with escaped ones
    css = css.replace('_STR_L_bracket_', '\\[')
    css = css.replace('_STR_R_bracket_', '\\]')

    # Handle a lot of edge cases with conversion
    css = css.replace(" > descORself > ", ' ')
    css = css.replace(" descORself > ", ' ')
    css = css.replace("/descORself/*", ' ')
    css = css.replace("/descORself/", ' ')
    css = css.replace("descORself/", ' ')
    css = css.replace("_STAR_=", "*=")
    css = css.replace("]/", "] ")
    css = css.replace("] *[", "] > [")
    css = css.replace("\'", '"')
    css = css.replace("[@", '[')

    return css
<commit_msg>Another update to the xpath-to-css converter<commit_after>"""
Convert XPath selectors into CSS selectors
"""

import re

_sub_regexes = {
    "tag": r"([a-zA-Z][a-zA-Z0-9]{0,10}|\*)",
    "attribute": r"[.a-zA-Z_:][-\w:.]*(\(\))?)",
    "value": r"\s*[\w/:][-/\w\s,:;.\S]*"
}

_validation_re = (
    r"(?P<node>"
    r"("
    r"^id\([\"\']?(?P<idvalue>%(value)s)[\"\']?\)"
    r"|"
    r"(?P<nav>//?)(?P<tag>%(tag)s)"
    r"(\[("
    r"(?P<matched>(?P<mattr>@?%(attribute)s=[\"\']"
    r"(?P<mvalue>%(value)s))[\"\']"
    r"|"
    r"(?P<contained>contains\((?P<cattr>@?%(attribute)s,\s*[\"\']"
    r"(?P<cvalue>%(value)s)[\"\']\))"
    r")\])?"
    r"(\[(?P<nth>\d+)\])?"
    r")"
    r")" % _sub_regexes
)

prog = re.compile(_validation_re)

class XpathException(Exception):
    pass

def _handle_brackets_in_strings(xpath):
    # Edge Case: Brackets in strings.
    # Example from GitHub.com -
    # '<input type="text" id="user[login]">' => '//*[@id="user[login]"]'
    # Need to tell apart string-brackets from regular brackets
    new_xpath = ""
    chunks = xpath.split('"')
    len_chunks = len(chunks)
    for chunk_num in range(len_chunks):
        if chunk_num % 2 != 0:
            chunks[chunk_num] = chunks[chunk_num].replace(
                '[', '_STR_L_bracket_')
            chunks[chunk_num] = chunks[chunk_num].replace(
                ']', '_STR_R_bracket_')
        new_xpath += chunks[chunk_num]
        if chunk_num != len_chunks - 1:
            new_xpath += '"'
    return new_xpath

def _filter_xpath_grouping(xpath):
    """
    This method removes the outer parentheses for xpath grouping.
    The xpath converter will break otherwise.
    Example:
    "(//button[@type='submit'])[1]" becomes "//button[@type='submit'][1]"
    """

    # First remove the first open parentheses
    xpath = xpath[1:]

    # Next remove the last closed parentheses
    index = xpath.rfind(')')
    if index == -1:
        raise XpathException("Invalid or unsupported Xpath: %s" % xpath)
    xpath = xpath[:index] + xpath[index + 1:]
    return xpath

def _get_raw_css_from_xpath(xpath):
    css = ""
    attr = ""
    position = 0

    while position < len(xpath):
        node = prog.match(xpath[position:])
        if node is None:
            raise XpathException("Invalid or unsupported Xpath: %s" % xpath)
        match = node.groupdict()

        if position != 0:
            nav = " " if match['nav'] == "//" else " > "
        else:
            nav = ""

        tag = "" if match['tag'] == "*" else match['tag'] or ""

        if match['idvalue']:
            attr = "#%s" % match['idvalue'].replace(" ", "#")
        elif match['matched']:
            if match['mattr'] == "@id":
                attr = "#%s" % match['mvalue'].replace(" ", "#")
            elif match['mattr'] == "@class":
                attr = ".%s" % match['mvalue'].replace(" ", ".")
            elif match['mattr'] in ["text()", "."]:
                attr = ":contains(^%s$)" % match['mvalue']
            elif match['mattr']:
                attr = '[%s="%s"]' % (match['mattr'].replace("@", ""),
                                      match['mvalue'])
        elif match['contained']:
            if match['cattr'].startswith("@"):
                attr = '[%s*="%s"]' % (match['cattr'].replace("@", ""),
                                       match['cvalue'])
            elif match['cattr'] == "text()":
                attr = ':contains("%s")' % match['cvalue']
            elif match['cattr'] == ".":
                attr = ':contains("%s")' % match['cvalue']
        else:
            attr = ""

        if match['nth']:
            nth = ":nth-of-type(%s)" % match['nth']
        else:
            nth = ""

        node_css = nav + tag + attr + nth
        css += node_css
        position += node.end()
    else:
        css = css.strip()
        return css

def convert_xpath_to_css(xpath):
    xpath = xpath.replace(" = '", "='")

    # **** Start of handling special xpath edge cases instantly ****

    # Handle a special edge case that converts to: 'tag.class:contains("TEXT")'
    c3 = "@class and contains(concat(' ', normalize-space(@class), ' '), ' "
    if c3 in xpath and xpath.count(c3) == 1 and xpath.count("[@") == 1:
        p2 = " ') and (contains(., '"
        if xpath.count(p2) == 1 and xpath.endswith("'))]") and (
                xpath.count("//") == 1 and (xpath.count(" ') and (") == 1)):
            s_contains = xpath.split(p2)[1].split("'))]")[0]
            s_tag = xpath.split("//")[1].split("[@class")[0]
            s_class = xpath.split(c3)[1].split(" ') and (")[0]
            return '%s.%s:contains("%s")' % (s_tag, s_class, s_contains)

    # Find instance of: //tag[@attribute='value' and (contains(., 'TEXT'))]
    data = re.match(
        r'''^\s*//(\S+)\[@(\S+)='(\S+)'\s+and\s+'''
        r'''\(contains\(\.,\s'(\S+)'\)\)\]''', xpath)
    if data:
        s_tag = data.group(1)
        s_atr = data.group(2)
        s_val = data.group(3)
        s_contains = data.group(4)
        return '%s[%s="%s"]:contains("%s")' % (s_tag, s_atr, s_val, s_contains)

    # Find instance of: //tag[@attribute1='value1' and (@attribute2='value2')]
    data = re.match(
        r'''^\s*//(\S+)\[@(\S+)='(\S+)'\s+and\s+'''
        r'''\(@(\S+)='(\S+)'\)\]''', xpath)
    if data:
        s_tag = data.group(1)
        s_atr1 = data.group(2)
        s_val1 = data.group(3)
        s_atr2 = data.group(4)
        s_val2 = data.group(5)
        return '%s[%s="%s"][%s="%s"]' % (s_tag, s_atr1, s_val1, s_atr2, s_val2)

    # **** End of handling special xpath edge cases instantly ****

    if xpath[0] != '"' and xpath[-1] != '"' and xpath.count('"') % 2 == 0:
        xpath = _handle_brackets_in_strings(xpath)
    xpath = xpath.replace("descendant-or-self::*/", "descORself/")
    if len(xpath) > 3:
        xpath = xpath[0:3] + xpath[3:].replace('//', '/descORself/')

    if " and contains(@" in xpath and xpath.count(" and contains(@") == 1:
        spot1 = xpath.find(" and contains(@")
        spot1 = spot1 + len(" and contains(@")
        spot2 = xpath.find(",", spot1)
        attr = xpath[spot1:spot2]
        swap = " and contains(@%s, " % attr
        if swap in xpath:
            swap_spot = xpath.find(swap)
            close_paren = xpath.find(']', swap_spot) - 1
            if close_paren > 1:
                xpath = xpath[:close_paren] + xpath[close_paren+1:]
                xpath = xpath.replace(swap, "_STAR_=")

    if xpath.startswith('('):
        xpath = _filter_xpath_grouping(xpath)

    css = _get_raw_css_from_xpath(xpath)

    attribute_defs = re.findall(r'(\[\w+\=\S+\])', css)
    for attr_def in attribute_defs:
        if (attr_def.count('[') == 1 and attr_def.count(']') == 1 and (
                attr_def.count('=') == 1) and attr_def.count('"') == 0 and (
                attr_def.count("'") == 0)) and attr_def.count(' ') == 0:
            # Now safe to manipulate
            q1 = attr_def.find('=') + 1
            q2 = attr_def.find(']')
            new_attr_def = attr_def[:q1] + "'" + attr_def[q1:q2] + "']"
            css = css.replace(attr_def, new_attr_def)

    # Replace the string-brackets with escaped ones
    css = css.replace('_STR_L_bracket_', '\\[')
    css = css.replace('_STR_R_bracket_', '\\]')

    # Handle a lot of edge cases with conversion
    css = css.replace(" > descORself > ", ' ')
    css = css.replace(" descORself > ", ' ')
    css = css.replace("/descORself/*", ' ')
    css = css.replace("/descORself/", ' ')
    css = css.replace("descORself > ", '')
    css = css.replace("descORself/", ' ')
    css = css.replace("descORself", ' ')
    css = css.replace("_STAR_=", "*=")
    css = css.replace("]/", "] ")
    css = css.replace("] *[", "] > [")
    css = css.replace("\'", '"')
    css = css.replace("[@", '[')

    return css
<|endoftext|>


<commit_before>from senpy.plugins import EmotionConversionPlugin
from senpy.models import EmotionSet, Emotion, Error

import logging
logger = logging.getLogger(__name__)

class CentroidConversion(EmotionConversionPlugin):
    def __init__(self, info):
        if 'centroids' not in info:
            raise Error('Centroid conversion plugins should provide '
                        'the centroids in their senpy file')
        if 'onyx:doesConversion' not in info:
            if 'centroids_direction' not in info:
                raise Error('Please, provide centroids direction')

            cf, ct = info['centroids_direction']
            info['onyx:doesConversion'] = [{
                'onyx:conversionFrom': cf,
                'onyx:conversionTo': ct
            }, {
                'onyx:conversionFrom': ct,
                'onyx:conversionTo': cf
            }]

        if 'aliases' in info:
            aliases = info['aliases']
            ncentroids = {}
            for k1, v1 in info['centroids'].items():
                nv1 = {}
                for k2, v2 in v1.items():
                    nv1[aliases.get(k2, k2)] = v2
                ncentroids[aliases.get(k1, k1)] = nv1
            info['centroids'] = ncentroids
        super(CentroidConversion, self).__init__(info)

    def _forward_conversion(self, original):
        """Sum the VAD value of all categories found."""
        res = Emotion()
        for e in original.onyx__hasEmotion:
            category = e.onyx__hasEmotionCategory
            if category in self.centroids:
                for dim, value in self.centroids[category].items():
                    try:
                        res[dim] += value
                    except Exception:
                        res[dim] = value
        return res

    def _backwards_conversion(self, original):
        """Find the closest category"""
        dimensions = set(c.keys() for c in centroids.values())
        neutralPoint = self.get("origin", None)
        neutralPoint = {k:neutralPoint[k] if k in neturalPoint else 0}

        def distance(centroid):
            return sum((centroid.get(k, neutralPoint[k]) - original.get(k, neutralPoint[k]))**2 for k in dimensions)

        emotion = min(centroids, key=lambda x: distance(centroids[x])

        result = Emotion(onyx__hasEmotionCategory=emotion)
        return result

    def convert(self, emotionSet, fromModel, toModel, params):

        cf, ct = self.centroids_direction
        logger.debug(
            '{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
        e = EmotionSet()
        if fromModel == cf and toModel == ct:
            e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
        elif fromModel == ct and toModel == cf:
            for i in emotionSet.onyx__hasEmotion:
                e.onyx__hasEmotion.append(self._backwards_conversion(i))
        else:
            raise Error('EMOTION MODEL NOT KNOWN')
        yield e
<commit_msg>Change conversion to Euclidean distance<commit_after>from senpy.plugins import EmotionConversionPlugin
from senpy.models import EmotionSet, Emotion, Error

import logging
logger = logging.getLogger(__name__)

class CentroidConversion(EmotionConversionPlugin):
    def __init__(self, info):
        if 'centroids' not in info:
            raise Error('Centroid conversion plugins should provide '
                        'the centroids in their senpy file')
        if 'onyx:doesConversion' not in info:
            if 'centroids_direction' not in info:
                raise Error('Please, provide centroids direction')

            cf, ct = info['centroids_direction']
            info['onyx:doesConversion'] = [{
                'onyx:conversionFrom': cf,
                'onyx:conversionTo': ct
            }, {
                'onyx:conversionFrom': ct,
                'onyx:conversionTo': cf
            }]

        if 'aliases' in info:
            aliases = info['aliases']
            ncentroids = {}
            for k1, v1 in info['centroids'].items():
                nv1 = {}
                for k2, v2 in v1.items():
                    nv1[aliases.get(k2, k2)] = v2
                ncentroids[aliases.get(k1, k1)] = nv1
            info['centroids'] = ncentroids

        super(CentroidConversion, self).__init__(info)

        self.dimensions = set()
        for c in self.centroids.values():
            self.dimensions.update(c.keys())
        self.neutralPoints = self.get("neutralPoints", dict())
        if not self.neutralPoints:
            for i in self.dimensions:
                self.neutralPoints[i] = self.get("neutralValue", 0)

    def _forward_conversion(self, original):
        """Sum the VAD value of all categories found."""
        res = Emotion()
        for e in original.onyx__hasEmotion:
            category = e.onyx__hasEmotionCategory
            if category in self.centroids:
                for dim, value in self.centroids[category].items():
                    try:
                        res[dim] += value
                    except Exception:
                        res[dim] = value
        return res

    def _backwards_conversion(self, original):
        """Find the closest category"""
        centroids = self.centroids
        neutralPoints = self.neutralPoints
        dimensions = self.dimensions

        def distance_k(centroid, original, k):
            # k component of the distance between the value and a given centroid
            return (centroid.get(k, neutralPoints[k]) - original.get(k, neutralPoints[k]))**2

        def distance(centroid):
            return sum(distance_k(centroid, original, k) for k in dimensions)

        emotion = min(centroids, key=lambda x: distance(centroids[x]))

        result = Emotion(onyx__hasEmotionCategory=emotion)
        return result

    def convert(self, emotionSet, fromModel, toModel, params):

        cf, ct = self.centroids_direction
        logger.debug(
            '{}\n{}\n{}\n{}'.format(emotionSet, fromModel, toModel, params))
        e = EmotionSet()
        if fromModel == cf and toModel == ct:
            e.onyx__hasEmotion.append(self._forward_conversion(emotionSet))
        elif fromModel == ct and toModel == cf:
            for i in emotionSet.onyx__hasEmotion:
                e.onyx__hasEmotion.append(self._backwards_conversion(i))
        else:
            raise Error('EMOTION MODEL NOT KNOWN')
        yield e
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Wed May 15 19:23:23 2013

@author: andres
"""
import networkx as nx
import networkb, numpy, json
from scipy import spatial

def mass_path_distance(bn,N_clus,mcs,op='first',correlation='both'):
  N=[]
  lmean=[]
  lmax=[]
  rmax=[]
  voxels=bn.get_node2voxel()
  affine=bn.get_affine()

  jumps=networkb.find_th_jumps(bn,N_clus)
  if op=='first':
    if len(jumps)==0:
      print 'no jumps'
      return {'N':[],'rmax':[],'lmax':[],'lmean':[]}
    pc=max(jumps)
  elif op=='biggest':
    gap=0
    pc=1
    gc=bn.get_gc()[0]
    th=bn.get_th()
    if len(jumps)==0:
      print 'no jumps'
      return {'N':[],'rmax':[],'lmax':[],'lmean':[]}
    for jump in jumps:
      j=th.index(jump)
      if gc[j-1]-gc[j]>gap:
        pc=jump
        gap=gc[j-1]-gc[j]
  print 'pc='+str(pc)
  G=bn.get_Graph(pc,correlation=correlation)
  print 'number of nodes in network at pc: '+str(G.number_of_nodes())
  cluster_list=nx.connected_components(G)
  edge_list=G.edges()
  print 'number of clusters: '+str(len(cluster_list))
  #cluster_list=bn.get_cluster_dic()[pc]
  N=[len(x) for x in cluster_list]
  """
  cluster_list=networkb.define_clusters(bn,jump_size,mcs)
  N=[len(x[1]) for x in cluster_list]
  """
  for i,nodelist in enumerate(cluster_list):
    if N[i]>10:
      print str(pc)+'  '+str(i)+', '+str(N[i])
    Gsub=bn.get_SubGraph(float(pc),nodelist,edge_list)
    A=numpy.zeros([N[i],3])
    pos=numpy.array([0,0,0,1])
    path_length=[]
    H = Gsub.copy()
    for j,node in enumerate(H):
      temp=[x for x in nx.single_source_shortest_path_length(Gsub, node).values() if x>0]
      path_length.extend(temp)
      pos[0:3]=voxels[str(node)]
      A[j,:]=numpy.dot(affine, numpy.transpose(pos))[0:3]
      Gsub.remove_node(node)
    D=spatial.distance.cdist(A,A)
    rmax.append(numpy.max(D))
    lmax.append(max(path_length))
    lmean.append(numpy.mean(path_length))
    del Gsub, D, A, pos
  json.dump({'N':N,'rmax':rmax,'lmax':lmax,'lmean':lmean},
            open(bn.cluster_properties_file,'w'))
  return {'N':N,'rmax':rmax,'lmax':lmax,'lmean':lmean}
<commit_msg>add help string to mass_path_distance and delete old comments<commit_after># -*- coding: utf-8 -*-
"""
Created on Wed May 15 19:23:23 2013

@author: andres
"""
import networkx as nx
import networkb, numpy, json
from scipy import spatial

def mass_path_distance(bn,N_clus,mcs,op='first',correlation='both'):
  """
  Returns mass, maximun radio, maximun path length and average path
  length for all clusters at critical threshold. If op='first' the
  critical threshold is the nearest to 1 and if op='biggest' the
  threshold is the one with the biggest jump in the percolation curve.
  """
  N=[]
  lmean=[]
  lmax=[]
  rmax=[]
  voxels=bn.get_node2voxel()
  affine=bn.get_affine()

  jumps=networkb.find_th_jumps(bn,N_clus)
  if op=='first':
    if len(jumps)==0:
      print 'no jumps'
      return {'N':[],'rmax':[],'lmax':[],'lmean':[]}
    pc=max(jumps)
  elif op=='biggest':
    gap=0
    pc=1
    gc=bn.get_gc()[0]
    th=bn.get_th()
    if len(jumps)==0:
      print 'no jumps'
      return {'N':[],'rmax':[],'lmax':[],'lmean':[]}
    for jump in jumps:
      j=th.index(jump)
      if gc[j-1]-gc[j]>gap:
        pc=jump
        gap=gc[j-1]-gc[j]
  print 'pc='+str(pc)
  G=bn.get_Graph(pc,correlation=correlation)
  print 'number of nodes in network at pc: '+str(G.number_of_nodes())
  cluster_list=nx.connected_components(G)
  edge_list=G.edges()
  print 'number of clusters: '+str(len(cluster_list))
  N=[len(x) for x in cluster_list]
  for i,nodelist in enumerate(cluster_list):
    if N[i]>10:
      print str(pc)+'  '+str(i)+', '+str(N[i])
    Gsub=bn.get_SubGraph(float(pc),nodelist,edge_list)
    A=numpy.zeros([N[i],3])
    pos=numpy.array([0,0,0,1])
    path_length=[]
    H = Gsub.copy()
    for j,node in enumerate(H):
      temp=[x for x in nx.single_source_shortest_path_length(Gsub, node).values() if x>0]
      path_length.extend(temp)
      pos[0:3]=voxels[str(node)]
      A[j,:]=numpy.dot(affine, numpy.transpose(pos))[0:3]
      Gsub.remove_node(node)
    D=spatial.distance.cdist(A,A)
    rmax.append(numpy.max(D))
    lmax.append(max(path_length))
    lmean.append(numpy.mean(path_length))
    del Gsub, D, A, pos
  json.dump({'N':N,'rmax':rmax,'lmax':lmax,'lmean':lmean},
            open(bn.cluster_properties_file,'w'))
  return {'N':N,'rmax':rmax,'lmax':lmax,'lmean':lmean}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-05-29 14:15
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0039_fix_reviewer_url'),
    ]

    operations = [
        migrations.AlterField(
            model_name='galley',
            name='type',
            field=models.CharField(choices=[('pdf', 'PDF'), ('epub', 'EPUB'), ('html', 'HTML'), ('xml', 'XML'), ('doc', 'Word (Doc)'), ('docx', 'Word (DOCX)'), ('odt', 'OpenDocument Text Document'), ('tex', 'LaTeX'), ('rtf', 'RTF'), ('other', 'Other')], max_length=100),
        ),
        migrations.AlterUniqueTogether(
            name='account',
            unique_together=set([('email', 'username')]),
        ),
    ]
<commit_msg>Updates choices on older migration<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-05-29 14:15
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0039_fix_reviewer_url'),
    ]

    operations = [
        migrations.AlterField(
            model_name='galley',
            name='type',
            field=models.CharField(choices=[('pdf', 'PDF'), ('epub', 'EPUB'), ('html', 'HTML'), ('xml', 'XML'), ('doc', 'Word (Doc)'), ('docx', 'Word (DOCX)'), ('odt', 'OpenDocument Text Document'), ('tex', 'LaTeX'), ('rtf', 'RTF'), ('other', 'Other'), ('image', 'Image')], max_length=100),
        ),
        migrations.AlterUniqueTogether(
            name='account',
            unique_together=set([('email', 'username')]),
        ),
    ]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright 2017 Eddie Antonio Santos <easantos@ualberta.ca>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import atexit
import os
import sys
import token
from io import BytesIO
from keyword import iskeyword
from pathlib import Path
from typing import (
    Any, AnyStr, Callable, IO, Iterable, Optional, Tuple, Union,
    overload, cast
)

import javac_parser

from .. import Language, SourceSummary
from ...lexical_analysis import Lexeme, Location, Position, Token
from ...vocabulary import Vocabulary, Vind

here = Path(__file__).parent

# XXX: Should probably be elsewhere.
class NoSourceRepresentationError(ValueError):
    """
    Raise when there is no way to convert the Vocabulary index into a
    token that can be inserted into the file.
    """

class JavaVocabulary(Vocabulary):
    """
    The vocabulary, except it returns from
    """
    first_entry_num = len(Vocabulary.SPECIAL_ENTRIES)

    def __init__(self, entries: Iterable[str], reprs: Iterable[str]) -> None:
        super().__init__(entries)
        # Create a look-up table for source representations.
        # The special tokens <unk>, <s>, </s> have NO reprs, thus are not
        # stored.
        self._index2repr = tuple(reprs)
        assert len(self._index2text) == self.first_entry_num + len(self._index2repr)

    def to_source_text(self, idx: Vind) -> str:
        if idx < self.first_entry_num:
            raise NoSourceRepresentationError(idx)
        return self._index2repr[idx - self.first_entry_num]

    @staticmethod
    def load() -> 'JavaVocabulary':
        entries = []
        reprs = []

        # Load from a tab-separated-values file
        with open(here / 'vocabulary.tsv') as vocab_file:
            first_entry = JavaVocabulary.first_entry_num
            for expected_num, line in enumerate(vocab_file, start=first_entry):
                # src_repr -- source representation
                num, entry, src_repr = line.rstrip().split()
                assert expected_num == int(num)
                entries.append(entry)
                reprs.append(src_repr)

        return JavaVocabulary(entries, reprs)

def to_str(source: Union[str, bytes, IO[bytes]]) -> str:
    """
    Coerce an input format to a Unicode string.
    """
    if isinstance(source, str):
        return source
    elif isinstance(source, bytes):
        # XXX: Assume it's UTF-8 encoded!
        return source.decode('UTF-8')
    else:
        raise NotImplementedError

class LazyVocabulary:
    def __init__(self, fn):
        self.fn = fn

    def __get__(self, obj, value):
        if not hasattr(self, 'value'):
            self.value = self.fn()
        return self.value

class Java(Language):
    """
    Defines the Java 8 programming language.
    """

    extensions = {'.java'}
    vocabulary = cast(Vocabulary, LazyVocabulary(JavaVocabulary.load))

    @property
    def java(self):
        """
        Lazily start up the Java server. This decreases the chances of things
        going horribly wrong when two seperate process initialize
        the Java language instance around the same time.
        """
        if not hasattr(self, '_java_server'):
            self._java_server = javac_parser.Java()
        return self._java_server

    def tokenize(self, source: Union[str, bytes, IO[bytes]]) -> Iterable[Token]:
        tokens = self.java.lex(to_str(source))
        # Each token is a tuple with the following structure
        # (reproduced from javac_parser.py):
        #   1. Lexeme type
        #   2. Value (as it appears in the source file)
        #   3. A 2-tuple of start line, start column
        #   4. A 2-tuple of end line, end column
        #   5. A whitespace-free representation of the value
        for name, _raw_value, start, end, normalized in tokens:
            # Omit the EOF token, as it's only useful for the parser.
            if name == 'EOF':
                continue
            # Take the NORMALIZED value, as Java allows unicode escapes in
            # ARBITRARY tokens and then things get hairy here.
            yield Token(name=name, value=normalized,
                        start=Position(line=start[0], column=start[1]),
                        end=Position(line=end[0], column=end[1]))

    def check_syntax(self, source: Union[str, bytes]) -> bool:
        return self.java.get_num_parse_errors(to_str(source)) == 0

    def summarize_tokens(self, source: Iterable[Token]) -> SourceSummary:
        toks = [tok for tok in source if tok.name != 'EOF']
        slines = set(line for tok in toks for line in tok.lines)
        return SourceSummary(n_tokens=len(toks), sloc=len(slines))

    def vocabularize_tokens(self, source: Iterable[Token]) -> Iterable[Tuple[Location, str]]:
        for token in source:
            yield token.location, token.name

java: Language = Java()
<commit_msg>Fix teardown bug again.<commit_after>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright 2017 Eddie Antonio Santos <easantos@ualberta.ca>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import atexit
import os
import sys
import token
from io import BytesIO
from keyword import iskeyword
from pathlib import Path
from typing import (
    Any, AnyStr, Callable, IO, Iterable, Optional, Tuple, Union,
    overload, cast
)

import javac_parser

from .. import Language, SourceSummary
from ...lexical_analysis import Lexeme, Location, Position, Token
from ...vocabulary import Vocabulary, Vind

here = Path(__file__).parent

# XXX: Should probably be elsewhere.
class NoSourceRepresentationError(ValueError):
    """
    Raise when there is no way to convert the Vocabulary index into a
    token that can be inserted into the file.
    """

class JavaVocabulary(Vocabulary):
    """
    The vocabulary, except it returns from
    """
    first_entry_num = len(Vocabulary.SPECIAL_ENTRIES)

    def __init__(self, entries: Iterable[str], reprs: Iterable[str]) -> None:
        super().__init__(entries)
        # Create a look-up table for source representations.
        # The special tokens <unk>, <s>, </s> have NO reprs, thus are not
        # stored.
        self._index2repr = tuple(reprs)
        assert len(self._index2text) == self.first_entry_num + len(self._index2repr)

    def to_source_text(self, idx: Vind) -> str:
        if idx < self.first_entry_num:
            raise NoSourceRepresentationError(idx)
        return self._index2repr[idx - self.first_entry_num]

    @staticmethod
    def load() -> 'JavaVocabulary':
        entries = []
        reprs = []

        # Load from a tab-separated-values file
        with open(here / 'vocabulary.tsv') as vocab_file:
            first_entry = JavaVocabulary.first_entry_num
            for expected_num, line in enumerate(vocab_file, start=first_entry):
                # src_repr -- source representation
                num, entry, src_repr = line.rstrip().split()
                assert expected_num == int(num)
                entries.append(entry)
                reprs.append(src_repr)

        return JavaVocabulary(entries, reprs)

def to_str(source: Union[str, bytes, IO[bytes]]) -> str:
    """
    Coerce an input format to a Unicode string.
    """
    if isinstance(source, str):
        return source
    elif isinstance(source, bytes):
        # XXX: Assume it's UTF-8 encoded!
        return source.decode('UTF-8')
    else:
        raise NotImplementedError

class LazyVocabulary:
    def __init__(self, fn):
        self.fn = fn

    def __get__(self, obj, value):
        if not hasattr(self, 'value'):
            self.value = self.fn()
        return self.value

class Java(Language):
    """
    Defines the Java 8 programming language.
    """

    extensions = {'.java'}
    vocabulary = cast(Vocabulary, LazyVocabulary(JavaVocabulary.load))

    @property
    def java(self):
        """
        Lazily start up the Java server. This decreases the chances of things
        going horribly wrong when two seperate process initialize
        the Java language instance around the same time.
        """
        if not hasattr(self, '_java_server'):
            self._java_server = javac_parser.Java()

            # Py4j usually crashes as Python is cleaning up after exit() so
            # decrement the servers' reference count to lessen the chance of
            # that happening.
            @atexit.register
            def remove_reference():
                del self._java_server

        return self._java_server

    def tokenize(self, source: Union[str, bytes, IO[bytes]]) -> Iterable[Token]:
        tokens = self.java.lex(to_str(source))
        # Each token is a tuple with the following structure
        # (reproduced from javac_parser.py):
        #   1. Lexeme type
        #   2. Value (as it appears in the source file)
        #   3. A 2-tuple of start line, start column
        #   4. A 2-tuple of end line, end column
        #   5. A whitespace-free representation of the value
        for name, _raw_value, start, end, normalized in tokens:
            # Omit the EOF token, as it's only useful for the parser.
            if name == 'EOF':
                continue
            # Take the NORMALIZED value, as Java allows unicode escapes in
            # ARBITRARY tokens and then things get hairy here.
            yield Token(name=name, value=normalized,
                        start=Position(line=start[0], column=start[1]),
                        end=Position(line=end[0], column=end[1]))

    def check_syntax(self, source: Union[str, bytes]) -> bool:
        return self.java.get_num_parse_errors(to_str(source)) == 0

    def summarize_tokens(self, source: Iterable[Token]) -> SourceSummary:
        toks = [tok for tok in source if tok.name != 'EOF']
        slines = set(line for tok in toks for line in tok.lines)
        return SourceSummary(n_tokens=len(toks), sloc=len(slines))

    def vocabularize_tokens(self, source: Iterable[Token]) -> Iterable[Tuple[Location, str]]:
        for token in source:
            yield token.location, token.name

java: Language = Java()
<|endoftext|>


<commit_before># coding: utf-8
#
#  This file is part of Sequana software
#
#  Copyright (c) 2016 - Sequana Development Team
#
#  File author(s):
#      Dimitri Desvillechabrol <dimitri.desvillechabrol@pasteur.fr>,
#          <d.desvillechabrol@gmail.com>
#
#  Distributed under the terms of the 3-clause BSD license.
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################
"""Generic module is the parent module of all other module"""
import os
import shutil
import io
import base64

from jinja2 import Environment, PackageLoader

#from reports import HTMLTable
from sequana.utils import config
from sequana import logger

__all__ = ["SequanaBaseModule"]

class SequanaBaseModule(object):
    """ Generic Module to write HTML reports.
    """
    def __init__(self, template_fn='standard.html', required_dir=None):
        if required_dir is None:
            self.required_dir=("css", "js", "images")
        else:
            self.required_dir = required_dir

        self.output_dir = config.output_dir
        self.path = "./"
        # Initiate jinja template
        env = Environment(
            loader=PackageLoader('sequana', 'resources/templates/')
        )
        self.template = env.get_template(template_fn)
        self._init_report()

    def _init_report(self):
        """ Create the report directory. All necessary directories are copied
        in working directory.
        """
        # Be aware of #465 issue. We need to check that the target file is
        # valid, in which case there is no need to copy the files.

        # Create report directory
        if os.path.isdir(config.output_dir) is False:
            os.mkdir(self.output_dir)

        for directory in self.required_dir:
            complete_directory = os.sep.join([self.output_dir, directory])
            if os.path.isdir(complete_directory) is False:
                os.mkdir(complete_directory)

        # Copy css/js necessary files
        for filename in config.css_list:
            target = os.sep.join([self.output_dir, 'css'])
            if os.path.isfile(target) is False:
                shutil.copy(filename, target)

        for filename in config.js_list:
            target = os.sep.join([self.output_dir, 'js'])
            if os.path.isfile(target) is False:
                shutil.copy(filename, target)

    def create_html(self, output_filename):
        """ Create HTML file with Jinja2.

        :param str output_filename: HTML output filename
        """
        if output_filename is None:
            return
        report_output = self.template.render(config=config, module=self)
        with open(os.sep.join([config.output_dir,output_filename]), "w") as fp:
            print(report_output, file=fp)

    def create_link(self, name, target, newtab=True, download=False):
        """ Create an HTML hyperlink with name and target.

        :param str target: the target url.
        :param bool newtab: open html page in a new tab.
        :param bool download: download the target.

        Return as string the HTML hyperlink to the target.
        """
        link = '{1}'
        return link.format(target, name)

    def create_hide_section(self, html_id, name, content, hide=False):
        """ Create an hideable section.

        :param str html_id: add short id to connect all elements.
        :param str name: name of the hyperlink to hide or display the content.
        :param str content: hideable HTML content.
        :param bool hide: set if the first state is hiding or not.

        Return tuple that contains HTML hyperlink and hideable section.
        """
        link = "{1}".format(html_id,
                                                                  name)
        content = "
\n{1}\n
".format(html_id,
                                                                    content)
        hidden = ''
        if hide:
            hidden = '\n$(".slidingDiv{0}").hide();'.format(html_id)
        js = """

        """.format(html_id, hidden)
        content = js + content
        return link, content

    def copy_file(self, filename, target_dir):
        """ Copy a file to a target directory in report dir. Return the
        relative path of your file.

        :param str filename: file to copy.
        :param str target_dir: directory where to copy.

        Return relative path of the new file location.
        """
        directory = config.output_dir + os.sep + target_dir
        try:
            os.makedirs(directory)
        except FileExistsError:
            if os.path.isdir(directory):
                pass
            else:
                msg = "{0} exist and it is not a directory".format(directory)
                logger.error(msg)
                raise FileExistsError
        try:
            shutil.copy(filename, directory)
        except FileNotFoundError:
            msg = "{0} doesn't exist".format(filename)
            raise FileNotFoundError (msg)
        return target_dir + os.sep + os.path.basename(filename)

    def add_float_right(self, content):
        """ Align a content to right.
        """
        return '
{0}
'.format(content)

    def add_code_section(self, content, language):
        """ Add code in your html.
        """
        html = ('

{1}'
                '

')
        return html.format(language, content)

    # will be removed in v > 0.5.2
    def __dataframe_to_html_table(self, dataframe, **kwargs):
        """ Convert dataframe in html.
        """
        html = HTMLTable(dataframe)
        return html.to_html(**kwargs)

    def include_svg_image(self, filename):
        """ Include SVG image in the html.
        """
        html = ('<object data="{0}" type="image/svg+xml">\n'
                '</object>')
        return html.format(filename)

    def png_to_embedded_png(self, png, style=None, alt=""):
        """ Include a PNG file as embedded file.
        """
        import base64
        with open(png, 'rb') as fp:
            png = base64.b64encode(fp.read()).decode()
        if style:
            html = '<img style="{0}" alt="{1}"'.format(style, alt)
        else:
            html = "'.format(html, png)

    def create_embedded_png(self, plot_function, input_arg, style=None,
                            **kwargs):
        """ Take as a plot function as input and create a html embedded png
        image. You must set the arguments name for the output to connect
        buffer.
        """
        buf = io.BytesIO()
        # add buffer as output of the plot function
        kwargs = dict({input_arg: buf}, **kwargs)
        try:
            plot_function(**kwargs)
            html = ''.format(
                base64.b64encode(buf.getvalue()).decode('utf-8'))
            buf.close()
        except:
            html = "image not created"
        return html

    def create_combobox(self, path_list, html_id, newtab=True):
        """ Create a dropdown menu with QueryJS.

        :param list path_list: list of links.

        return html div and js script as string.
        """
        option_list = ("
{0}
\n".format(self.create_link(
                       os.path.basename(path), path, newtab))
                       for path in path_list)
        html = """

        {0}

Subchromosome
        """.format('\n'.join(option_list), html_id)
        return html
<commit_msg>Fix regression bug<commit_after># coding: utf-8
#
#  This file is part of Sequana software
#
#  Copyright (c) 2016 - Sequana Development Team
#
#  File author(s):
#      Dimitri Desvillechabrol <dimitri.desvillechabrol@pasteur.fr>,
#          <d.desvillechabrol@gmail.com>
#
#  Distributed under the terms of the 3-clause BSD license.
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################
"""Generic module is the parent module of all other module"""
import os
import shutil
import io
import base64

from jinja2 import Environment, PackageLoader

#from reports import HTMLTable
from sequana.utils import config
from sequana import logger

__all__ = ["SequanaBaseModule"]

class SequanaBaseModule(object):
    """ Generic Module to write HTML reports.
    """
    def __init__(self, template_fn='standard.html', required_dir=None):
        if required_dir is None:
            self.required_dir=("css", "js", "images")
        else:
            self.required_dir = required_dir

        self.output_dir = config.output_dir
        self.path = "./"
        # Initiate jinja template
        env = Environment(
            loader=PackageLoader('sequana', 'resources/templates/')
        )
        self.template = env.get_template(template_fn)
        self._init_report()

    def _init_report(self):
        """ Create the report directory. All necessary directories are copied
        in working directory.
        """
        # Be aware of #465 issue. We need to check that the target file is
        # valid, in which case there is no need to copy the files.

        # Create report directory
        if os.path.isdir(config.output_dir) is False:
            os.mkdir(self.output_dir)

        for directory in self.required_dir:
            complete_directory = os.sep.join([self.output_dir, directory])
            if os.path.isdir(complete_directory) is False:
                os.mkdir(complete_directory)

        # Copy css/js necessary files
        for filename in config.css_list:
            target = os.sep.join([self.output_dir, 'css'])
            if os.path.isfile(target) is False:
                shutil.copy(filename, target)

        for filename in config.js_list:
            target = os.sep.join([self.output_dir, 'js'])
            if os.path.isfile(target) is False:
                shutil.copy(filename, target)

    def create_html(self, output_filename):
        """ Create HTML file with Jinja2.

        :param str output_filename: HTML output filename
        """
        if output_filename is None:
            return
        report_output = self.template.render(config=config, module=self)
        with open(os.sep.join([config.output_dir,output_filename]), "w") as fp:
            print(report_output, file=fp)

    def create_link(self, name, target, newtab=True, download=False):
        """ Create an HTML hyperlink with name and target.

        :param str target: the target url.
        :param bool newtab: open html page in a new tab.
        :param bool download: download the target.

        Return as string the HTML hyperlink to the target.
        """
        link = '{1}'
        return link.format(target, name)

    def create_hide_section(self, html_id, name, content, hide=False):
        """ Create an hideable section.

        :param str html_id: add short id to connect all elements.
        :param str name: name of the hyperlink to hide or display the content.
        :param str content: hideable HTML content.
        :param bool hide: set if the first state is hiding or not.

        Return tuple that contains HTML hyperlink and hideable section.
        """
        link = "{1}".format(html_id,
                                                                  name)
        content = "
\n{1}\n
".format(html_id,
                                                                    content)
        hidden = ''
        if hide:
            hidden = '\n$(".slidingDiv{0}").hide();'.format(html_id)
        js = """

        """.format(html_id, hidden)
        content = js + content
        return link, content

    def copy_file(self, filename, target_dir):
        """ Copy a file to a target directory in report dir. Return the
        relative path of your file.

        :param str filename: file to copy.
        :param str target_dir: directory where to copy.

        Return relative path of the new file location.
        """
        directory = config.output_dir + os.sep + target_dir
        try:
            os.makedirs(directory)
        except FileExistsError:
            if os.path.isdir(directory):
                pass
            else:
                msg = "{0} exist and it is not a directory".format(directory)
                logger.error(msg)
                raise FileExistsError
        try:
            shutil.copy(filename, directory)
        except FileNotFoundError:
            msg = "{0} doesn't exist".format(filename)
            raise FileNotFoundError (msg)
        return target_dir + os.sep + os.path.basename(filename)

    def add_float_right(self, content):
        """ Align a content to right.
        """
        return '
{0}
'.format(content)

    def add_code_section(self, content, language):
        """ Add code in your html.
        """
        html = ('

{1}'
                '

')
        return html.format(language, content)

    # will be removed in v > 0.5.2
    def __dataframe_to_html_table(self, dataframe, **kwargs):
        """ Convert dataframe in html.
        """
        html = HTMLTable(dataframe)
        return html.to_html(**kwargs)

    def include_svg_image(self, filename):
        """ Include SVG image in the html.
        """
        html = ('<object data="{0}" type="image/svg+xml">\n'
                '</object>')
        return html.format(filename)

    def png_to_embedded_png(self, png, style=None, alt=""):
        """ Include a PNG file as embedded file.
        """
        import base64
        with open(png, 'rb') as fp:
            png = base64.b64encode(fp.read()).decode()
        if style:
            html = '<img style="{0}" alt="{1}"'.format(style, alt)
        else:
            html = ''.format(html, png)

    def create_embedded_png(self, plot_function, input_arg, style=None,
                            **kwargs):
        """ Take as a plot function as input and create a html embedded png
        image. You must set the arguments name for the output to connect
        buffer.
        """
        buf = io.BytesIO()
        # add buffer as output of the plot function
        kwargs = dict({input_arg: buf}, **kwargs)
        try:
            plot_function(**kwargs)
            html = ''.format(
                base64.b64encode(buf.getvalue()).decode('utf-8'))
            buf.close()
        except:
            html = "image not created"
        return html

    def create_combobox(self, path_list, html_id, newtab=True):
        """ Create a dropdown menu with QueryJS.

        :param list path_list: list of links.

        return html div and js script as string.
        """
        option_list = ("
{0}
\n".format(self.create_link(
                       os.path.basename(path), path, newtab))
                       for path in path_list)
        html = """

        {0}

Subchromosome
        """.format('\n'.join(option_list), html_id)
        return html
<|endoftext|>


<commit_before>#!/usr/bin/env python
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:

"""
A scripting wrapper around 4D registration (SpaceTimeRealign)
"""

import os
import os.path as op
import numpy as np
import numpy.linalg as npl

import nibabel as nib
import nibabel.eulerangles as euler
from nibabel.optpkg import optional_package
matplotlib, HAVE_MPL, _ = optional_package('matplotlib')
if HAVE_MPL:
    import matplotlib.pyplot as plt

from .groupwise_registration import SpaceTimeRealign
import nipy.externals.argparse as argparse
import nipy.algorithms.slicetiming as st
timefuncs = st.timefuncs.SLICETIME_FUNCTIONS

__all__ = ["space_time_realign", "aff2euler"]

def aff2euler(affine):
    """
    Compute Euler angles from 4 x 4 `affine`

    Parameters
    ----------
    affine : 4 by 4 array
        An affine transformation matrix

    Returns
    -------
    The Euler angles associated with the affine
    """
    return euler.mat2euler(aff2rot_zooms(affine)[0])

def aff2rot_zooms(affine):
    """
    Compute a rotation matrix and zooms from 4 x 4 `affine`

    Parameters
    ----------
    affine : 4 by 4 array
        An affine transformation matrix

    Returns
    -------
    R: 3 by 3 array
       A rotation matrix in 3D

    zooms: length 3 1-d array
         Vector with voxel sizes.
    """
    RZS = affine[:3, :3]
    zooms = np.sqrt(np.sum(RZS * RZS, axis=0))
    RS = RZS / zooms
    # Adjust zooms to make RS correspond (below) to a true
    # rotation matrix.
    if npl.det(RS) < 0:
        zooms[0] *= -1
        RS[:,0] *= -1
    # retrieve rotation matrix from RS with polar decomposition.
    # Discard shears
    P, S, Qs = npl.svd(RS)
    R = np.dot(P, Qs)
    return R, zooms

def space_time_realign(input, tr, slice_order='descending', slice_dim=2,
                       slice_dir=1, apply=True, make_figure=False,
                       out_name=None):
    """
    This is a scripting interface to `nipy.algorithms.registration.SpaceTimeRealign`

    Parameters
    ----------
    input : str or list
        A full path to a file-name (4D nifti time-series) , or to a directory
        containing 4D nifti time-series, or a list of full-paths to files.
    tr : float
        The repetition time
    slice_order : str (optional)
        This is the order of slice-times in the acquisition. This is used as a
        key into the ``SLICETIME_FUNCTIONS`` dictionary from
        :mod:`nipy.algorithms.slicetiming.timefuncs`. Default: 'descending'.
    slice_dim : int (optional)
        Denotes the axis in `images` that is the slice axis.  In a 4D image,
        this will often be axis = 2 (default).
    slice_dir : int (optional)
        1 if the slices were acquired slice 0 first (default), slice -1 last,
        or -1 if acquire slice -1 first, slice 0 last.
    apply : bool (optional)
        Whether to apply the transformation and produce an output. Default:
        True.
    make_figure : bool (optional)
        Whether to generate a .png figure with the parameters across scans.
    out_name : bool (optional)
        Specify an output location (full path) for the files that are
        generated. Default: generate files in the path of the inputs (with an
        `_mc` suffix added to the file-names.

    Returns
    -------
    transforms : ndarray
        An (n_times_points,) shaped array containing
       `nipy.algorithms.registration.affine.Rigid` class instances for each time
        point in the time-series. These can be used as affine transforms by
        referring to their `.as_affine` attribute.
    """
    if make_figure and ~HAVE_MPL:
        e_s = "You need to have matplotlib installed to run this function with"
        e_s += " `make_figure` set to `True`"
        raise RuntimeError(e_s)

    # If we got only a single file, we motion correct that one:
    if op.isfile(input):
        if not (input.endswith('.nii') or input.endswith('.nii.gz')):
            e_s = "Input needs to be a nifti file ('.nii' or '.nii.gz'"
            raise ValueError(e_s)
        fnames = [input]
        input = nib.load(input)
    # If this is a full-path to a directory containing files, it's still a
    # string:
    elif isinstance(input, str):
        list_of_files = os.listdir(input)
        fnames = [op.join(input, f) for f in np.sort(list_of_files)
                  if (f.endswith('.nii') or f.endswith('.nii.gz')) ]
        input = [nib.load(x) for x in fnames]
    # Assume that it's a list of full-paths to files:
    else:
       input = [nib.load(x) for x in input]

    slice_times = timefuncs[slice_order]
    slice_info = [slice_dim,
                  slice_dir]

    reggy = SpaceTimeRealign(input,
                             tr,
                             slice_times,
                             slice_info)

    reggy.estimate(align_runs=True)

    # We now have the transformation parameters in here:
    transforms = np.squeeze(np.array(reggy._transforms))
    rot = np.array([t.rotation for t in transforms])
    trans = np.array([t.translation for t in transforms])

    if apply:
        new_reggy = reggy.resample(align_runs=True)
        for run_idx, new_im in enumerate(new_reggy):
            new_data = new_im.get_data()
            # We use the top 4 by 4 as the affine for the new file we will
            # create:
            new_aff = new_im.affine[:4, :4]
            new_ni = nib.Nifti1Image(new_data, new_aff)
            # Save it out to a '.nii.gz' file:
            old_fname_split = op.split(fnames[run_idx][0])
            # We retain the file-name adding '_mc' regardless of where it's saved
            new_fname = old_fname_split[1].split('.')[0] + '_mc.nii.gz'
            if out_name is None:
                new_path = old_fname_split[0]
            else:
                new_path = out_name
            new_ni.to_filename(op.join(new_path, new_fname))

    if make_figure:
        figure, ax = plt.subplots(2)
        figure.set_size_inches([8, 6])
        ax[0].plot(rot)
        ax[0].set_xlabel('Time (TR)')
        ax[0].set_ylabel('Translation (mm)')
        ax[1].plot(trans)
        ax[1].set_xlabel('Time (TR)')
        ax[1].set_ylabel('Rotation (radians)')
        figure.savefig(op.join(os.path.split(fnames[0])[0],
                                    'mc_params.png'))

    return transforms
<commit_msg>BF: Resolve error handling logic.<commit_after>#!/usr/bin/env python
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:

"""
A scripting wrapper around 4D registration (SpaceTimeRealign)
"""

import os
import os.path as op
import numpy as np
import numpy.linalg as npl

import nibabel as nib
import nibabel.eulerangles as euler
from nibabel.optpkg import optional_package
matplotlib, HAVE_MPL, _ = optional_package('matplotlib')

if HAVE_MPL:
    import matplotlib.pyplot as plt

from .groupwise_registration import SpaceTimeRealign
import nipy.externals.argparse as argparse
import nipy.algorithms.slicetiming as st
timefuncs = st.timefuncs.SLICETIME_FUNCTIONS

__all__ = ["space_time_realign", "aff2euler"]

def aff2euler(affine):
    """
    Compute Euler angles from 4 x 4 `affine`

    Parameters
    ----------
    affine : 4 by 4 array
        An affine transformation matrix

    Returns
    -------
    The Euler angles associated with the affine
    """
    return euler.mat2euler(aff2rot_zooms(affine)[0])

def aff2rot_zooms(affine):
    """
    Compute a rotation matrix and zooms from 4 x 4 `affine`

    Parameters
    ----------
    affine : 4 by 4 array
        An affine transformation matrix

    Returns
    -------
    R: 3 by 3 array
       A rotation matrix in 3D

    zooms: length 3 1-d array
         Vector with voxel sizes.
    """
    RZS = affine[:3, :3]
    zooms = np.sqrt(np.sum(RZS * RZS, axis=0))
    RS = RZS / zooms
    # Adjust zooms to make RS correspond (below) to a true
    # rotation matrix.
    if npl.det(RS) < 0:
        zooms[0] *= -1
        RS[:,0] *= -1
    # retrieve rotation matrix from RS with polar decomposition.
    # Discard shears
    P, S, Qs = npl.svd(RS)
    R = np.dot(P, Qs)
    return R, zooms

def space_time_realign(input, tr, slice_order='descending', slice_dim=2,
                       slice_dir=1, apply=True, make_figure=False,
                       out_name=None):
    """
    This is a scripting interface to `nipy.algorithms.registration.SpaceTimeRealign`

    Parameters
    ----------
    input : str or list
        A full path to a file-name (4D nifti time-series) , or to a directory
        containing 4D nifti time-series, or a list of full-paths to files.
    tr : float
        The repetition time
    slice_order : str (optional)
        This is the order of slice-times in the acquisition. This is used as a
        key into the ``SLICETIME_FUNCTIONS`` dictionary from
        :mod:`nipy.algorithms.slicetiming.timefuncs`. Default: 'descending'.
    slice_dim : int (optional)
        Denotes the axis in `images` that is the slice axis.  In a 4D image,
        this will often be axis = 2 (default).
    slice_dir : int (optional)
        1 if the slices were acquired slice 0 first (default), slice -1 last,
        or -1 if acquire slice -1 first, slice 0 last.
    apply : bool (optional)
        Whether to apply the transformation and produce an output. Default:
        True.
    make_figure : bool (optional)
        Whether to generate a .png figure with the parameters across scans.
    out_name : bool (optional)
        Specify an output location (full path) for the files that are
        generated. Default: generate files in the path of the inputs (with an
        `_mc` suffix added to the file-names.

    Returns
    -------
    transforms : ndarray
        An (n_times_points,) shaped array containing
       `nipy.algorithms.registration.affine.Rigid` class instances for each time
        point in the time-series. These can be used as affine transforms by
        referring to their `.as_affine` attribute.
    """
    if make_figure:
        if not HAVE_MPL:
            e_s ="You need to have matplotlib installed to run this function"
            e_s += " with `make_figure` set to `True`"
            raise RuntimeError(e_s)

    # If we got only a single file, we motion correct that one:
    if op.isfile(input):
        if not (input.endswith('.nii') or input.endswith('.nii.gz')):
            e_s = "Input needs to be a nifti file ('.nii' or '.nii.gz'"
            raise ValueError(e_s)
        fnames = [input]
        input = nib.load(input)
    # If this is a full-path to a directory containing files, it's still a
    # string:
    elif isinstance(input, str):
        list_of_files = os.listdir(input)
        fnames = [op.join(input, f) for f in np.sort(list_of_files)
                  if (f.endswith('.nii') or f.endswith('.nii.gz')) ]
        input = [nib.load(x) for x in fnames]
    # Assume that it's a list of full-paths to files:
    else:
       input = [nib.load(x) for x in input]

    slice_times = timefuncs[slice_order]
    slice_info = [slice_dim,
                  slice_dir]

    reggy = SpaceTimeRealign(input,
                             tr,
                             slice_times,
                             slice_info)

    reggy.estimate(align_runs=True)

    # We now have the transformation parameters in here:
    transforms = np.squeeze(np.array(reggy._transforms))
    rot = np.array([t.rotation for t in transforms])
    trans = np.array([t.translation for t in transforms])

    if apply:
        new_reggy = reggy.resample(align_runs=True)
        for run_idx, new_im in enumerate(new_reggy):
            new_data = new_im.get_data()
            # We use the top 4 by 4 as the affine for the new file we will
            # create:
            new_aff = new_im.affine[:4, :4]
            new_ni = nib.Nifti1Image(new_data, new_aff)
            # Save it out to a '.nii.gz' file:
            old_fname_split = op.split(fnames[run_idx][0])
            # We retain the file-name adding '_mc' regardless of where it's saved
            new_fname = old_fname_split[1].split('.')[0] + '_mc.nii.gz'
            if out_name is None:
                new_path = old_fname_split[0]
            else:
                new_path = out_name
            new_ni.to_filename(op.join(new_path, new_fname))

    if make_figure:
        figure, ax = plt.subplots(2)
        figure.set_size_inches([8, 6])
        ax[0].plot(rot)
        ax[0].set_xlabel('Time (TR)')
        ax[0].set_ylabel('Translation (mm)')
        ax[1].plot(trans)
        ax[1].set_xlabel('Time (TR)')
        ax[1].set_ylabel('Rotation (radians)')
        figure.savefig(op.join(os.path.split(fnames[0])[0],
                                    'mc_params.png'))

    return transforms
<|endoftext|>


<commit_before>import subprocess
import glob

biomass_to_c = 0.5

above_to_below_natrl_forest = 0.26
above_to_below_mangrove = 0.608

# Spreadsheet with annual gain rates
gain_spreadsheet = 'gain_rate_continent_ecozone_age_20181017.xlsx'

# Woods Hole biomass 2000 version 4 tiles
biomass_dir = 's3://gfw2-data/climate/WHRC_biomass/WHRC_V4/Processed/'

# Lola Fatoyinbo aboveground mangrove biomass tiles
pattern_mangrove_biomass = 'mangrove_agb_t_ha'
mangrove_biomass_dir = 's3://gfw2-data/climate/carbon_model/mangrove_biomass/processed/20181019/'

# Annual Hansen loss tiles (2001-2015)
loss_dir = 's3://gfw2-data/forest_change/hansen_2015/Loss_tiles/'

# Hansen gain tiles (2001-2012)
pattern_gain = 'Hansen_GFC2015_gain'
gain_dir = 's3://gfw2-data/forest_change/tree_cover_gain/gaindata_2012/'

# Tree cover density 2000 tiles
pattern_tcd = 'Hansen_GFC2014_treecover2000'
tcd_dir = 's3://gfw2-data/forest_cover/2000_treecover/'

# Intact forest landscape 2000 tiles
pattern_ifl = 'res_ifl_2000'
ifl_dir = 's3://gfw2-data/climate/carbon_model/other_emissions_inputs/ifl_2000/'

# Processed FAO ecozone shapefile
cont_ecozone_shp = 'fao_ecozones_fra_2000_continents_assigned_dissolved_FINAL_20180906.zip'

# Directory and names for the continent-ecozone tiles, raw and processed
pattern_cont_eco_raw = 'fao_ecozones_continents_raw'
pattern_cont_eco_processed = 'fao_ecozones_continents_processed'
cont_eco_zip = 's3://gfw2-data/climate/carbon_model/fao_ecozones/fao_ecozones_fra_2000_continents_assigned_dissolved_FINAL_20180906.zip'
cont_eco_raw_dir = 's3://gfw2-data/climate/carbon_model/fao_ecozones/ecozone_continent/20181002/raw/'
cont_eco_dir = 's3://gfw2-data/climate/carbon_model/fao_ecozones/ecozone_continent/20181002/processed/'

# Number of gain years for non-mangrove natural forests
pattern_gain_year_count_natrl_forest = 'gain_year_count_natural_forest'
gain_year_count_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/gain_year_count_natural_forest/20181031/'

# Number of gain years for mangroves
pattern_gain_year_count_mangrove = 'gain_year_count_mangrove'
gain_year_count_mangrove_dir = 's3://gfw2-data/climate/carbon_model/gain_year_count_mangrove/20181031/'

# Forest age category tiles
pattern_age_cat_natrl_forest = 'forest_age_category_natural_forest'
age_cat_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/forest_age_category_natural_forest/20180921/'

# Annual aboveground biomass gain rate for non-mangrove natural forests
pattern_annual_gain_AGB_natrl_forest = 'annual_gain_rate_AGB_t_ha_natural_forest'
annual_gain_AGB_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/annual_gain_rate_AGB_natural_forest/20181102/'

# Annual aboveground biomass gain rate for mangroves
pattern_annual_gain_AGB_mangrove = 'annual_gain_rate_AGB_t_ha_mangrove'
annual_gain_AGB_mangrove_dir = 's3://gfw2-data/climate/carbon_model/annual_gain_rate_AGB_mangrove/20181102/'

# Annual belowground biomass gain rate for non-mangrove natural forests
pattern_annual_gain_BGB_natrl_forest = 'annual_gain_rate_BGB_t_ha_natural_forest'
annual_gain_BGB_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/annual_gain_rate_BGB_natural_forest/20181102/'

# Annual belowground biomass gain rate for mangroves
pattern_annual_gain_BGB_mangrove = 'annual_gain_rate_BGB_t_ha_mangrove'
annual_gain_BGB_mangrove_dir = 's3://gfw2-data/climate/carbon_model/annual_gain_rate_BGB_mangrove/20181102/'

# Cumulative aboveground gain for natural forests
pattern_cumul_gain_AGC_natrl_forest = 'cumul_gain_AGC_t_ha_natural_forest_2001_15'
cumul_gain_AGC_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_AGC_natural_forest/20181104/'

# Cumulative aboveground gain for mangroves
pattern_cumul_gain_AGC_mangrove = 'cumul_gain_AGC_t_ha_mangrove_2001_15'
cumul_gain_AGC_mangrove_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_AGC_mangrove/20181102/'

# Cumulative aboveground gain for natural forests
pattern_cumul_gain_BGC_natrl_forest = 'cumul_gain_BGC_t_ha_natural_forest_2001_15'
cumul_gain_BGC_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_BGC_natural_forest/20181104/'

# Cumulative aboveground gain for mangroves
pattern_cumul_gain_BGC_mangrove = 'cumul_gain_BGC_t_ha_mangrove_2001_15'
cumul_gain_BGC_mangrove_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_BGC_mangrove/20181102/'

# Annual aboveground gain rate for all forest types
pattern_annual_gain_combo = 'annual_gain_rate_AGB_BGB_t_ha_all_forest_types'
annual_gain_combo_dir = 's3://gfw2-data/climate/carbon_model/annual_AGC_gain_rate_all_forest_types/20181105/'

# Cumulative gain for all forest types
pattern_cumul_gain_combo = 'cumul_gain_AGC_BGC_t_ha_all_forest_types_2001_15'
cumul_gain_combo_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_all_forest_types/20181105/'

def s3_folder_download(source, dest):
    cmd = ['aws', 's3', 'cp', source, dest, '--recursive']
    subprocess.check_call(cmd)

def s3_file_download(source, dest):
    cmd = ['aws', 's3', 'cp', source, dest]
    subprocess.check_call(cmd)

# Lists the tiles in a folder in s3
def tile_list(source):

    ## For an s3 folder in a bucket using AWSCLI
    # Captures the list of the files in the folder
    out = subprocess.Popen(['aws', 's3', 'ls', source], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()

    # Writes the output string to a text file for easier interpretation
    biomass_tiles = open("biomass_tiles.txt", "w")
    biomass_tiles.write(stdout)
    biomass_tiles.close()

    file_list = []

    # Iterates through the text file to get the names of the tiles and appends them to list
    with open("biomass_tiles.txt", 'r') as tile:
        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only tifs will be in the tile list
            if '.tif' in tile_name:

                # For stripping down standard tree biomass tiles to the tile id
                if '_biomass.tif' in tile_name:

                    tile_short_name = tile_name.replace('_biomass.tif', '')
                    file_list.append(tile_short_name)

                # For stripping down mangrove biomass tiles to the tile id
                if pattern_mangrove_biomass in tile_name:

                    tile_short_name = tile_name.replace('{}_'.format(pattern_mangrove_biomass), '')
                    tile_short_name = tile_short_name.replace('.tif', '')
                    file_list.append(tile_short_name)
                    file_list = file_list[0:]

    return file_list

# Gets the bounding coordinates of a tile
def coords(tile_id):
    NS = tile_id.split("_")[0][-1:]
    EW = tile_id.split("_")[1][-1:]

    if NS == 'S':
        ymax =-1*int(tile_id.split("_")[0][:2])
    else:
        ymax = int(str(tile_id.split("_")[0][:2]))

    if EW == 'W':
        xmin = -1*int(str(tile_id.split("_")[1][:3]))
    else:
        xmin = int(str(tile_id.split("_")[1][:3]))

    ymin = str(int(ymax) - 10)
    xmax = str(int(xmin) + 10)

    return ymax, xmin, ymin, xmax

# Rasterizes the shapefile within the bounding coordinates of a tile
def rasterize(in_shape, out_tif, xmin, ymin, xmax, ymax, tr=None, ot=None, gainEcoCon=None, anodata=None):
    cmd = ['gdal_rasterize', '-co', 'COMPRESS=LZW',

           # Input raster is ingested as 1024x1024 pixel tiles (rather than the default of 1 pixel wide strips
           '-co', 'TILED=YES', '-co', 'BLOCKXSIZE=1024', '-co', 'BLOCKYSIZE=1024',
           '-te', str(xmin), str(ymin), str(xmax), str(ymax),
           '-tr', tr, tr, '-ot', ot, '-a', gainEcoCon, '-a_nodata',
           anodata, in_shape, '{}.tif'.format(out_tif)]

    subprocess.check_call(cmd)

    return out_tif

# Uploads tile to specified location
def upload_final(pattern, upload_dir, tile_id):

    file = '{}_{}.tif'.format(pattern, tile_id)

    print "Uploading {}".format(file)
    cmd = ['aws', 's3', 'cp', file, upload_dir]

    try:
        subprocess.check_call(cmd)
    except:
        print "Error uploading output tile"

##### Not currently using the below functions

def wgetloss(tile_id):
    print "download hansen loss tile"
    cmd = ['wget', r'http://glad.geog.umd.edu/Potapov/GFW_2015/tiles/{}.tif'.format(tile_id)]

    subprocess.check_call(cmd)

def wget2015data(tile_id, filetype):

    outfile = '{0}_{1}_h.tif'.format(tile_id, filetype)

    website = 'https://storage.googleapis.com/earthenginepartners-hansen/GFC-2015-v1.3/Hansen_GFC-2015-v1.3_{0}_{1}.tif'.format(filetype, tile_id)
    cmd = ['wget', website, '-O', outfile]
    print cmd

    subprocess.check_call(cmd)

    return outfile

def rasterize_shapefile(xmin, ymax, xmax, ymin, shapefile, output_tif, attribute_field):
    layer = shapefile.replace(".shp", "")
    # attribute_field = 'old_100'
    cmd= ['gdal_rasterize', '-te', str(xmin), str(ymin), str(xmax), str(ymax), '-a', attribute_field, '-co', 'COMPRESS=LZW', '-tr', '.00025', '.00025', '-tap', '-a_nodata', '0', '-l', layer, shapefile, output_tif]

    subprocess.check_call(cmd)

    return output_tif

def resample_00025(input_tif, resampled_tif):
    # resample to .00025
    cmd = ['gdal_translate', input_tif, resampled_tif, '-tr', '.00025', '.00025', '-co', 'COMPRESS=LZW']
    subprocess.check_call(cmd)
<commit_msg>Trying to get annual gain rates and cumulative gain for above and below for all forest types. Testing out script.<commit_after>import subprocess
import glob

biomass_to_c = 0.5

above_to_below_natrl_forest = 0.26
above_to_below_mangrove = 0.608

# Spreadsheet with annual gain rates
gain_spreadsheet = 'gain_rate_continent_ecozone_age_20181017.xlsx'

# Woods Hole biomass 2000 version 4 tiles
biomass_dir = 's3://gfw2-data/climate/WHRC_biomass/WHRC_V4/Processed/'

# Lola Fatoyinbo aboveground mangrove biomass tiles
pattern_mangrove_biomass = 'mangrove_agb_t_ha'
mangrove_biomass_dir = 's3://gfw2-data/climate/carbon_model/mangrove_biomass/processed/20181019/'

# Annual Hansen loss tiles (2001-2015)
loss_dir = 's3://gfw2-data/forest_change/hansen_2015/Loss_tiles/'

# Hansen gain tiles (2001-2012)
pattern_gain = 'Hansen_GFC2015_gain'
gain_dir = 's3://gfw2-data/forest_change/tree_cover_gain/gaindata_2012/'

# Tree cover density 2000 tiles
pattern_tcd = 'Hansen_GFC2014_treecover2000'
tcd_dir = 's3://gfw2-data/forest_cover/2000_treecover/'

# Intact forest landscape 2000 tiles
pattern_ifl = 'res_ifl_2000'
ifl_dir = 's3://gfw2-data/climate/carbon_model/other_emissions_inputs/ifl_2000/'

# Processed FAO ecozone shapefile
cont_ecozone_shp = 'fao_ecozones_fra_2000_continents_assigned_dissolved_FINAL_20180906.zip'

# Directory and names for the continent-ecozone tiles, raw and processed
pattern_cont_eco_raw = 'fao_ecozones_continents_raw'
pattern_cont_eco_processed = 'fao_ecozones_continents_processed'
cont_eco_zip = 's3://gfw2-data/climate/carbon_model/fao_ecozones/fao_ecozones_fra_2000_continents_assigned_dissolved_FINAL_20180906.zip'
cont_eco_raw_dir = 's3://gfw2-data/climate/carbon_model/fao_ecozones/ecozone_continent/20181002/raw/'
cont_eco_dir = 's3://gfw2-data/climate/carbon_model/fao_ecozones/ecozone_continent/20181002/processed/'

# Number of gain years for non-mangrove natural forests
pattern_gain_year_count_natrl_forest = 'gain_year_count_natural_forest'
gain_year_count_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/gain_year_count_natural_forest/20181031/'

# Number of gain years for mangroves
pattern_gain_year_count_mangrove = 'gain_year_count_mangrove'
gain_year_count_mangrove_dir = 's3://gfw2-data/climate/carbon_model/gain_year_count_mangrove/20181031/'

# Forest age category tiles
pattern_age_cat_natrl_forest = 'forest_age_category_natural_forest'
age_cat_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/forest_age_category_natural_forest/20180921/'

# Annual aboveground biomass gain rate for non-mangrove natural forests
pattern_annual_gain_AGB_natrl_forest = 'annual_gain_rate_AGB_t_ha_natural_forest'
annual_gain_AGB_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/annual_gain_rate_AGB_natural_forest/20181102/'

# Annual aboveground biomass gain rate for mangroves
pattern_annual_gain_AGB_mangrove = 'annual_gain_rate_AGB_t_ha_mangrove'
annual_gain_AGB_mangrove_dir = 's3://gfw2-data/climate/carbon_model/annual_gain_rate_AGB_mangrove/20181102/'

# Annual belowground biomass gain rate for non-mangrove natural forests
pattern_annual_gain_BGB_natrl_forest = 'annual_gain_rate_BGB_t_ha_natural_forest'
annual_gain_BGB_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/annual_gain_rate_BGB_natural_forest/20181102/'

# Annual belowground biomass gain rate for mangroves
pattern_annual_gain_BGB_mangrove = 'annual_gain_rate_BGB_t_ha_mangrove'
annual_gain_BGB_mangrove_dir = 's3://gfw2-data/climate/carbon_model/annual_gain_rate_BGB_mangrove/20181102/'

# Cumulative aboveground gain for natural forests
pattern_cumul_gain_AGC_natrl_forest = 'cumul_gain_AGC_t_ha_natural_forest_2001_15'
cumul_gain_AGC_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_AGC_natural_forest/20181104/'

# Cumulative aboveground gain for mangroves
pattern_cumul_gain_AGC_mangrove = 'cumul_gain_AGC_t_ha_mangrove_2001_15'
cumul_gain_AGC_mangrove_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_AGC_mangrove/20181102/'

# Cumulative aboveground gain for natural forests
pattern_cumul_gain_BGC_natrl_forest = 'cumul_gain_BGC_t_ha_natural_forest_2001_15'
cumul_gain_BGC_natrl_forest_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_BGC_natural_forest/20181104/'

# Cumulative aboveground gain for mangroves
pattern_cumul_gain_BGC_mangrove = 'cumul_gain_BGC_t_ha_mangrove_2001_15'
cumul_gain_BGC_mangrove_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_BGC_mangrove/20181102/'

# Annual aboveground gain rate for all forest types
pattern_annual_gain_combo = 'annual_gain_rate_AGB_BGB_t_ha_all_forest_types'
annual_gain_combo_dir = 's3://gfw2-data/climate/carbon_model/annual_gain_rate_all_forest_types/20181105/'

# Cumulative gain for all forest types
pattern_cumul_gain_combo = 'cumul_gain_AGC_BGC_t_ha_all_forest_types_2001_15'
cumul_gain_combo_dir = 's3://gfw2-data/climate/carbon_model/cumulative_gain_all_forest_types/20181105/'

def s3_folder_download(source, dest):
    cmd = ['aws', 's3', 'cp', source, dest, '--recursive']
    subprocess.check_call(cmd)

def s3_file_download(source, dest):
    cmd = ['aws', 's3', 'cp', source, dest]
    subprocess.check_call(cmd)

# Lists the tiles in a folder in s3
def tile_list(source):

    ## For an s3 folder in a bucket using AWSCLI
    # Captures the list of the files in the folder
    out = subprocess.Popen(['aws', 's3', 'ls', source], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()

    # Writes the output string to a text file for easier interpretation
    biomass_tiles = open("biomass_tiles.txt", "w")
    biomass_tiles.write(stdout)
    biomass_tiles.close()

    file_list = []

    # Iterates through the text file to get the names of the tiles and appends them to list
    with open("biomass_tiles.txt", 'r') as tile:
        for line in tile:
            num = len(line.strip('\n').split(" "))
            tile_name = line.strip('\n').split(" ")[num - 1]

            # Only tifs will be in the tile list
            if '.tif' in tile_name:

                # For stripping down standard tree biomass tiles to the tile id
                if '_biomass.tif' in tile_name:

                    tile_short_name = tile_name.replace('_biomass.tif', '')
                    file_list.append(tile_short_name)

                # For stripping down mangrove biomass tiles to the tile id
                if pattern_mangrove_biomass in tile_name:

                    tile_short_name = tile_name.replace('{}_'.format(pattern_mangrove_biomass), '')
                    tile_short_name = tile_short_name.replace('.tif', '')
                    file_list.append(tile_short_name)
                    file_list = file_list[0:]

    return file_list

# Gets the bounding coordinates of a tile
def coords(tile_id):
    NS = tile_id.split("_")[0][-1:]
    EW = tile_id.split("_")[1][-1:]

    if NS == 'S':
        ymax =-1*int(tile_id.split("_")[0][:2])
    else:
        ymax = int(str(tile_id.split("_")[0][:2]))

    if EW == 'W':
        xmin = -1*int(str(tile_id.split("_")[1][:3]))
    else:
        xmin = int(str(tile_id.split("_")[1][:3]))

    ymin = str(int(ymax) - 10)
    xmax = str(int(xmin) + 10)

    return ymax, xmin, ymin, xmax

# Rasterizes the shapefile within the bounding coordinates of a tile
def rasterize(in_shape, out_tif, xmin, ymin, xmax, ymax, tr=None, ot=None, gainEcoCon=None, anodata=None):
    cmd = ['gdal_rasterize', '-co', 'COMPRESS=LZW',

           # Input raster is ingested as 1024x1024 pixel tiles (rather than the default of 1 pixel wide strips
           '-co', 'TILED=YES', '-co', 'BLOCKXSIZE=1024', '-co', 'BLOCKYSIZE=1024',
           '-te', str(xmin), str(ymin), str(xmax), str(ymax),
           '-tr', tr, tr, '-ot', ot, '-a', gainEcoCon, '-a_nodata',
           anodata, in_shape, '{}.tif'.format(out_tif)]

    subprocess.check_call(cmd)

    return out_tif

# Uploads tile to specified location
def upload_final(pattern, upload_dir, tile_id):

    file = '{}_{}.tif'.format(pattern, tile_id)

    print "Uploading {}".format(file)
    cmd = ['aws', 's3', 'cp', file, upload_dir]

    try:
        subprocess.check_call(cmd)
    except:
        print "Error uploading output tile"

##### Not currently using the below functions

def wgetloss(tile_id):
    print "download hansen loss tile"
    cmd = ['wget', r'http://glad.geog.umd.edu/Potapov/GFW_2015/tiles/{}.tif'.format(tile_id)]

    subprocess.check_call(cmd)

def wget2015data(tile_id, filetype):

    outfile = '{0}_{1}_h.tif'.format(tile_id, filetype)

    website = 'https://storage.googleapis.com/earthenginepartners-hansen/GFC-2015-v1.3/Hansen_GFC-2015-v1.3_{0}_{1}.tif'.format(filetype, tile_id)
    cmd = ['wget', website, '-O', outfile]
    print cmd

    subprocess.check_call(cmd)

    return outfile

def rasterize_shapefile(xmin, ymax, xmax, ymin, shapefile, output_tif, attribute_field):
    layer = shapefile.replace(".shp", "")
    # attribute_field = 'old_100'
    cmd= ['gdal_rasterize', '-te', str(xmin), str(ymin), str(xmax), str(ymax), '-a', attribute_field, '-co', 'COMPRESS=LZW', '-tr', '.00025', '.00025', '-tap', '-a_nodata', '0', '-l', layer, shapefile, output_tif]

    subprocess.check_call(cmd)

    return output_tif

def resample_00025(input_tif, resampled_tif):
    # resample to .00025
    cmd = ['gdal_translate', input_tif, resampled_tif, '-tr', '.00025', '.00025', '-co', 'COMPRESS=LZW']
    subprocess.check_call(cmd)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os

from datetime import datetime

from django.db import models
from django.conf import settings
from django.core.files.storage import default_storage
from django.utils.translation import ugettext_lazy as _

from imagekit.models import ImageSpec
from imagekit.processors import Anchor, ResizeToFill, ResizeToFit, Transpose
from mptt.fields import TreeForeignKey
from mptt.managers import TreeManager
from mptt.models import MPTTModel

from galeria import EXIF

DISPLAY_IMAGE_PROCESSORS = getattr(settings, 'GALERIA_DISPLAY_IMAGE_PROCESSORS', [
    Transpose(Transpose.AUTO),
    ResizeToFit(width=640, height=640)
])
THUMBNAIL_IMAGE_PROCESSORS = getattr(settings, 'GALERIA_THUMBNAIL_IMAGE_PROCESSORS', [
    Transpose(Transpose.AUTO),
    ResizeToFill(width=200, height=150, anchor=Anchor.CENTER)
])
COVER_IMAGE_PROCESSORS = getattr(settings, 'GALERIA_COVER_IMAGE_PROCESSORS', [
    Transpose(Transpose.AUTO),
    ResizeToFill(width=200, height=150, anchor=Anchor.CENTER)
])

ORDER_CHOICES = (
    ('-date_added', _('Descending by addition date')),
    ('date_added',  _('Ascending by addition date')),
    ('-date_taken', _('Descending by date taken')),
    ('date_taken',  _('Ascending by date taken')),
    ('-date_modified', _('Descending by modification date')),
    ('date_modified',  _('Ascending by modification date')),
)

class AlbumManager(TreeManager):
    use_for_related_fields = True

    def public(self):
        return super(AlbumManager, self).filter(is_public=True)

    def public_root_nodes(self):
        return super(AlbumManager, self).root_nodes().filter(is_public=True)

class Album(MPTTModel):
    title = models.CharField(_('title'), max_length=256)
    slug = models.SlugField(
        _('slug'),
        max_length=256,
        help_text=_('Automatically built from the title. A slug is a short '
                    'label generally used in URLs.'),
    )
    description = models.TextField(_('description'), blank=True)
    is_public = models.BooleanField(
        _('is public'),
        default=True,
        help_text=_('Only public albums will be displayed in the default views.')
    )
    order = models.CharField(
        _('order'),
        max_length=16,
        choices=ORDER_CHOICES,
        default='-date_added',
        help_text=_('The default order of pictures for this album.')
    )
    parent = models.ForeignKey(
        'self',
        null=True,
        blank=True,
        verbose_name=_('parent album'),
        related_name='children'
    )
    cover = models.ForeignKey(
        'galeria.Picture',
        verbose_name=_('cover'),
        related_name='cover',
        null=True,
        blank=True,
        on_delete=models.SET_NULL
    )
    date_added = models.DateTimeField(_('date added'), auto_now_add=True)

    objects = AlbumManager()

    class Meta:
        unique_together = (('slug', 'parent'),)
        verbose_name = _('album')
        verbose_name_plural = _('albums')

    def __unicode__(self):
        return unicode(self.title)

    @models.permalink
    def get_absolute_url(self):
        return ('galeria-album', None, {
            'slug' : str(self.slug)
        })

    @property
    def available_cover(self):
        if self.cover:
            return self.cover
        elif self.pictures.public():
            return self.pictures.public().order_by('?')[0]
        else:
            return None

    @property
    def ordered_pictures(self):
        return self.pictures.public().order_by(self.order)

    def get_album_tree(self):
        tree = [self.slug]
        if self.parent:
            tree = self.parent.get_album_tree() + tree
        return tree

def picture_imagefield_path(instance, filename):
    album_tree = '/'.join(instance.album.get_album_tree())
    return os.path.join('galeria', album_tree, filename)

class PictureManager(models.Manager):
    use_for_related_fields = True

    def public(self):
        return super(PictureManager, self).filter(is_public=True, album__is_public=True)

class Picture(models.Model):
    title = models.CharField(_('title'), max_length=256)
    slug = models.SlugField(
        _('slug'),
        max_length=256,
        help_text=_('Automatically built from the title. A slug is a short '
                    'label generally used in URLs.'),
    )
    date_added = models.DateTimeField(_('date added'), auto_now_add=True)
    date_modified = models.DateTimeField(_('date modified'), auto_now=True)
    date_taken = models.DateTimeField(_('date taken'), null=True, editable=False)
    original_image = models.ImageField(_('image'), upload_to=picture_imagefield_path)
    display_image = ImageSpec(
        DISPLAY_IMAGE_PROCESSORS,
        image_field='original_image',
        options={'quality': 90},
    )
    thumbnail_image = ImageSpec(
        THUMBNAIL_IMAGE_PROCESSORS,
        image_field='original_image',
        format='JPEG',
        options={'quality': 75},
    )
    cover_image = ImageSpec(
        COVER_IMAGE_PROCESSORS,
        image_field='original_image',
        format='JPEG',
        options={'quality': 75},
    )
    description = models.TextField(_('description'), blank=True)
    is_public = models.BooleanField(
        _('is public'),
        default=True,
        help_text=_('Public images will be displayed in the default views.')
    )
    album = TreeForeignKey(
        'galeria.Album',
        verbose_name=_('album'),
        related_name='pictures'
    )

    objects = PictureManager()

    class Meta:
        ordering = ('-date_added',)
        get_latest_by = 'date_added'
        verbose_name = _('picture')
        verbose_name_plural = _('pictures')

    @property
    def EXIF(self):
        image_file = default_storage.open(self.original_image.name, 'rb')
        try:
            return EXIF.process_file(image_file)
        except Exception, e:
            # TODO: log exception
            try:
                return EXIF.process_file(image_file, details=False)
            except Exception, e:
                # TODO: log exception
                return {}
        finally:
            image_file.close()

    def save(self, *args, **kwargs):
        super(Picture, self).save(*args, **kwargs)
        if not self.date_taken:
            exif_date = self.EXIF.get('EXIF DateTimeOriginal', None)
            if exif_date:
                self.date_taken = datetime.strptime(str(exif_date), '%Y:%m:%d %H:%M:%S')
                super(Picture, self).save(*args, **kwargs)

    def __unicode__(self):
        return unicode(self.title)

    @models.permalink
    def get_absolute_url(self):
        return ('galeria-picture', None, {
            'album_slug' : str(self.album.slug),
            'pk' : str(self.pk),
            'slug' : str(self.slug)
        })
<commit_msg>prevent error when bad formatted EXIF data<commit_after># -*- coding: utf-8 -*-

import os

from datetime import datetime

from django.db import models
from django.conf import settings
from django.core.files.storage import default_storage
from django.utils.translation import ugettext_lazy as _

from imagekit.models import ImageSpec
from imagekit.processors import Anchor, ResizeToFill, ResizeToFit, Transpose
from mptt.fields import TreeForeignKey
from mptt.managers import TreeManager
from mptt.models import MPTTModel

from galeria import EXIF

DISPLAY_IMAGE_PROCESSORS = getattr(settings, 'GALERIA_DISPLAY_IMAGE_PROCESSORS', [
    Transpose(Transpose.AUTO),
    ResizeToFit(width=640, height=640)
])
THUMBNAIL_IMAGE_PROCESSORS = getattr(settings, 'GALERIA_THUMBNAIL_IMAGE_PROCESSORS', [
    Transpose(Transpose.AUTO),
    ResizeToFill(width=200, height=150, anchor=Anchor.CENTER)
])
COVER_IMAGE_PROCESSORS = getattr(settings, 'GALERIA_COVER_IMAGE_PROCESSORS', [
    Transpose(Transpose.AUTO),
    ResizeToFill(width=200, height=150, anchor=Anchor.CENTER)
])

ORDER_CHOICES = (
    ('-date_added', _('Descending by addition date')),
    ('date_added',  _('Ascending by addition date')),
    ('-date_taken', _('Descending by date taken')),
    ('date_taken',  _('Ascending by date taken')),
    ('-date_modified', _('Descending by modification date')),
    ('date_modified',  _('Ascending by modification date')),
)

class AlbumManager(TreeManager):
    use_for_related_fields = True

    def public(self):
        return super(AlbumManager, self).filter(is_public=True)

    def public_root_nodes(self):
        return super(AlbumManager, self).root_nodes().filter(is_public=True)

class Album(MPTTModel):
    title = models.CharField(_('title'), max_length=256)
    slug = models.SlugField(
        _('slug'),
        max_length=256,
        help_text=_('Automatically built from the title. A slug is a short '
                    'label generally used in URLs.'),
    )
    description = models.TextField(_('description'), blank=True)
    is_public = models.BooleanField(
        _('is public'),
        default=True,
        help_text=_('Only public albums will be displayed in the default views.')
    )
    order = models.CharField(
        _('order'),
        max_length=16,
        choices=ORDER_CHOICES,
        default='-date_added',
        help_text=_('The default order of pictures for this album.')
    )
    parent = models.ForeignKey(
        'self',
        null=True,
        blank=True,
        verbose_name=_('parent album'),
        related_name='children'
    )
    cover = models.ForeignKey(
        'galeria.Picture',
        verbose_name=_('cover'),
        related_name='cover',
        null=True,
        blank=True,
        on_delete=models.SET_NULL
    )
    date_added = models.DateTimeField(_('date added'), auto_now_add=True)

    objects = AlbumManager()

    class Meta:
        unique_together = (('slug', 'parent'),)
        verbose_name = _('album')
        verbose_name_plural = _('albums')

    def __unicode__(self):
        return unicode(self.title)

    @models.permalink
    def get_absolute_url(self):
        return ('galeria-album', None, {
            'slug' : str(self.slug)
        })

    @property
    def available_cover(self):
        if self.cover:
            return self.cover
        elif self.pictures.public():
            return self.pictures.public().order_by('?')[0]
        else:
            return None

    @property
    def ordered_pictures(self):
        return self.pictures.public().order_by(self.order)

    def get_album_tree(self):
        tree = [self.slug]
        if self.parent:
            tree = self.parent.get_album_tree() + tree
        return tree

def picture_imagefield_path(instance, filename):
    album_tree = '/'.join(instance.album.get_album_tree())
    return os.path.join('galeria', album_tree, filename)

class PictureManager(models.Manager):
    use_for_related_fields = True

    def public(self):
        return super(PictureManager, self).filter(is_public=True, album__is_public=True)

class Picture(models.Model):
    title = models.CharField(_('title'), max_length=256)
    slug = models.SlugField(
        _('slug'),
        max_length=256,
        help_text=_('Automatically built from the title. A slug is a short '
                    'label generally used in URLs.'),
    )
    date_added = models.DateTimeField(_('date added'), auto_now_add=True)
    date_modified = models.DateTimeField(_('date modified'), auto_now=True)
    date_taken = models.DateTimeField(_('date taken'), null=True, editable=False)
    original_image = models.ImageField(_('image'), upload_to=picture_imagefield_path)
    display_image = ImageSpec(
        DISPLAY_IMAGE_PROCESSORS,
        image_field='original_image',
        options={'quality': 90},
    )
    thumbnail_image = ImageSpec(
        THUMBNAIL_IMAGE_PROCESSORS,
        image_field='original_image',
        format='JPEG',
        options={'quality': 75},
    )
    cover_image = ImageSpec(
        COVER_IMAGE_PROCESSORS,
        image_field='original_image',
        format='JPEG',
        options={'quality': 75},
    )
    description = models.TextField(_('description'), blank=True)
    is_public = models.BooleanField(
        _('is public'),
        default=True,
        help_text=_('Public images will be displayed in the default views.')
    )
    album = TreeForeignKey(
        'galeria.Album',
        verbose_name=_('album'),
        related_name='pictures'
    )

    objects = PictureManager()

    class Meta:
        ordering = ('-date_added',)
        get_latest_by = 'date_added'
        verbose_name = _('picture')
        verbose_name_plural = _('pictures')

    @property
    def EXIF(self):
        image_file = default_storage.open(self.original_image.name, 'rb')
        try:
            return EXIF.process_file(image_file)
        except Exception, e:
            # TODO: log exception
            try:
                return EXIF.process_file(image_file, details=False)
            except Exception, e:
                # TODO: log exception
                return {}
        finally:
            image_file.close()

    def save(self, *args, **kwargs):
        super(Picture, self).save(*args, **kwargs)
        if not self.date_taken:
            exif_date = self.EXIF.get('EXIF DateTimeOriginal', None)
            if exif_date:
                try:
                    self.date_taken = datetime.strptime(str(exif_date), '%Y:%m:%d %H:%M:%S')
                except ValueError:
                    # In case of bad formatted EXIF date...
                    pass
                super(Picture, self).save(*args, **kwargs)

    def __unicode__(self):
        return unicode(self.title)

    @models.permalink
    def get_absolute_url(self):
        return ('galeria-picture', None, {
            'album_slug' : str(self.album.slug),
            'pk' : str(self.pk),
            'slug' : str(self.slug)
        })
<|endoftext|>