<commit_before>#!/usr/bin/env python

#
# Filter a parallel corpus
#

import heapq
import logging
import math
import optparse
import random
import sys

import numpy

from defaultconfig import Config

logging.basicConfig(format = "%(asctime)-15s %(message)s")
log = logging.getLogger("filter")
log.setLevel(logging.DEBUG)

class FilterStrategy(object):
  def __init__(self,config):
    pass

  def filter(self,source,target):
    return True

class RandomFilterStrategy(FilterStrategy):
  def __init__(self,config):
    self.threshold = config.getfloat("random", "threshold", 0.1)
    random.seed()

  def filter(self, source, target):
    return random.random() < self.threshold

class ScoreFilterStrategy(FilterStrategy):
  """Filter strategy that is based on a file with sentence scores. There are three
  possible ways of specifying how to filter:
    i) threshold - filter all sentence pairs whose score is less than the threshold
    ii) proportion - filter all but a certain proportion (eg a tenth) of the sentences
    iii) count - filter all but a given count of the sentences.
    """
  def __init__(self,config):
    section = "score"
    self.score_file = config.get(section,"score_file")
    self.ignore_score = config.get(section, "ignore_score", "99999")
    option_names = ("threshold", "proportion", "count")
    options = [config.config.has_option(section,o) for o in option_names]
    if sum(options) != 1:
      raise RuntimeError("Must specify exactly one of %s for score filter" % str(option_names))
    if options[0]:
      # threshold
      self.threshold = config.getfloat(section,option_names[0])
    else:
      # proportion or count
      if options[2]:
        count = config.getint(section,option_names[2])
      else:
        # need to count entries
        count = 0
        ignore_count = 0
        for line in open(self.score_file):
          if line[:-1] != self.ignore_score:
             count = count + 1
          else:
            ignore_count = ignore_count + 1
        count = int(count * config.getfloat(section,option_names[1]))
      log.info("Retaining at least %d entries and ignoring %d" % (count, ignore_count))
      # Find the threshold
      self.threshold = sorted(\
        [float(line[:-1]) for line in open(self.score_file)], reverse=True)[ignore_count + count]
      #self.threshold = heapq.nlargest(count, \
      #  [float(line[:-1]) for line in open(self.score_file)])[-1]

    self.sfh = open(self.score_file)
    log.info("Thresholding scores at " + str(self.threshold))

  def filter(self,source,target):
    score = self.sfh.readline()
    if not score:
      raise RuntimeError("score file truncated")
    return score[:-1] == self.ignore_score  or float(score[:-1]) >= self.threshold

def main():
  parser = optparse.OptionParser(usage = "Usage: %prog [options] config-file")
  (options,args) = parser.parse_args()
  if len(args) < 1:
    parser.error("No configuration file specified")

  log.info("Loading configuration from " + args[0])
  config = Config(args[0])
  log.debug("Configuration:\n" + str(config))

  # Required general parameters
  source_lang = config.get("general", "source_language")
  target_lang = config.get("general", "target_language")
  input_stem = config.get("general", "input_stem")
  output_stem = config.get("general", "output_stem")
  strategy = config.get("general", "strategy", "")

  # Optional general parameters
  alignment_stem = config.get("general", "alignment_stem", "")
  alignment_type = config.get("general", "alignment_type", "grow-diag-final-and")
  domain_file_in = config.get("general", "domain_file", "")
  domain_file_out = config.get("general", "domain_file_out", "")

  strategy_class = globals()[strategy + "FilterStrategy"]
  strategy = strategy_class(config)

  source_input_fh = open(input_stem + "." + source_lang)
  target_input_fh = open(input_stem + "." + target_lang)
  source_output_fh = open(output_stem + "." + source_lang, "w")
  target_output_fh = open(output_stem + "." + target_lang, "w")

  alignment_input_fh = None
  alignment_output_fh = None
  if alignment_stem:
    alignment_input_fh = open(alignment_stem + "." + alignment_type)
    alignment_output_fh = open(output_stem + "." + alignment_type,"w")

  domain_boundaries = {}
  if domain_file_in:
    dfh = open(domain_file_in)
    for line in dfh:
      line_no,name = line[:-1].split()
      domain_boundaries[int(line_no)] = name

  domain_output_fh = None
  if domain_file_out:
    domain_output_fh = open(domain_file_out, "w")

  #log.info(str(domain_boundaries))

  retained = 0
  line_no = 0
  for source_line in source_input_fh:
    target_line = target_input_fh.readline()
    if alignment_input_fh:
      align_line = alignment_input_fh.readline()
    if strategy.filter(source_line,target_line):
      retained = retained + 1
      print>>source_output_fh, source_line,
      print>>target_output_fh, target_line,
      if alignment_input_fh:
        print>>alignment_output_fh, align_line,
    line_no = line_no + 1
    # check if this is a domain boundary
    if domain_boundaries and domain_boundaries.has_key(line_no):
      print>>domain_output_fh,"%d %s" % (retained,domain_boundaries[line_no])
  log.info("Lines retained: %d" % retained)

if __name__ == "__main__":
  main()
<commit_msg>delete reference to numpy. Doesn't need it<commit_after>#!/usr/bin/env python

#
# Filter a parallel corpus
#

import heapq
import logging
import math
import optparse
import random
import sys

from defaultconfig import Config

logging.basicConfig(format = "%(asctime)-15s %(message)s")
log = logging.getLogger("filter")
log.setLevel(logging.DEBUG)

class FilterStrategy(object):
  def __init__(self,config):
    pass

  def filter(self,source,target):
    return True

class RandomFilterStrategy(FilterStrategy):
  def __init__(self,config):
    self.threshold = config.getfloat("random", "threshold", 0.1)
    random.seed()

  def filter(self, source, target):
    return random.random() < self.threshold

class ScoreFilterStrategy(FilterStrategy):
  """Filter strategy that is based on a file with sentence scores. There are three
  possible ways of specifying how to filter:
    i) threshold - filter all sentence pairs whose score is less than the threshold
    ii) proportion - filter all but a certain proportion (eg a tenth) of the sentences
    iii) count - filter all but a given count of the sentences.
    """
  def __init__(self,config):
    section = "score"
    self.score_file = config.get(section,"score_file")
    self.ignore_score = config.get(section, "ignore_score", "99999")
    option_names = ("threshold", "proportion", "count")
    options = [config.config.has_option(section,o) for o in option_names]
    if sum(options) != 1:
      raise RuntimeError("Must specify exactly one of %s for score filter" % str(option_names))
    if options[0]:
      # threshold
      self.threshold = config.getfloat(section,option_names[0])
    else:
      # proportion or count
      if options[2]:
        count = config.getint(section,option_names[2])
      else:
        # need to count entries
        count = 0
        ignore_count = 0
        for line in open(self.score_file):
          if line[:-1] != self.ignore_score:
             count = count + 1
          else:
            ignore_count = ignore_count + 1
        count = int(count * config.getfloat(section,option_names[1]))
      log.info("Retaining at least %d entries and ignoring %d" % (count, ignore_count))
      # Find the threshold
      self.threshold = sorted(\
        [float(line[:-1]) for line in open(self.score_file)], reverse=True)[ignore_count + count]
      #self.threshold = heapq.nlargest(count, \
      #  [float(line[:-1]) for line in open(self.score_file)])[-1]

    self.sfh = open(self.score_file)
    log.info("Thresholding scores at " + str(self.threshold))

  def filter(self,source,target):
    score = self.sfh.readline()
    if not score:
      raise RuntimeError("score file truncated")
    return score[:-1] == self.ignore_score  or float(score[:-1]) >= self.threshold

def main():
  parser = optparse.OptionParser(usage = "Usage: %prog [options] config-file")
  (options,args) = parser.parse_args()
  if len(args) < 1:
    parser.error("No configuration file specified")

  log.info("Loading configuration from " + args[0])
  config = Config(args[0])
  log.debug("Configuration:\n" + str(config))

  # Required general parameters
  source_lang = config.get("general", "source_language")
  target_lang = config.get("general", "target_language")
  input_stem = config.get("general", "input_stem")
  output_stem = config.get("general", "output_stem")
  strategy = config.get("general", "strategy", "")

  # Optional general parameters
  alignment_stem = config.get("general", "alignment_stem", "")
  alignment_type = config.get("general", "alignment_type", "grow-diag-final-and")
  domain_file_in = config.get("general", "domain_file", "")
  domain_file_out = config.get("general", "domain_file_out", "")

  strategy_class = globals()[strategy + "FilterStrategy"]
  strategy = strategy_class(config)

  source_input_fh = open(input_stem + "." + source_lang)
  target_input_fh = open(input_stem + "." + target_lang)
  source_output_fh = open(output_stem + "." + source_lang, "w")
  target_output_fh = open(output_stem + "." + target_lang, "w")

  alignment_input_fh = None
  alignment_output_fh = None
  if alignment_stem:
    alignment_input_fh = open(alignment_stem + "." + alignment_type)
    alignment_output_fh = open(output_stem + "." + alignment_type,"w")

  domain_boundaries = {}
  if domain_file_in:
    dfh = open(domain_file_in)
    for line in dfh:
      line_no,name = line[:-1].split()
      domain_boundaries[int(line_no)] = name

  domain_output_fh = None
  if domain_file_out:
    domain_output_fh = open(domain_file_out, "w")

  #log.info(str(domain_boundaries))

  retained = 0
  line_no = 0
  for source_line in source_input_fh:
    target_line = target_input_fh.readline()
    if alignment_input_fh:
      align_line = alignment_input_fh.readline()
    if strategy.filter(source_line,target_line):
      retained = retained + 1
      print>>source_output_fh, source_line,
      print>>target_output_fh, target_line,
      if alignment_input_fh:
        print>>alignment_output_fh, align_line,
    line_no = line_no + 1
    # check if this is a domain boundary
    if domain_boundaries and domain_boundaries.has_key(line_no):
      print>>domain_output_fh,"%d %s" % (retained,domain_boundaries[line_no])
  log.info("Lines retained: %d" % retained)

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import json
import subprocess
from urllib.parse import urlencode
from urllib.request import urlopen

key_file = __file__.rsplit('/', maxsplit=1)[0] + '/metoffice_api_key'
with open(key_file) as f:
    api_key = f.read()

def fetch_uk_outlook():
    outlook_url = ('http://datapoint.metoffice.gov.uk/public/data/txt/'
                   'wxfcs/regionalforecast/json/515')
    params = {'key': api_key}

    response = urlopen(outlook_url + '?' + urlencode(params))
    payload = json.loads(response.read().decode('utf-8'))
    periods = payload['RegionalFcst']['FcstPeriods']['Period']
    return periods

def asrun(ascript):
    "Run the given AppleScript and return the standard output and error."
    osa = subprocess.Popen(['osascript', '-'],
                           stdin=subprocess.PIPE,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.DEVNULL)
    return osa.communicate(ascript)[0]

def set_frame_contents(frame_name, text):
    script = '''\
tell application "Adobe InDesign CS4"
\ttell the front document
\t\tset the contents of text frame "{frame}" to "{contents}"
\tend tell
end tell
'''
    asrun(script.format(frame=frame_name, contents=text).encode())

if __name__ == '__main__':
    outlook_text = fetch_uk_outlook()
    day2 = outlook_text[0]['Paragraph'][2]['$']
    outlook_obj = [p for p in outlook_text
                   if p['id'] == 'day3to5']
    day3to5 = outlook_obj[0]['Paragraph']['$']
    set_frame_contents('Weather-Today', day2)
    set_frame_contents('Weather-Outlook', day3to5)
<commit_msg>Improve API key reading<commit_after>#!/usr/bin/env python3

import json
import subprocess
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import urlopen

api_key = Path(__file__).with_name('metoffice_api_key').read_text().rstrip()

def fetch_uk_outlook():
    outlook_url = ('http://datapoint.metoffice.gov.uk/public/data/txt/'
                   'wxfcs/regionalforecast/json/515')
    params = {'key': api_key}

    response = urlopen(outlook_url + '?' + urlencode(params))
    payload = json.loads(response.read().decode('utf-8'))
    periods = payload['RegionalFcst']['FcstPeriods']['Period']
    return periods

def asrun(ascript):
    "Run the given AppleScript and return the standard output and error."
    osa = subprocess.Popen(['osascript', '-'],
                           stdin=subprocess.PIPE,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.DEVNULL)
    return osa.communicate(ascript)[0]

def set_frame_contents(frame_name, text):
    script = '''\
tell application "Adobe InDesign CS4"
\ttell the front document
\t\tset the contents of text frame "{frame}" to "{contents}"
\tend tell
end tell
'''
    asrun(script.format(frame=frame_name, contents=text).encode())

if __name__ == '__main__':
    outlook_text = fetch_uk_outlook()
    day2 = outlook_text[0]['Paragraph'][2]['$']
    outlook_obj = [p for p in outlook_text
                   if p['id'] == 'day3to5']
    day3to5 = outlook_obj[0]['Paragraph']['$']
    set_frame_contents('Weather-Today', day2)
    set_frame_contents('Weather-Outlook', day3to5)
<|endoftext|>


<commit_before> #!/usr/bin/env python

import os, subprocess, inspect, yaml, shutil, sys

class ModCheck (object):
    """handles checking of modification times to see if we need to
       render the file or not"""

    def __init__ (self):
        self.modData = {}

    def check (self, sourceImage, computedFilename, outputConfig):
        """returns true if the source file has a greater modification date than
           the destination file, which would mean we need to re-render that
           destination file"""

        # if we haven't already stated this file, stat this file
        if sourceImage not in self.modData:
            self.modData[sourceImage] = os.stat(sourceImage).st_mtime

        srcModTime = self.modData[sourceImage]
        try:
            destModTime = os.stat(computedFilename).st_mtime
        except Exception:
            destModTime = 0
        if srcModTime > destModTime:
            return True
        else:
            return False

def launch (command):
    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate("")
    code = proc.returncode
    if code != 0:
        print unicode(err, errors='ignore')
        raise Exception('Command failed, check output')

def illustrator (infile, outfile, outputConfig, pluginConfig):
    """Render an image with illustrator"""
    print 'Rendering', infile, 'to', outfile, 'at dpi', outputConfig['dpi'], 'with Illustrator.'

    if 'srcdpi' not in outputConfig:
        outputConfig['srcdpi'] = 72
    if 'scaling' not in outputConfig:
        dpi = float(outputConfig['dpi'])
        srcdpi = float(outputConfig['srcdpi'])
        outputConfig['scaling'] = (dpi / srcdpi) * 100.0

    command = [
        "osascript",
        pluginConfig['script_dir'] + "/illustrator-render",
        infile,
        outfile,
        str(outputConfig['scaling'])
    ]
    launch(command)

def inkscape (infile, outfile, outputConfig, pluginConfig):
    """Render an image with inkscape"""
    print 'Rendering', infile, 'to', outfile, 'at dpi', outputConfig['dpi'], 'with Inkscape.'
    command = [
        "inkscape",
        "-d",
        str(outputConfig['dpi']),
        "-e",
        outfile,
        infile
    ]
    launch(command)

def checkProps (names, pluginConfig, outputConfig):
    """Checks to see if properties names[i] in pluginConfig is in the corresponding
       array / is the corresponding string in outputConfig, if name is even in
       outputConfig.  Returns True if everything matches, false otherwise"""
    for name in names:
        if name in pluginConfig and name in outputConfig:
            if type(outputConfig[name]) != list:
                outputConfig[name] = [outputConfig[name]]
            if pluginConfig[name] not in outputConfig[name]:
                return False
    return True

# list of ti build props to check
TiBuildProps = [
    'simtype',
    'devicefamily',
    'platform',
    'deploytype',
    'command'
]

def compile (pluginConfig):
    """invoked by Titanium's build scripts, runs the compilation process"""

    # hacky crap to get directory of currently running script file
    pluginConfig['script_dir'] = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    sys.path.insert(0, pluginConfig['script_dir'])

    # try to load images.yaml in a few different places
    test = None

    try:
        test = yaml.load(open('images.yaml', 'r'))
    except Exception:
        pass

    try:
        test = yaml.load(open('../images.yaml', 'r'))
    except Exception:
        pass

    try:
        test = yaml.load(open(os.path.join(pluginConfig['script_dir'], 'images.yaml'), 'r'))
    except Exception:
        pass

    # Titanium Studio needs the absolute path to the project
    try:
        test = yaml.load(open(os.path.join(pluginConfig['project_dir'], 'images.yaml'), 'r'))
    except Exception:
        pass

    if test == None:
        raise Exception('images.yaml file not found!')

    modCheck = ModCheck()

    backends = {}
    backends['illustrator'] = illustrator
    backends['inkscape'] = inkscape

    for name, desc in test.items():
        print 'Rendering group', name

        # Load up renderizer plugins for this backend
        plugins = []
        if 'plugins' in desc:
            for plugin in desc['plugins']:
                module = __import__(plugin)
                plugins.append(module.plugin(desc))

        for outputConfig in desc['output']:

            # validation
            if 'backend' not in desc:
                raise Exception(
                    "You must specify a render backend for output number " +
                    str(desc['output'].index(outputConfig)) + " in image group "
                    + name
                )

            if desc['backend'] not in backends :
                raise Exception(
                    "Invalid backend " + outputConfig['backend'] +
                    "specified in image group " + name
                )

            groupMatchesTiProps = False

            # check properties on the group itself; if these don't match we set
            # groupMatchesTiProps to false.  This will cause the output files
            # for this group to be removed, thereby cleaning the output
            # directory of links which dont match the current build
            # configuration
            if checkProps(TiBuildProps, pluginConfig, desc):
                groupMatchesTiProps = True

            # ensure the outputConfig directory exists
            try:
                os.makedirs(os.path.join(
                    pluginConfig['project_dir'],
                    outputConfig['path']
                ))
            except os.error:
                pass

            # ensure build directory exists
            try:
                os.makedirs(os.path.join(
                    pluginConfig['project_dir'],
                    'build', 'images',
                    outputConfig['path']
                ))
            except os.error:
                pass

            # render each sourceImage
            for sourceImage in desc['images']:

                # handle `rename` and `append` properties
                if 'rename' in outputConfig:
                    basename = outputConfig['rename']
                else:
                    basename = os.path.splitext(os.path.basename(sourceImage))[0]
                    if 'append' in outputConfig:
                        basename += outputConfig['append']
                    basename += '.png'

                # compute filename and temp filename
                computedFilename = os.path.join(
                    pluginConfig['project_dir'],
                    outputConfig['path'],
                    basename
                )

                tempFilename = os.path.join(
                    pluginConfig['project_dir'],
                    'build', 'images',
                    outputConfig['path'],
                    basename
                )

                # generate path of source image
                sourceImage = os.path.join(
                    pluginConfig['project_dir'],
                    "images",
                    sourceImage
                )

                # check to see if we need to re-render the image; if so, render
                if not modCheck.check(sourceImage, tempFilename, outputConfig):
                    print sourceImage, 'not modified, not rendering'
                else:
                    # if we've gotten to this point, we're ready to render
                    renderBackend = backends[desc['backend']]

                    # loop over renderizer plugins, call the beforeRender function
                    for plugin in plugins:
                        plugin.beforeRender(sourceImage, tempFilename, outputConfig, pluginConfig)

                    renderBackend(
                        sourceImage,
                        tempFilename,
                        outputConfig,
                        pluginConfig
                    )

                    for plugin in plugins:
                        ret = plugin.afterRender(sourceImage, tempFilename, outputConfig, pluginConfig, computedFilename)
                        if ret != None:
                            computedFilename = ret

                # remove the output file; it will be replaced with a symlink to
                # the rendered image, if applicable
                try:
                    os.remove(computedFilename)
                except:
                    pass

                # replace the output file with a symlink to the rendered image
                # in build/images/, if the property check suceeds
                if checkProps(TiBuildProps, pluginConfig, outputConfig) and groupMatchesTiProps:
                    os.symlink(tempFilename, computedFilename)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--platform')
    args = parser.parse_args()
    config = {'platform': args.platform}
    # assume we were run in the project directory if we weren't run from
    # titanium
    config['project_dir'] = os.getcwd()
    compile(config)

<commit_msg>Allow prefixes for output<commit_after> #!/usr/bin/env python

import os, subprocess, inspect, yaml, shutil, sys

class ModCheck (object):
    """handles checking of modification times to see if we need to
       render the file or not"""

    def __init__ (self):
        self.modData = {}

    def check (self, sourceImage, computedFilename, outputConfig):
        """returns true if the source file has a greater modification date than
           the destination file, which would mean we need to re-render that
           destination file"""

        # if we haven't already stated this file, stat this file
        if sourceImage not in self.modData:
            self.modData[sourceImage] = os.stat(sourceImage).st_mtime

        srcModTime = self.modData[sourceImage]
        try:
            destModTime = os.stat(computedFilename).st_mtime
        except Exception:
            destModTime = 0
        if srcModTime > destModTime:
            return True
        else:
            return False

def launch (command):
    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate("")
    code = proc.returncode
    if code != 0:
        print unicode(err, errors='ignore')
        raise Exception('Command failed, check output')

def illustrator (infile, outfile, outputConfig, pluginConfig):
    """Render an image with illustrator"""
    print 'Rendering', infile, 'to', outfile, 'at dpi', outputConfig['dpi'], 'with Illustrator.'

    if 'srcdpi' not in outputConfig:
        outputConfig['srcdpi'] = 72
    if 'scaling' not in outputConfig:
        dpi = float(outputConfig['dpi'])
        srcdpi = float(outputConfig['srcdpi'])
        outputConfig['scaling'] = (dpi / srcdpi) * 100.0

    command = [
        "osascript",
        pluginConfig['script_dir'] + "/illustrator-render",
        infile,
        outfile,
        str(outputConfig['scaling'])
    ]
    launch(command)

def inkscape (infile, outfile, outputConfig, pluginConfig):
    """Render an image with inkscape"""
    print 'Rendering', infile, 'to', outfile, 'at dpi', outputConfig['dpi'], 'with Inkscape.'
    command = [
        "inkscape",
        "-d",
        str(outputConfig['dpi']),
        "-e",
        outfile,
        infile
    ]
    launch(command)

def checkProps (names, pluginConfig, outputConfig):
    """Checks to see if properties names[i] in pluginConfig is in the corresponding
       array / is the corresponding string in outputConfig, if name is even in
       outputConfig.  Returns True if everything matches, false otherwise"""
    for name in names:
        if name in pluginConfig and name in outputConfig:
            if type(outputConfig[name]) != list:
                outputConfig[name] = [outputConfig[name]]
            if pluginConfig[name] not in outputConfig[name]:
                return False
    return True

# list of ti build props to check
TiBuildProps = [
    'simtype',
    'devicefamily',
    'platform',
    'deploytype',
    'command'
]

def compile (pluginConfig):
    """invoked by Titanium's build scripts, runs the compilation process"""

    # hacky crap to get directory of currently running script file
    pluginConfig['script_dir'] = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    sys.path.insert(0, pluginConfig['script_dir'])

    # try to load images.yaml in a few different places
    test = None

    try:
        test = yaml.load(open('images.yaml', 'r'))
    except Exception:
        pass

    try:
        test = yaml.load(open('../images.yaml', 'r'))
    except Exception:
        pass

    try:
        test = yaml.load(open(os.path.join(pluginConfig['script_dir'], 'images.yaml'), 'r'))
    except Exception:
        pass

    # Titanium Studio needs the absolute path to the project
    try:
        test = yaml.load(open(os.path.join(pluginConfig['project_dir'], 'images.yaml'), 'r'))
    except Exception:
        pass

    if test == None:
        raise Exception('images.yaml file not found!')

    modCheck = ModCheck()

    backends = {}
    backends['illustrator'] = illustrator
    backends['inkscape'] = inkscape

    for name, desc in test.items():
        print 'Rendering group', name

        # Load up renderizer plugins for this backend
        plugins = []
        if 'plugins' in desc:
            for plugin in desc['plugins']:
                module = __import__(plugin)
                plugins.append(module.plugin(desc))

        for outputConfig in desc['output']:

            # validation
            if 'backend' not in desc:
                raise Exception(
                    "You must specify a render backend for output number " +
                    str(desc['output'].index(outputConfig)) + " in image group "
                    + name
                )

            if desc['backend'] not in backends :
                raise Exception(
                    "Invalid backend " + outputConfig['backend'] +
                    "specified in image group " + name
                )

            groupMatchesTiProps = False

            # check properties on the group itself; if these don't match we set
            # groupMatchesTiProps to false.  This will cause the output files
            # for this group to be removed, thereby cleaning the output
            # directory of links which dont match the current build
            # configuration
            if checkProps(TiBuildProps, pluginConfig, desc):
                groupMatchesTiProps = True

            # ensure the outputConfig directory exists
            try:
                os.makedirs(os.path.join(
                    pluginConfig['project_dir'],
                    outputConfig['path']
                ))
            except os.error:
                pass

            # ensure build directory exists
            try:
                os.makedirs(os.path.join(
                    pluginConfig['project_dir'],
                    'build', 'images',
                    outputConfig['path']
                ))
            except os.error:
                pass

            # render each sourceImage
            for sourceImage in desc['images']:

                # handle `rename` and `append` properties
                if 'rename' in outputConfig:
                    basename = outputConfig['rename']
                else:
                    basename = os.path.splitext(os.path.basename(sourceImage))[0]
                    if 'append' in outputConfig:
                        basename += outputConfig['append']
                    if 'prepend' in outputConfig:
                        basename = outputConfig['prepend'] + basename
                    basename += '.png'

                # compute filename and temp filename
                computedFilename = os.path.join(
                    pluginConfig['project_dir'],
                    outputConfig['path'],
                    basename
                )

                tempFilename = os.path.join(
                    pluginConfig['project_dir'],
                    'build', 'images',
                    outputConfig['path'],
                    basename
                )

                # generate path of source image
                sourceImage = os.path.join(
                    pluginConfig['project_dir'],
                    "images",
                    sourceImage
                )

                # check to see if we need to re-render the image; if so, render
                if not modCheck.check(sourceImage, tempFilename, outputConfig):
                    print sourceImage, 'not modified, not rendering'
                else:
                    # if we've gotten to this point, we're ready to render
                    renderBackend = backends[desc['backend']]

                    # loop over renderizer plugins, call the beforeRender function
                    for plugin in plugins:
                        plugin.beforeRender(sourceImage, tempFilename, outputConfig, pluginConfig)

                    renderBackend(
                        sourceImage,
                        tempFilename,
                        outputConfig,
                        pluginConfig
                    )

                    for plugin in plugins:
                        ret = plugin.afterRender(sourceImage, tempFilename, outputConfig, pluginConfig, computedFilename)
                        if ret != None:
                            computedFilename = ret

                # remove the output file; it will be replaced with a symlink to
                # the rendered image, if applicable
                try:
                    os.remove(computedFilename)
                except:
                    pass

                # replace the output file with a symlink to the rendered image
                # in build/images/, if the property check suceeds
                if checkProps(TiBuildProps, pluginConfig, outputConfig) and groupMatchesTiProps:
                    os.symlink(tempFilename, computedFilename)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--platform')
    args = parser.parse_args()
    config = {'platform': args.platform}
    # assume we were run in the project directory if we weren't run from
    # titanium
    config['project_dir'] = os.getcwd()
    compile(config)

<|endoftext|>


<commit_before>'''
This file contains utility functions for interactive scripts such as
dx for tab-completion, resolving naming conflicts, etc.
'''

import os
import dxpy
from dxpy.utils.resolver import *

def startswith(text):
    return (lambda string: string.startswith(text))

def escape_completion_name_str(string):
    return string.replace('\\', '\\\\\\\\').replace(' ', '\ ').replace(':', '\\\\:').replace('/', '\\\\/').replace('*', '\\\\\\\\*').replace('?', '\\\\\\\\?')

def get_folder_matches(text, delim_pos, dxproj, folderpath):
    '''
    :param text: String to be tab-completed; still in escaped form
    :type text: string
    :param delim_pos: index of last unescaped "/" in text
    :type delim_pos: int
    :param dxproj: DXProject handler to use
    :type dxproj: DXProject
    :param folderpath: Unescaped path in which to search for folder matches
    :type folderpath: string
    :returns: List of matches
    :rtype: list of strings

    Members of the returned list are guaranteed to start with *text*
    and be in escaped form for consumption by the command-line.
    '''
    try:
        folder_names = map(lambda folder_name:
                               folder_name[folder_name.rfind('/') + 1:],
                           dxproj.list_folder(folder=folderpath,
                                              only='folders')['folders'])
        return filter(startswith(text),
                      map(lambda folder_name:
                              text[:delim_pos + 1] + \
                              escape_folder_str(folder_name) + \
                              '/',
                          folder_names + ['.', '..']))
    except:
        return []

def get_data_matches(text, delim_pos, dxproj, folderpath, classname=None,
                     typespec=None):
    '''
    :param text: String to be tab-completed; still in escaped form
    :type text: string
    :param delim_pos: index of last unescaped "/" or ":" in text
    :type delim_pos: int
    :param dxproj: DXProject handler to use
    :type dxproj: DXProject
    :param folderpath: Unescaped path in which to search for data object matches
    :type folderpath: string
    :param classname: Data object class by which to restrict the search (None for no restriction on class)
    :type classname: string
    :returns: List of matches
    :rtype: list of strings

    Members of the returned list are guaranteed to start with *text*
    and be in escaped form for consumption by the command-line.
    '''

    try:
        results = list(dxpy.find_data_objects(project=dxproj.get_id(),
                                              folder=folderpath,
                                              recurse=False,
                                              visibility='either' if text != '' and delim_pos != len(text) - 1 else 'visible',
                                              classname=classname,
                                              limit=100,
                                              describe=True,
                                              typename=typespec))
        names = map(lambda result: result['describe']['name'], results)
        return filter(startswith(text),
                      map(lambda name:
                              ('' if text == '' else text[:delim_pos + 1]) + escape_completion_name_str(name),
                          names))
    except:
        return []

def path_completer(text, expected=None, classes=None, perm_level=None,
                   include_current_proj=False, typespec=None):
    '''
    :param text: String to tab-complete to a path matching the syntax project-name:folder/entity_or_folder_name
    :type text: string
    :param expected: "folder", "entity", "project", or None (no restriction) as to the types of answers to look for
    :type expected: string
    :param classes: if expected="entity", the possible data object classes that are acceptable
    :type classes: list of strings
    :param perm_level: the minimum permissions level required, e.g. "VIEW" or "CONTRIBUTE"
    :type perm_level: string
    :param include_current_proj: Indicate whether the current project's name should be a potential result
    :type include_current_proj: boolean

    Returns a list of matches to the text and restricted by the
    requested parameters.
    '''

    colon_pos = get_last_pos_of_char(':', text)
    slash_pos = get_last_pos_of_char('/', text)
    delim_pos = max(colon_pos, slash_pos)

    # First get projects if necessary
    matches = []
    if colon_pos < 0 and slash_pos < 0:
        # Might be tab-completing a project, but don't ever include
        # whatever's set as dxpy.WORKSPACE_ID unless expected == "project"
        # Also, don't bother if text=="" and expected is NOT "project"
        if text != "" or expected == 'project':
            results = filter(lambda result: result['id'] != dxpy.WORKSPACE_ID or include_current_proj,
                             list(dxpy.find_projects(describe=True, level=perm_level)))
            matches += filter(startswith(text),
                              [(escape_completion_name_str(result['describe']['name']) + ':') for result in results])

    if expected == 'project':
        return matches

    # Attempt to tab-complete to a folder or data object name
    if colon_pos < 0 and slash_pos >= 0:
        # Not tab-completing a project, and the project is unambiguous
        # (use dxpy.WORKSPACE_ID)
        if dxpy.WORKSPACE_ID is not None:
            dxproj = dxpy.get_handler(dxpy.WORKSPACE_ID)
            folderpath, entity_name = clean_folder_path(text)
            matches += get_folder_matches(text, slash_pos, dxproj, folderpath)
            if expected != 'folder':
                if classes is not None:
                    for classname in classes:
                        matches += get_data_matches(text, slash_pos, dxproj,
                                                    folderpath, classname,
                                                    typespec)
                else:
                    matches += get_data_matches(text, slash_pos, dxproj,
                                                folderpath, typespec=typespec)
    else:
        # project is ambiguous, but attempt to resolve to an object or folder
        proj_ids, folderpath, entity_name = resolve_path(text, multi_projects=True)
        for proj in proj_ids:
            dxproj = dxpy.get_handler(proj)
            matches += get_folder_matches(text, delim_pos, dxproj, folderpath)
            if expected != 'folder':
                if classes is not None:
                    for classname in classes:
                        matches += get_data_matches(text, delim_pos, dxproj,
                                                    folderpath, classname,
                                                    typespec)
                else:
                    matches += get_data_matches(text, delim_pos, dxproj,
                                                folderpath, typespec=typespec)
    return matches

class DXPathCompleter():
    '''
    This class can be used as a tab-completer with the modules
    readline and rlcompleter.  Note that to tab-complete data object
    names with spaces, the delimiters set for the completer must not
    include spaces.
    '''
    def __init__(self, expected=None, classes=None, typespec=None, include_current_proj=False):
        self.matches = []
        self.expected = expected
        self.classes = classes
        self.typespec = typespec
        self.include_current_proj = include_current_proj

    def _populate_matches(self, prefix):
        self.matches = path_completer(prefix, self.expected, self.classes,
                                      typespec=self.typespec,
                                      include_current_proj=self.include_current_proj)

    def get_matches(self, line, point, prefix, suffix):
        # This implementation is reliant on bash behavior that ':' is
        # treated as a word separator for determining prefix
        if get_last_pos_of_char(' ', line) != point - 1:
            prefix = split_unescaped(' ', line[:point])[-1]
        self.matches = path_completer(prefix, self.expected, self.classes,
                                      typespec=self.typespec,
                                      include_current_proj=self.include_current_proj)
        if get_last_pos_of_char(':', prefix) != -1:
            for i in range(len(self.matches)):
                self.matches[i] = self.matches[i][get_last_pos_of_char(':', self.matches[i]) + 1:]

        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self.matches = path_completer(text, self.expected, self.classes,
                                          typespec=self.typespec,
                                          include_current_proj=self.include_current_proj)

        if state < len(self.matches):
            return self.matches[state]
        else:
            return None

class DXAppCompleter():
    def __init__(self, installed=None):
        self.matches = []
        self.installed = installed

    def _populate_matches(self, prefix):
        appnames = [result['describe']['name'] for result in dxpy.find_apps(describe=True) if self.installed is None or (self.installed == result['describe']['installed'])]
        self.matches = [name for name in appnames if name.startswith(prefix)]
        if prefix != '':
            appnames_with_prefix = [('app-' + name) for name in appnames]
            self.matches += [name for name in appnames_with_prefix if name.startswith(prefix)]

    def get_matches(self, line, point, prefix, suffix):
        self._populate_matches(prefix)
        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self._populate_matches(text)

        if state < len(self.matches):
            return self.matches[state]
        else:
            return None

class LocalCompleter():
    def __init__(self):
        self.matches = []

    def _populate_matches(self, prefix):
        self.matches = [match for match in [path.replace(' ', '\ ') for path in os.listdir(os.getcwdu())] if match.startswith(prefix)]

    def get_matches(self, line, point, prefix, suffix):
        self._populate_matches(prefix)
        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self._populate_matches(text)

        if state < len(self.matches):
            return self.matches[state].replace(' ', '\ ')
        else:
            return None

class NoneCompleter():
    def get_matches(self, line, point, prefix, suffix):
        return []

    def __call__(self, text, state):
        return None

class ListCompleter():
    def __init__(self, completions):
        self.completions = completions
        self.matches = []

    def _populate_matches(self, prefix):
        self.matches = [ans for ans in self.completions if ans.startswith(prefix)]

    def get_matches(self, line, point, prefix, suffix):
        self._populate_matches(prefix)
        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self._populate_matches(text)

        if state < len(self.matches):
            return self.matches[state]
        else:
            return None

class MultiCompleter():
    def __init__(self, completers):
        self.completers = completers
        self.matches = []

    def _populate_matches(self, prefix):
        self.matches = []
        for completer in self.completers:
            completer._populate_matches(prefix)
            self.matches += completer.matches

    def get_matches(self, line, point, prefix, suffix):
        self._populate_matches(prefix)
        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self._populate_matches(text)

        if state < len(self.matches):
            return self.matches[state]
        else:
            return None
<commit_msg>Fixing tab completion for names with parentheses<commit_after>'''
This file contains utility functions for interactive scripts such as
dx for tab-completion, resolving naming conflicts, etc.
'''

import os
import dxpy
from dxpy.utils.resolver import *

def startswith(text):
    return (lambda string: string.startswith(text))

def escape_completion_name_str(string):
    return string.replace('\\', '\\\\\\\\').replace(' ', '\ ').replace(':', '\\\\:').replace('/', '\\\\/').replace('*', '\\\\\\\\*').replace('?', '\\\\\\\\?').replace('(', '\\(').replace(')', '\\)')

def get_folder_matches(text, delim_pos, dxproj, folderpath):
    '''
    :param text: String to be tab-completed; still in escaped form
    :type text: string
    :param delim_pos: index of last unescaped "/" in text
    :type delim_pos: int
    :param dxproj: DXProject handler to use
    :type dxproj: DXProject
    :param folderpath: Unescaped path in which to search for folder matches
    :type folderpath: string
    :returns: List of matches
    :rtype: list of strings

    Members of the returned list are guaranteed to start with *text*
    and be in escaped form for consumption by the command-line.
    '''
    try:
        folder_names = map(lambda folder_name:
                               folder_name[folder_name.rfind('/') + 1:],
                           dxproj.list_folder(folder=folderpath,
                                              only='folders')['folders'])
        return filter(startswith(text),
                      map(lambda folder_name:
                              text[:delim_pos + 1] + \
                              escape_completion_name_str(folder_name) + \
                              '/',
                          folder_names + ['.', '..']))
    except:
        return []

def get_data_matches(text, delim_pos, dxproj, folderpath, classname=None,
                     typespec=None):
    '''
    :param text: String to be tab-completed; still in escaped form
    :type text: string
    :param delim_pos: index of last unescaped "/" or ":" in text
    :type delim_pos: int
    :param dxproj: DXProject handler to use
    :type dxproj: DXProject
    :param folderpath: Unescaped path in which to search for data object matches
    :type folderpath: string
    :param classname: Data object class by which to restrict the search (None for no restriction on class)
    :type classname: string
    :returns: List of matches
    :rtype: list of strings

    Members of the returned list are guaranteed to start with *text*
    and be in escaped form for consumption by the command-line.
    '''

    try:
        results = list(dxpy.find_data_objects(project=dxproj.get_id(),
                                              folder=folderpath,
                                              recurse=False,
                                              visibility='either' if text != '' and delim_pos != len(text) - 1 else 'visible',
                                              classname=classname,
                                              limit=100,
                                              describe=True,
                                              typename=typespec))
        names = map(lambda result: result['describe']['name'], results)
        return filter(startswith(text),
                      map(lambda name:
                              ('' if text == '' else text[:delim_pos + 1]) + escape_completion_name_str(name),
                          names))
    except:
        return []

def path_completer(text, expected=None, classes=None, perm_level=None,
                   include_current_proj=False, typespec=None):
    '''
    :param text: String to tab-complete to a path matching the syntax project-name:folder/entity_or_folder_name
    :type text: string
    :param expected: "folder", "entity", "project", or None (no restriction) as to the types of answers to look for
    :type expected: string
    :param classes: if expected="entity", the possible data object classes that are acceptable
    :type classes: list of strings
    :param perm_level: the minimum permissions level required, e.g. "VIEW" or "CONTRIBUTE"
    :type perm_level: string
    :param include_current_proj: Indicate whether the current project's name should be a potential result
    :type include_current_proj: boolean

    Returns a list of matches to the text and restricted by the
    requested parameters.
    '''

    colon_pos = get_last_pos_of_char(':', text)
    slash_pos = get_last_pos_of_char('/', text)
    delim_pos = max(colon_pos, slash_pos)

    # First get projects if necessary
    matches = []
    if colon_pos < 0 and slash_pos < 0:
        # Might be tab-completing a project, but don't ever include
        # whatever's set as dxpy.WORKSPACE_ID unless expected == "project"
        # Also, don't bother if text=="" and expected is NOT "project"
        if text != "" or expected == 'project':
            results = filter(lambda result: result['id'] != dxpy.WORKSPACE_ID or include_current_proj,
                             list(dxpy.find_projects(describe=True, level=perm_level)))
            matches += filter(startswith(text),
                              [(escape_completion_name_str(result['describe']['name']) + ':') for result in results])

    if expected == 'project':
        return matches

    # Attempt to tab-complete to a folder or data object name
    if colon_pos < 0 and slash_pos >= 0:
        # Not tab-completing a project, and the project is unambiguous
        # (use dxpy.WORKSPACE_ID)
        if dxpy.WORKSPACE_ID is not None:
            dxproj = dxpy.get_handler(dxpy.WORKSPACE_ID)
            folderpath, entity_name = clean_folder_path(text)
            matches += get_folder_matches(text, slash_pos, dxproj, folderpath)
            if expected != 'folder':
                if classes is not None:
                    for classname in classes:
                        matches += get_data_matches(text, slash_pos, dxproj,
                                                    folderpath, classname,
                                                    typespec)
                else:
                    matches += get_data_matches(text, slash_pos, dxproj,
                                                folderpath, typespec=typespec)
    else:
        # project is ambiguous, but attempt to resolve to an object or folder
        proj_ids, folderpath, entity_name = resolve_path(text, multi_projects=True)
        for proj in proj_ids:
            dxproj = dxpy.get_handler(proj)
            matches += get_folder_matches(text, delim_pos, dxproj, folderpath)
            if expected != 'folder':
                if classes is not None:
                    for classname in classes:
                        matches += get_data_matches(text, delim_pos, dxproj,
                                                    folderpath, classname,
                                                    typespec)
                else:
                    matches += get_data_matches(text, delim_pos, dxproj,
                                                folderpath, typespec=typespec)
    return matches

class DXPathCompleter():
    '''
    This class can be used as a tab-completer with the modules
    readline and rlcompleter.  Note that to tab-complete data object
    names with spaces, the delimiters set for the completer must not
    include spaces.
    '''
    def __init__(self, expected=None, classes=None, typespec=None, include_current_proj=False):
        self.matches = []
        self.expected = expected
        self.classes = classes
        self.typespec = typespec
        self.include_current_proj = include_current_proj

    def _populate_matches(self, prefix):
        self.matches = path_completer(prefix, self.expected, self.classes,
                                      typespec=self.typespec,
                                      include_current_proj=self.include_current_proj)

    def get_matches(self, line, point, prefix, suffix):
        # This implementation is reliant on bash behavior that ':' is
        # treated as a word separator for determining prefix
        if get_last_pos_of_char(' ', line) != point - 1:
            prefix = split_unescaped(' ', line[:point])[-1]
        self.matches = path_completer(prefix, self.expected, self.classes,
                                      typespec=self.typespec,
                                      include_current_proj=self.include_current_proj)
        if get_last_pos_of_char(':', prefix) != -1:
            for i in range(len(self.matches)):
                self.matches[i] = self.matches[i][get_last_pos_of_char(':', self.matches[i]) + 1:]

        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self.matches = path_completer(text, self.expected, self.classes,
                                          typespec=self.typespec,
                                          include_current_proj=self.include_current_proj)

        if state < len(self.matches):
            return self.matches[state]
        else:
            return None

class DXAppCompleter():
    def __init__(self, installed=None):
        self.matches = []
        self.installed = installed

    def _populate_matches(self, prefix):
        appnames = [result['describe']['name'] for result in dxpy.find_apps(describe=True) if self.installed is None or (self.installed == result['describe']['installed'])]
        self.matches = [name for name in appnames if name.startswith(prefix)]
        if prefix != '':
            appnames_with_prefix = [('app-' + name) for name in appnames]
            self.matches += [name for name in appnames_with_prefix if name.startswith(prefix)]

    def get_matches(self, line, point, prefix, suffix):
        self._populate_matches(prefix)
        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self._populate_matches(text)

        if state < len(self.matches):
            return self.matches[state]
        else:
            return None

class LocalCompleter():
    def __init__(self):
        self.matches = []

    def _populate_matches(self, prefix):
        self.matches = [match for match in [path.replace(' ', '\ ') for path in os.listdir(os.getcwdu())] if match.startswith(prefix)]

    def get_matches(self, line, point, prefix, suffix):
        self._populate_matches(prefix)
        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self._populate_matches(text)

        if state < len(self.matches):
            return self.matches[state].replace(' ', '\ ')
        else:
            return None

class NoneCompleter():
    def get_matches(self, line, point, prefix, suffix):
        return []

    def __call__(self, text, state):
        return None

class ListCompleter():
    def __init__(self, completions):
        self.completions = completions
        self.matches = []

    def _populate_matches(self, prefix):
        self.matches = [ans for ans in self.completions if ans.startswith(prefix)]

    def get_matches(self, line, point, prefix, suffix):
        self._populate_matches(prefix)
        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self._populate_matches(text)

        if state < len(self.matches):
            return self.matches[state]
        else:
            return None

class MultiCompleter():
    def __init__(self, completers):
        self.completers = completers
        self.matches = []

    def _populate_matches(self, prefix):
        self.matches = []
        for completer in self.completers:
            completer._populate_matches(prefix)
            self.matches += completer.matches

    def get_matches(self, line, point, prefix, suffix):
        self._populate_matches(prefix)
        return self.matches

    def __call__(self, text, state):
        if state == 0:
            self._populate_matches(text)

        if state < len(self.matches):
            return self.matches[state]
        else:
            return None
<|endoftext|>


<commit_before>from flask import Flask
from flask import request
import os
import xml.etree.ElementTree as ET
from threading import Thread
import email_lib

app = Flask(__name__)

xml = None

def send_email(xml):
    print "2"
    email_lib.prueba()
    print xml
    return None

@app.route('/webhook', methods=['POST','GET'])
def webhook():
    print "webhook"
    xml = "hola"
    t = Thread(target=send_email, args=(xml,))
    t.start()
    print "acabando"
    #Jasper resend the notification unless it receives a status 200 confirming the reception
    return '',200

@app.route('/response', methods=['POST','GET'])
def response():
    print xml #Comprobar como comparto la variable.
    return "Acabamos de procesar su peticion, en breve recibira un email con los detalles"

if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))
    app.run(debug=True, port=port, host='0.0.0.0', threaded=True)
<commit_msg>Create prueba.py<commit_after>from flask import Flask
from flask import request
import os
import xml.etree.ElementTree as ET
from threading import Thread
import email_lib

app = Flask(__name__)

global xml = None

def send_email(xml):
    print "2"
    email_lib.prueba()
    print xml
    return None

@app.route('/webhook', methods=['POST','GET'])
def webhook():
    print "webhook"
    xml = "hola"
    t = Thread(target=send_email, args=(xml,))
    t.start()
    print "acabando"
    #Jasper resend the notification unless it receives a status 200 confirming the reception
    return '',200

@app.route('/response', methods=['POST','GET'])
def response():
    print xml #Comprobar como comparto la variable.
    return "Acabamos de procesar su peticion, en breve recibira un email con los detalles"

if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))
    app.run(debug=True, port=port, host='0.0.0.0', threaded=True)
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
"""
Utilities not related to neuronal network simulation.

"""
from numpy import linspace
from os import path, listdir
from importlib import import_module
import itertools

def path_to_modline(filepath):
    """Returns a string representation for module import of a file."""
    # Import the set of parameters
    pathsplit = filepath.split('/')
    modname = pathsplit[-1][:-3]
    return '.'.join(pathsplit[:-1] + [modname])

def print_dict(dictio, level=0):
    """Pretty print a dictionnary."""
    for k in dictio:
        if type(dictio[k]) == type({}):
            print ' '*2*level + str(k)
            print_dict(dictio[k], level + 1)
        else:
            print ' '*2*level + str(k) + ': ' + str(dictio[k])

def gen_parameters(template_file, params, output_dir):
    """Generate parameter sets from a template.

    template: filename with a dictionnary of parameters in it
    params: dictionnary of parameters with start, stop, step and unit
    output_dir: directory to put the created parameter sets in

    Example of params:
    params = {('Input', 'g_Ein0'): {'range': linspace(start=0, stop=1, num=4),
                                    'unit': siemens*meter**-2},
              ('Common', 'inter_conn_rate', '*', '*'): {'range': linspace(start=0.2, stop=0.8, num=2),
                                                        'unit': 1}
    }
    """
    template = get_template(template_file)

    # Map params dict key to indices
    params_map = []
    for key in params:
        params_map.append(key)

    # Build range list
    range_list = []
    for key, value in zip(params.keys(), params.values()):
        range_list.append(value['range'])

    # Iterate on the cartesian product of the ranges to create new sets
    for comb in itertools.product(*range_list):
        fname = ''

        # Set the new value for each variable of the combination
        for ind_var in xrange(len(comb)):
            var_key = params_map[ind_var]
            var_value = comb[ind_var]
            var_unit = params[var_key]['unit']

            change_dict_key(template, var_key, var_value*var_unit)

            fname += '__'.join([_.replace('*', '') for _ in var_key])
            fname += '_' + str(comb[ind_var]).replace('.', '_') + '_'
        fname += '.py'

        # Write the new parameter set to a file
        with open(path.join(output_dir, fname), 'w') as f:
            f.writelines(["from brian.stdunits import *\n",
                          "from brian.units import *\n"])
            f.write('PARAMETERS = ' + str(template) + '\n')

    # Put a __init__.py to make the modules importable
    f = open(path.join(output_dir, '__init__.py'), 'w')
    f.close()

def get_template(template_file, varname="PARAMETERS"):
    """Return the parameter variable from a template_file"""
    module = import_module(path_to_modline(template_file))
    return getattr(module, varname)

def change_dict_key(dic, path, new_value, anykey='*'):
    """Change0 *in place* the dictionnary key value to new_value.

    If key is '*', then change all dictionnary key values to new_value.
    """
    # Stop case
    if len(path) == 1:
        key = path[0]
        if key == anykey:
            for k in dic:
                dic[k] = new_value
        elif dic.has_key(key):
            dic[key] = new_value

    # Recursion
    else:
        sub = path[0]
        if sub == anykey:
            for k in dic:
                change_dict_key(dic[k], path[1:], new_value, anykey)
        else:
            change_dict_key(dic[sub], path[1:], new_value, anykey)

    return dic

def pairs(n, no_ident=True):
    """All possible pairs (i, j) with i and j < n"""
    list_pairs = []
    for i in xrange(n):
        for j in xrange(i, n):
            if not (no_ident and i == j):
                list_pairs.append((i, j))
    return list_pairs

def listdir_filter(dir, file_filter):
    """List all files in dir and apply a filter to select them"""
    files = listdir(dir)
    filtered_files = filter(file_filter, files)
    return [path.join(dir, fname) for fname in filtered_files]

def to1d(x, y, lenx):
    """1-D coordinate from 2-D coordinate"""
    return x + y*lenx

from brian import *
if __name__ == '__main__':
    d = {('Common', 'inter_conn_strength', '*', '*'):
            {'range': linspace(start=0., stop=1., num=10),
             'unit': 1},
         ('Common', 'inter_conn_rate', '*', '*'):
            {'range': linspace(start=0., stop=1, num=10),
             'unit': 1}
    }
    gen_parameters('paramsets/std_beta.py', d, 'runs/interco10x10/beta')
<commit_msg>Add a number to parameter filename (to prevent same name for same parameters)<commit_after># -*- coding:utf-8 -*-
"""
Utilities not related to neuronal network simulation.

"""
from numpy import linspace
from os import path, listdir
from importlib import import_module
import itertools

def path_to_modline(filepath):
    """Returns a string representation for module import of a file."""
    # Import the set of parameters
    pathsplit = filepath.split('/')
    modname = pathsplit[-1][:-3]
    return '.'.join(pathsplit[:-1] + [modname])

def print_dict(dictio, level=0):
    """Pretty print a dictionnary."""
    for k in dictio:
        if type(dictio[k]) == type({}):
            print ' '*2*level + str(k)
            print_dict(dictio[k], level + 1)
        else:
            print ' '*2*level + str(k) + ': ' + str(dictio[k])

def gen_parameters(template_file, params, output_dir):
    """Generate parameter sets from a template.

    template: filename with a dictionnary of parameters in it
    params: dictionnary of parameters with start, stop, step and unit
    output_dir: directory to put the created parameter sets in

    Example of params:
    params = {('Input', 'g_Ein0'): {'range': linspace(start=0, stop=1, num=4),
                                    'unit': siemens*meter**-2},
              ('Common', 'inter_conn_rate', '*', '*'): {'range': linspace(start=0.2, stop=0.8, num=2),
                                                        'unit': 1}
    }
    """
    template = get_template(template_file)

    # Map params dict key to indices
    params_map = []
    for key in params:
        params_map.append(key)

    # Build range list
    range_list = []
    for key, value in zip(params.keys(), params.values()):
        range_list.append(value['range'])

    # Iterate on the cartesian product of the ranges to create new sets
    index=0
    for comb in itertools.product(*range_list):
        index+=1
        fname = 'parset'+str(0)*(5-len(str(index)))+str(index)+"__" # assume less than 10000 parameters sets

        # Set the new value for each variable of the combination
        for ind_var in xrange(len(comb)):
            var_key = params_map[ind_var]
            var_value = comb[ind_var]
            var_unit = params[var_key]['unit']

            change_dict_key(template, var_key, var_value*var_unit)

            fname += '__'.join([_.replace('*', '') for _ in var_key])
            fname += '_' + str(comb[ind_var]).replace('.', '_') + '_'
        fname += '.py'

        # Write the new parameter set to a file
        with open(path.join(output_dir, fname), 'w') as f:
            f.writelines(["from brian.stdunits import *\n",
                          "from brian.units import *\n"])
            f.write('PARAMETERS = ' + str(template) + '\n')

    # Put a __init__.py to make the modules importable
    f = open(path.join(output_dir, '__init__.py'), 'w')
    f.close()

def get_template(template_file, varname="PARAMETERS"):
    """Return the parameter variable from a template_file"""
    module = import_module(path_to_modline(template_file))
    return getattr(module, varname)

def change_dict_key(dic, path, new_value, anykey='*'):
    """Change0 *in place* the dictionnary key value to new_value.

    If key is '*', then change all dictionnary key values to new_value.
    """
    # Stop case
    if len(path) == 1:
        key = path[0]
        if key == anykey:
            for k in dic:
                dic[k] = new_value
        elif dic.has_key(key):
            dic[key] = new_value

    # Recursion
    else:
        sub = path[0]
        if sub == anykey:
            for k in dic:
                change_dict_key(dic[k], path[1:], new_value, anykey)
        else:
            change_dict_key(dic[sub], path[1:], new_value, anykey)

    return dic

def pairs(n, no_ident=True):
    """All possible pairs (i, j) with i and j < n"""
    list_pairs = []
    for i in xrange(n):
        for j in xrange(i, n):
            if not (no_ident and i == j):
                list_pairs.append((i, j))
    return list_pairs

def listdir_filter(dir, file_filter):
    """List all files in dir and apply a filter to select them"""
    files = listdir(dir)
    filtered_files = filter(file_filter, files)
    return [path.join(dir, fname) for fname in filtered_files]

def to1d(x, y, lenx):
    """1-D coordinate from 2-D coordinate"""
    return x + y*lenx

from brian import *
if __name__ == '__main__':
    d = {('Common', 'inter_conn_strength', '*', '*'):
            {'range': linspace(start=0., stop=1., num=10),
             'unit': 1},
         ('Common', 'inter_conn_rate', '*', '*'):
            {'range': linspace(start=0., stop=1, num=10),
             'unit': 1}
    }
    gen_parameters('paramsets/std_beta.py', d, 'runs/interco10x10/beta')
<|endoftext|>


<commit_before><commit_msg>before cleanup<commit_after><|endoftext|>


<commit_before>import os
import sys
import json

from flask import (
  Flask,
  render_template,
  make_response,
  request,
  jsonify
)

from . import assets
from . import tasks

app = Flask(__name__)
assets.init(app)

def sort(iter, col, dir="ASC"):
  if dir == "ASC":
    reversed = False
  elif dir == "DESC":
    reversed = True

  return sorted(iter, key=lambda r:r[col], reverse=reversed)

def execute(q_str, col=1):
  results = tasks.count.delay(q_str).get()
  return sort(results, col=col)

def ensure_dir(path):
  dirname = os.path.dirname(path)
  if not os.path.exists(dirname):
    os.makedirs(dirname)

@app.template_filter('value')
def value(k):
  return k.split(':',1)[-1]

@app.template_filter('commas')
def commas(val):
  return "{:,d}".format(val)

@app.route('/')
def index():
  # this call should be cached
  relations = tasks.relations.delay().get()

  return render_template(
    'index.html',
    features = relations,
  )

@app.route('/firebase')
def firebase():
  return render_template(
    'firebase.html'
  )

@app.route('/schemas')
def schema():
  # todo: if it's safe todo so move this transformation
  # to tasks.py
  relations = [
    dict(id=name, name=name, fields=schema['fields'])
    for name, schema in tasks.relations.delay().get()
  ]

  return jsonify(schemas=relations)

@app.route('/query')
def query_endpoint():

  q = request.args['q']

  results = tasks.query.delay(q).get()
  response = make_response(results)
  response.headers['content-type'] = "application/json"
  return response

@app.route("/insights")
def list_insights():
  return jsonify(insights=list(insights.all()))

@app.route("/insights/<id>")
def get_insight(id):
  return jsonify(insight=insights.get(id))

from . import insights<commit_msg>Firebase route no longer needed<commit_after>import os
import sys
import json

from flask import (
  Flask,
  render_template,
  make_response,
  request,
  jsonify
)

from . import assets
from . import tasks

app = Flask(__name__)
assets.init(app)

def sort(iter, col, dir="ASC"):
  if dir == "ASC":
    reversed = False
  elif dir == "DESC":
    reversed = True

  return sorted(iter, key=lambda r:r[col], reverse=reversed)

def execute(q_str, col=1):
  results = tasks.count.delay(q_str).get()
  return sort(results, col=col)

def ensure_dir(path):
  dirname = os.path.dirname(path)
  if not os.path.exists(dirname):
    os.makedirs(dirname)

@app.template_filter('value')
def value(k):
  return k.split(':',1)[-1]

@app.template_filter('commas')
def commas(val):
  return "{:,d}".format(val)

@app.route('/')
def index():
  # this call should be cached
  relations = tasks.relations.delay().get()

  return render_template(
    'index.html',
    features = relations,
  )

@app.route('/schemas')
def schema():
  # todo: if it's safe todo so move this transformation
  # to tasks.py
  relations = [
    dict(id=name, name=name, fields=schema['fields'])
    for name, schema in tasks.relations.delay().get()
  ]

  return jsonify(schemas=relations)

@app.route('/query')
def query_endpoint():

  q = request.args['q']

  results = tasks.query.delay(q).get()
  response = make_response(results)
  response.headers['content-type'] = "application/json"
  return response

@app.route("/insights")
def list_insights():
  return jsonify(insights=list(insights.all()))

@app.route("/insights/<id>")
def get_insight(id):
  return jsonify(insight=insights.get(id))

from . import insights<|endoftext|>


<commit_before>from flask import Flask
from flask import request
import os
import xml.etree.ElementTree as ET
from threading import Thread
import email_lib

app = Flask(__name__)

def send_email(xml):
    print "2"
    email_lib.prueba()
    print xml
    return None

@app.route('/webhook', methods=['POST','GET'])
def webhook():
    print "webhook"
    global xml = "hola"
    t = Thread(target=send_email, args=(xml,))
    t.start()
    print "acabando"
    #Jasper resend the notification unless it receives a status 200 confirming the reception
    return '',200

@app.route('/response', methods=['POST','GET'])
def response():
    print xml #Comprobar como comparto la variable.
    return "Acabamos de procesar su peticion, en breve recibira un email con los detalles"

if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))
    app.run(debug=True, port=port, host='0.0.0.0', threaded=True)
<commit_msg>Create prueba.py<commit_after>from flask import Flask
from flask import request
import os
import xml.etree.ElementTree as ET
from threading import Thread
import email_lib

app = Flask(__name__)

xml = ""

def send_email(xml):
    print "2"
    email_lib.prueba()
    print xml
    return None

@app.route('/webhook', methods=['POST','GET'])
def webhook():
    print "webhook"
    xml = "hola"
    t = Thread(target=send_email, args=(xml,))
    t.start()
    print "acabando"
    #Jasper resend the notification unless it receives a status 200 confirming the reception
    return '',200

@app.route('/response', methods=['POST','GET'])
def response():
    print xml #Comprobar como comparto la variable.
    return "Acabamos de procesar su peticion, en breve recibira un email con los detalles"

if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))
    app.run(debug=True, port=port, host='0.0.0.0', threaded=True)
<|endoftext|>


<commit_before>import re

from maya import cmds, mel

from ks.maya import mcc

__also_reload__ = ['ks.maya.mcc']

def get_cache_channels(cache_path):
    if cache_path is None:
        return []
    try:
        return mcc.get_channels(cache_path)
    except mcc.ParseError as e:
        cmds.warning('Could not parse MCC for channel data; %r' % e)
        channels = cmds.cacheFile(q=True, fileName=cache_path, channelName=True)
        return [(c, None) for c in channels]

def get_point_counts(meshes):
    return [(mesh, cmds.getAttr(mesh + '.vrts', size=True)) for mesh in meshes]

def basename(name):
    return re.sub(r'[^:|]*[:|]', '', name)

def simple_name(input_name):
    """Get a simpler name for comparisons.

    We remove some common words, trailing numbers, namespaces, paths, and
    collapse all non-alphanumerics into underscores.

    """
    name = input_name.lower()
    name = basename(name)
    name = re.sub(r'(deformed|orig(inal)?s?|geo(metry)?|shapes?|spl?ines?)', '_', name)
    name = re.sub(r'[\d_]+$', '', name)
    name = re.sub(r'[^a-z0-9]', '_', name)
    name = name.strip('_')
    return name

def get_reference_namespace(reference):
    try:
        namespace = cmds.referenceQuery(reference, namespace=True, xxx=True)
    except RuntimeError:
        return
    except TypeError:
        nodes = cmds.referenceQuery(reference, nodes=True)
        if nodes:
            name = nodes[0]
            name = name.split('|', 1)[0]
            namespace = name.rsplit(':', 1)[0]
        else:
            namespace = None
    if namespace:
        return namespace.strip(':')

def get_transform(input_node):
    """Walk the heirarchy looking for the nearest parent which is a transform."""
    node = input_node
    while True:
        type_ = cmds.nodeType(node)
        if type_ == 'transform':
            return node
        relatives = cmds.listRelatives(node, parent=True)
        if not relatives:
            return None
        node = relatives[0]

def delete_cache(node):
    print '# Deleting cache:', node
    mel.eval('deleteCacheFile(3, {"keep", "%s", "geometry"})' % node)

def iter_existing_cache_connections():
    """Yield data about every existing cache connection in the scene.

    :returns: Iterator of ``(cacheFile, fileName, channel, transform, shape)``
        tuples for each cache connection.

    It is possible for ``transform`` or ``shape`` to be ``None`` when the
    connection cannot be fully resolved. In every case that the connection is
    not complete, ``shape`` will be ``None``.

    """

    cache_nodes = cmds.ls(type='cacheFile') or []
    for cache_node in cache_nodes:
        cache_path = cmds.cacheFile(cache_node, q=True, fileName=True)[0]

        ## Identify what it is connected to.

        channel = cmds.getAttr(cache_node + '.channel[0]')

        switch = cmds.listConnections(cache_node + '.outCacheData[0]')
        if not switch:
            cmds.warning('cacheFile %r is not connected' % cache_node)
            yield cache_node, cache_path, channel, None, None
            continue
        switch = switch[0]
        switch_type = cmds.nodeType(switch)

        # Pass through blends.
        if switch_type == 'cacheBlend':
            blend = switch
            switch = cmds.listConnections(blend + '.outCacheData[0]')
            if not switch:
                cmds.warning('cacheBlend %r is not connected' % blend)
                yield cache_node, cache_path, channel, None, None
                continue
            switch = switch[0]
            switch_type = cmds.nodeType(switch)

        if switch_type != 'historySwitch':
            cmds.warning('Unknown cache node layout; found %s %r' % (switch_type, switch))
            yield cache_node, cache_path, channel, None, None
            continue

        # The switch hooks onto a transform, but we want the shapes.
        transform = (cmds.listConnections(switch + '.outputGeometry[0]') or (None, ))[0]
        if transform is None:
            cmds.warning('Unknown cache node layout; nothing connected to %r' % switch)
            yield cache_node, cache_path, channel, None, None
            continue

        # Pass through groupParts.
        while True:
            transform_type = cmds.nodeType(transform)
            if transform_type == 'groupParts':
                transform = cmds.listConnections(transform + '.outputGeometry')[0]
            elif transform_type == 'transform':
                break
            else:
                cmds.warning('Unknown cache node layout; found %s %r' % (transform_type, transform))
                yield cache_node, cache_path, channel, None, None
                continue

        shapes = cmds.listRelatives(transform, children=True, shapes=True) or []

        # Maya will often add a "Deformed" copy of a mesh. Sometimes there is
        # a "Orig". Sometimes there are both.
        if len(shapes) > 1:
            a = basename(shapes[0])
            for other in shapes[1:]:
                b = basename(other)
                if not (b[:len(a)] == a and
                    b[len(a):] in ('Deformed', 'Orig')
                ):
                    break
            else:
                shapes = [shapes[0]]
        if len(shapes) != 1:
            cmds.warning('Could not identify shape connected to %r; found %r' % (cache_node, shapes))
            yield cache_node, cache_path, channel, transform, None
            continue
        shape = shapes[0]

        yield cache_node, cache_path, channel, transform, shape

def get_existing_cache_mappings():
    """Inspect the scene and determine which channels map to what.

    :return: ``dict`` mapping cache XML files to ``dict`` mapping shapes to
        channel names. E.g.::

            {
                '/path/to/cache.xml': {
                    'pSphereShape1': 'sphere_data',
                    ...
                },
                ...
            }

    Note that the returned shape is the one that the cache is connected to,
    which is often a duplicate of the original. Also that multiple caches may
    map to the same shape via a cacheBlend.

    """

    mappings = {}
    for cache_node, cache_path, channel, transform, shape in iter_existing_cache_connections():
        if shape is None:
            continue
        mapping = mappings.setdefault(cache_path, {})
        mapping[shape] = channel
    return mappings

<commit_msg>Geocache: Fix control flow for cache network error<commit_after>import re

from maya import cmds, mel

from ks.maya import mcc

__also_reload__ = ['ks.maya.mcc']

def get_cache_channels(cache_path):
    if cache_path is None:
        return []
    try:
        return mcc.get_channels(cache_path)
    except mcc.ParseError as e:
        cmds.warning('Could not parse MCC for channel data; %r' % e)
        channels = cmds.cacheFile(q=True, fileName=cache_path, channelName=True)
        return [(c, None) for c in channels]

def get_point_counts(meshes):
    return [(mesh, cmds.getAttr(mesh + '.vrts', size=True)) for mesh in meshes]

def basename(name):
    return re.sub(r'[^:|]*[:|]', '', name)

def simple_name(input_name):
    """Get a simpler name for comparisons.

    We remove some common words, trailing numbers, namespaces, paths, and
    collapse all non-alphanumerics into underscores.

    """
    name = input_name.lower()
    name = basename(name)
    name = re.sub(r'(deformed|orig(inal)?s?|geo(metry)?|shapes?|spl?ines?)', '_', name)
    name = re.sub(r'[\d_]+$', '', name)
    name = re.sub(r'[^a-z0-9]', '_', name)
    name = name.strip('_')
    return name

def get_reference_namespace(reference):
    try:
        namespace = cmds.referenceQuery(reference, namespace=True, xxx=True)
    except RuntimeError:
        return
    except TypeError:
        nodes = cmds.referenceQuery(reference, nodes=True)
        if nodes:
            name = nodes[0]
            name = name.split('|', 1)[0]
            namespace = name.rsplit(':', 1)[0]
        else:
            namespace = None
    if namespace:
        return namespace.strip(':')

def get_transform(input_node):
    """Walk the heirarchy looking for the nearest parent which is a transform."""
    node = input_node
    while True:
        type_ = cmds.nodeType(node)
        if type_ == 'transform':
            return node
        relatives = cmds.listRelatives(node, parent=True)
        if not relatives:
            return None
        node = relatives[0]

def delete_cache(node):
    print '# Deleting cache:', node
    mel.eval('deleteCacheFile(3, {"keep", "%s", "geometry"})' % node)

def iter_existing_cache_connections():
    """Yield data about every existing cache connection in the scene.

    :returns: Iterator of ``(cacheFile, fileName, channel, transform, shape)``
        tuples for each cache connection.

    It is possible for ``transform`` or ``shape`` to be ``None`` when the
    connection cannot be fully resolved. In every case that the connection is
    not complete, ``shape`` will be ``None``.

    """

    cache_nodes = cmds.ls(type='cacheFile') or []
    for cache_node in cache_nodes:
        cache_path = cmds.cacheFile(cache_node, q=True, fileName=True)[0]

        ## Identify what it is connected to.

        channel = cmds.getAttr(cache_node + '.channel[0]')

        switch = cmds.listConnections(cache_node + '.outCacheData[0]')
        if not switch:
            cmds.warning('cacheFile %r is not connected' % cache_node)
            yield cache_node, cache_path, channel, None, None
            continue
        switch = switch[0]
        switch_type = cmds.nodeType(switch)

        # Pass through blends.
        if switch_type == 'cacheBlend':
            blend = switch
            switch = cmds.listConnections(blend + '.outCacheData[0]')
            if not switch:
                cmds.warning('cacheBlend %r is not connected' % blend)
                yield cache_node, cache_path, channel, None, None
                continue
            switch = switch[0]
            switch_type = cmds.nodeType(switch)

        if switch_type != 'historySwitch':
            cmds.warning('Unknown cache node layout; expected historySwitch, found %s %r' % (switch_type, switch))
            yield cache_node, cache_path, channel, None, None
            continue

        # The switch hooks onto a transform, but we want the shapes.
        transform = (cmds.listConnections(switch + '.outputGeometry[0]') or (None, ))[0]
        if transform is None:
            cmds.warning('Unknown cache node layout; nothing connected to %r' % switch)
            yield cache_node, cache_path, channel, None, None
            continue

        # Pass through groupParts. The control flow is a little wacky here, be
        # careful.
        while transform is not None:
            transform_type = cmds.nodeType(transform)
            if transform_type == 'groupParts':
                transform = (cmds.listConnections(transform + '.outputGeometry') or (None, ))[0]
                continue
            break
        if transform is None:
            transform_type = 'None'
        if transform_type != 'transform':
            cmds.warning('Unknown cache node layout; expected transform, found %s %r' % (transform_type, transform))
            yield cache_node, cache_path, channel, None, None
            continue

        shapes = cmds.listRelatives(transform, children=True, shapes=True) or []

        # Maya will often add a "Deformed" copy of a mesh. Sometimes there is
        # a "Orig". Sometimes there are both.
        if len(shapes) > 1:
            a = basename(shapes[0])
            for other in shapes[1:]:
                b = basename(other)
                if not (b[:len(a)] == a and
                    b[len(a):] in ('Deformed', 'Orig')
                ):
                    break
            else:
                shapes = [shapes[0]]
        if len(shapes) != 1:
            cmds.warning('Could not identify shape connected to %r; found %r' % (cache_node, shapes))
            yield cache_node, cache_path, channel, transform, None
            continue
        shape = shapes[0]

        yield cache_node, cache_path, channel, transform, shape

def get_existing_cache_mappings():
    """Inspect the scene and determine which channels map to what.

    :return: ``dict`` mapping cache XML files to ``dict`` mapping shapes to
        channel names. E.g.::

            {
                '/path/to/cache.xml': {
                    'pSphereShape1': 'sphere_data',
                    ...
                },
                ...
            }

    Note that the returned shape is the one that the cache is connected to,
    which is often a duplicate of the original. Also that multiple caches may
    map to the same shape via a cacheBlend.

    """

    mappings = {}
    for cache_node, cache_path, channel, transform, shape in iter_existing_cache_connections():
        if shape is None:
            continue
        mapping = mappings.setdefault(cache_path, {})
        mapping[shape] = channel
    return mappings

<|endoftext|>


<commit_before>from elasticsearch import Elasticsearch
import datetime
import requests
import sys
from validate_sample_record import *

INDEXED_SAMPLES = dict()
ORGANISM = dict()
SPECIMEN_FROM_ORGANISM = dict()
CELL_SPECIMEN = dict()
CELL_CULTURE = dict()
CELL_LINE = dict()
POOL_SPECIMEN = dict()
TOTAL_RECORDS_TO_UPDATE = 0

def main():
    ruleset_version = get_ruleset_version()
    es = Elasticsearch(['wp-np3-e2', 'wp-np3-e3'])
    print(f"The program starts at {datetime.datetime.now()}")
    etags = get_existing_etags()
    print(f"There are {len(etags)} records with etags in ES")
    print(f"Finish retrieving existing etags at {datetime.datetime.now()}")
    print("Importing FAANG data")
    if len(etags) > 0:
        fetch_records_by_project_via_etag(etags)
    else:
        fetch_records_by_project()

def get_existing_etags():
    url_schema = 'http://wp-np3-e2.ebi.ac.uk:9200/{}/_search?_source=biosampleId,etag&sort=biosampleId&size=100000'
    results = dict()
    for item in ("organism", "specimen"):
        url = url_schema.format(item)
        response = requests.get(url).json()
        for result in response['hits']['hits']:
            if 'etag' in result['_source']:
                results[result['_source']['biosampleId']] = result['_source']['etag']
    return results

def fetch_records_by_project_via_etag(etags):
    global TOTAL_RECORDS_TO_UPDATE
    hash = dict()
    with open("etag_list_2019-02-26.txt", 'r') as f:
        for line in f:
            line = line.rstrip()
            data = line.split("\t")
            if data[0] in etags and etags[data[0]] == data[1]:
                INDEXED_SAMPLES[data[0]] = 1
                continue
            else:
                single = fetch_single_record(data[0])
                single['etag'] = data[1]
                if not check_is_faang(single):
                    continue
                material =single['characteristics']['Material'][0]['text']
                if material == 'organism':
                    ORGANISM[data[0]] = single
                elif material == 'specimen from organism':
                    SPECIMEN_FROM_ORGANISM[data[0]] = single
                elif material == 'cell specimen':
                    CELL_SPECIMEN[data[0]] = single
                elif material == 'cell culture':
                    CELL_CULTURE[data[0]] = single
                elif material == 'cell line':
                    CELL_LINE[data[0]] = single
                elif material == 'pool of specimens':
                    POOL_SPECIMEN[data[0]] = single
                hash.setdefault(material, 0)
                hash[material] += 1
    for k, v in hash.items():
        TOTAL_RECORDS_TO_UPDATE += v
        print(f"There are {v} {k} records needing update")
    if TOTAL_RECORDS_TO_UPDATE == 0:
        print("All records have not been modified since last importation.")
        print(f"Exit program at {datetime.datetime.now()}")
        sys.exit(0)
    if TOTAL_RECORDS_TO_UPDATE <=20:
        for item in ORGANISM, SPECIMEN_FROM_ORGANISM, CELL_SPECIMEN, CELL_CULTURE, CELL_LINE, POOL_SPECIMEN:
            for k in item:
                print(f"To be updated: {k}")
    print(f"The sum is {TOTAL_RECORDS_TO_UPDATE}")
    print(f"Finish comparing etags and retrieving necessary records at {datetime.datetime.now()}")

def fetch_records_by_project():
    global TOTAL_RECORDS_TO_UPDATE
    biosamples = list()
    hash = dict()
    url = 'https://www.ebi.ac.uk/biosamples/samples?size=1000&filter=attr%3Aproject%3AFAANG'
    while url:
        response = requests.get(url).json()
        if 'next' in response['_links']:
            url = response['_links']['next']['href']
            for biosample in response['_embedded']['samples']:
                biosample = deal_with_decimal_degrees(biosample)
                biosamples.append(biosample)
        else:
            url = ''
    for biosample in biosamples:
        if not check_is_faang(biosample):
            continue
        material = biosample['characteristics']['Material'][0]['text']
        if material == 'organism':
            ORGANISM[biosample['accession']] = biosample
        elif material == 'specimen from organism':
            SPECIMEN_FROM_ORGANISM[biosample['accession']] = biosample
        elif material == 'cell specimen':
            CELL_SPECIMEN[biosample['accession']] = biosample
        elif material == 'cell culture':
            CELL_CULTURE[biosample['accession']] = biosample
        elif material == 'cell line':
            CELL_LINE[biosample['accession']] = biosample
        elif material == 'pool of specimens':
            POOL_SPECIMEN[biosample['accession']] = biosample
        hash.setdefault(material, 0)
        hash[material] += 1
    for k, v in hash.items():
        TOTAL_RECORDS_TO_UPDATE += v
        print(f"There are {v} {k} records needing update")
    print(f"The sum is {TOTAL_RECORDS_TO_UPDATE}")

def fetch_single_record(biosampleId):
    url_schema = 'https://www.ebi.ac.uk/biosamples/samples/{}.json?curationdomain=self.FAANG_DCC_curation'
    url = url_schema.format(biosampleId)
    return requests.get(url).json()

def check_is_faang(item):
    if 'characteristics' in item and 'project' in item['characteristics']:
        for project in item['characteristics']['project']:
            if 'text' in project and project['text'].lower() == 'faang':
                return True
    return False

def deal_with_decimal_degrees(item):
    if item['characteristics']['Material'][0]['text'] == 'organism':
        try:
            if item['characteristics']['birth location latitude'][0]['unit'] == 'decimal degree' or \
                    item['characteristics']['birth location longitude'][0]['unit'] == 'decimal degree':
                url = "https://www.ebi.ac.uk/biosamples/samples/{}.json?curationdomain=self.FAANG_DCC_curation".format(
                    item['accession'])
                return requests.get(url).json()
        except KeyError:
            pass
    else:
        return item

if __name__ == "__main__":
    main()
<commit_msg>Adding new condition<commit_after>from elasticsearch import Elasticsearch
import datetime
import requests
import sys
from validate_sample_record import *
from get_all_etags import fetch_biosample_ids

INDEXED_SAMPLES = dict()
ORGANISM = dict()
SPECIMEN_FROM_ORGANISM = dict()
CELL_SPECIMEN = dict()
CELL_CULTURE = dict()
CELL_LINE = dict()
POOL_SPECIMEN = dict()
TOTAL_RECORDS_TO_UPDATE = 0

def main():
    ruleset_version = get_ruleset_version()
    es = Elasticsearch(['wp-np3-e2', 'wp-np3-e3'])
    print(f"The program starts at {datetime.datetime.now()}")
    etags = get_existing_etags()
    print(f"There are {len(etags)} records with etags in ES")
    print(f"Finish retrieving existing etags at {datetime.datetime.now()}")
    print("Importing FAANG data")
    biosamples = fetch_biosample_ids()
    if len(etags) == 0 or len(biosamples)/len(etags) > 2:
        fetch_records_by_project()
    else:
        fetch_records_by_project_via_etag(etags)

def get_existing_etags():
    url_schema = 'http://wp-np3-e2.ebi.ac.uk:9200/{}/_search?_source=biosampleId,etag&sort=biosampleId&size=100000'
    results = dict()
    for item in ("organism", "specimen"):
        url = url_schema.format(item)
        response = requests.get(url).json()
        for result in response['hits']['hits']:
            if 'etag' in result['_source']:
                results[result['_source']['biosampleId']] = result['_source']['etag']
    return results

def fetch_records_by_project_via_etag(etags):
    global TOTAL_RECORDS_TO_UPDATE
    hash = dict()
    with open("etag_list_2019-02-26.txt", 'r') as f:
        for line in f:
            line = line.rstrip()
            data = line.split("\t")
            if data[0] in etags and etags[data[0]] == data[1]:
                INDEXED_SAMPLES[data[0]] = 1
                continue
            else:
                single = fetch_single_record(data[0])
                single['etag'] = data[1]
                if not check_is_faang(single):
                    continue
                material =single['characteristics']['Material'][0]['text']
                if material == 'organism':
                    ORGANISM[data[0]] = single
                elif material == 'specimen from organism':
                    SPECIMEN_FROM_ORGANISM[data[0]] = single
                elif material == 'cell specimen':
                    CELL_SPECIMEN[data[0]] = single
                elif material == 'cell culture':
                    CELL_CULTURE[data[0]] = single
                elif material == 'cell line':
                    CELL_LINE[data[0]] = single
                elif material == 'pool of specimens':
                    POOL_SPECIMEN[data[0]] = single
                hash.setdefault(material, 0)
                hash[material] += 1
    for k, v in hash.items():
        TOTAL_RECORDS_TO_UPDATE += v
        print(f"There are {v} {k} records needing update")
    if TOTAL_RECORDS_TO_UPDATE == 0:
        print("All records have not been modified since last importation.")
        print(f"Exit program at {datetime.datetime.now()}")
        sys.exit(0)
    if TOTAL_RECORDS_TO_UPDATE <=20:
        for item in ORGANISM, SPECIMEN_FROM_ORGANISM, CELL_SPECIMEN, CELL_CULTURE, CELL_LINE, POOL_SPECIMEN:
            for k in item:
                print(f"To be updated: {k}")
    print(f"The sum is {TOTAL_RECORDS_TO_UPDATE}")
    print(f"Finish comparing etags and retrieving necessary records at {datetime.datetime.now()}")

def fetch_records_by_project():
    global TOTAL_RECORDS_TO_UPDATE
    biosamples = list()
    hash = dict()
    url = 'https://www.ebi.ac.uk/biosamples/samples?size=1000&filter=attr%3Aproject%3AFAANG'
    while url:
        response = requests.get(url).json()
        if 'next' in response['_links']:
            url = response['_links']['next']['href']
            for biosample in response['_embedded']['samples']:
                biosample = deal_with_decimal_degrees(biosample)
                biosamples.append(biosample)
        else:
            url = ''
    for biosample in biosamples:
        if not check_is_faang(biosample):
            continue
        material = biosample['characteristics']['Material'][0]['text']
        if material == 'organism':
            ORGANISM[biosample['accession']] = biosample
        elif material == 'specimen from organism':
            SPECIMEN_FROM_ORGANISM[biosample['accession']] = biosample
        elif material == 'cell specimen':
            CELL_SPECIMEN[biosample['accession']] = biosample
        elif material == 'cell culture':
            CELL_CULTURE[biosample['accession']] = biosample
        elif material == 'cell line':
            CELL_LINE[biosample['accession']] = biosample
        elif material == 'pool of specimens':
            POOL_SPECIMEN[biosample['accession']] = biosample
        hash.setdefault(material, 0)
        hash[material] += 1
    for k, v in hash.items():
        TOTAL_RECORDS_TO_UPDATE += v
        print(f"There are {v} {k} records needing update")
    print(f"The sum is {TOTAL_RECORDS_TO_UPDATE}")

def fetch_single_record(biosampleId):
    url_schema = 'https://www.ebi.ac.uk/biosamples/samples/{}.json?curationdomain=self.FAANG_DCC_curation'
    url = url_schema.format(biosampleId)
    return requests.get(url).json()

def check_is_faang(item):
    if 'characteristics' in item and 'project' in item['characteristics']:
        for project in item['characteristics']['project']:
            if 'text' in project and project['text'].lower() == 'faang':
                return True
    return False

def deal_with_decimal_degrees(item):
    if item['characteristics']['Material'][0]['text'] == 'organism':
        try:
            if item['characteristics']['birth location latitude'][0]['unit'] == 'decimal degree' or \
                    item['characteristics']['birth location longitude'][0]['unit'] == 'decimal degree':
                url = "https://www.ebi.ac.uk/biosamples/samples/{}.json?curationdomain=self.FAANG_DCC_curation".format(
                    item['accession'])
                return requests.get(url).json()
        except KeyError:
            pass
    else:
        return item

if __name__ == "__main__":
    main()
<|endoftext|>