<commit_before>#!/usr/bin/python
# Compresses the core Blockly files into a single JavaScript file.
#
# Copyright 2012 Google Inc.
# http://blockly.googlecode.com/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script generates two files:
#   blockly_compressed.js
#   blockly_uncompressed.js
# The compressed file is a concatenation of all of Blockly's core files which
# have been run through Google's Closure Compiler.  This is done using the
# online API (which takes a few seconds and requires an Internet connection).
# The uncompressed file is a script that loads in each of Blockly's core files
# one by one.  This takes much longer for a browser to load, but is useful
# when debugging code since line numbers are meaningful and variables haven't
# been renamed.  The uncompressed file also allows for a faster developement
# cycle since there is no need to rebuild or recompile, just reload.

import glob, httplib, json, os, re, sys, threading, urllib

def import_path(fullpath):
  """Import a file with full path specification.
  Allows one to import from any directory, something __import__ does not do.

  Args:
      fullpath:  Path and filename of import.

  Returns:
      An imported module.
  """
  path, filename = os.path.split(fullpath)
  filename, ext = os.path.splitext(filename)
  sys.path.append(path)
  module = __import__(filename)
  reload(module) # Might be out of date
  del sys.path[-1]
  return module

HEADER = ('// Do not edit this file; automatically generated by build.py.\n'
          '"use strict";\n')

class Gen_uncompressed(threading.Thread):
  """Generate a JavaScript file that loads Blockly's raw files.
  Runs in a separate thread.
  """
  def __init__(self, search_paths):
    threading.Thread.__init__(self)
    self.search_paths = search_paths

  def run(self):
    target_filename = 'blockly_uncompressed.js'
    f = open(target_filename, 'w')
    f.write(HEADER)
    f.write("""
window.BLOCKLY_DIR = (function() {
  // Find name of current directory.
  var scripts = document.getElementsByTagName('script');
  var re = new RegExp('(.+)[\/]blockly_uncompressed\.js$');
  for (var x = 0, script; script = scripts[x]; x++) {
    var match = re.exec(script.src);
    if (match) {
      return match[1];
    }
  }
  alert('Could not detect Blockly\\'s directory name.');
  return '';
})();

window.BLOCKLY_BOOT = function() {
// Execute after Closure has loaded.
if (!window.goog) {
  alert('Error: Closure not found.  Read this:\\n' +
        'http://code.google.com/p/blockly/wiki/Closure\\n');
}

// Build map of all dependencies (used and unused).
var dir = window.BLOCKLY_DIR.match(/[^\\/]+$/)[0];
""")
    add_dependency = []
    base_path = calcdeps.FindClosureBasePath(self.search_paths)
    for dep in calcdeps.BuildDependenciesFromFiles(self.search_paths):
      add_dependency.append(calcdeps.GetDepsLine(dep, base_path))
    add_dependency = '\n'.join(add_dependency)
    # Find the Blockly directory name and replace it with a JS variable.
    # This allows blockly_uncompressed.js to be compiled on one computer and be
    # used on another, even if the directory name differs.
    m = re.search('[\\/]([^\\/]+)[\\/]core[\\/]blockly.js', add_dependency)
    add_dependency = re.sub('([\\/])' + re.escape(m.group(1)) +
        '([\\/]core[\\/])', '\\1" + dir + "\\2', add_dependency)
    f.write(add_dependency + '\n')

    provides = []
    for dep in calcdeps.BuildDependenciesFromFiles(self.search_paths):
      if not dep.filename.startswith('../'):
        provides.extend(dep.provides)
    provides.sort()
    f.write('\n')
    f.write('// Load Blockly.\n')
    for provide in provides:
      f.write('goog.require(\'%s\');\n' % provide)

    f.write("""
delete window.BLOCKLY_DIR;
delete window.BLOCKLY_BOOT;
};

document.write('<script type="text/javascript" src="' + window.BLOCKLY_DIR +
    '/../closure-library-read-only/closure/goog/base.js"></script>');
document.write('<script type="text/javascript">window.BLOCKLY_BOOT()</script>');
""")
    f.close()
    print('SUCCESS: ' + target_filename)

class Gen_compressed(threading.Thread):
  """Generate a JavaScript file that contains all of Blockly's core and all
  required parts of Closure, compiled together.
  Uses the Closure Compiler's online API.
  Runs in a separate thread.
  """
  def __init__(self, search_paths):
    threading.Thread.__init__(self)
    self.search_paths = search_paths

  def run(self):
    self.gen_core()
    self.gen_generator('javascript')
    self.gen_generator('python')
    self.gen_language('de')
    self.gen_language('en')
    self.gen_language('pt_br')
    self.gen_language('vi')
    self.gen_language('zh_tw')

  def gen_core(self):
    target_filename = 'blockly_compressed.js'
    # Define the parameters for the POST request.
    params = [
        ('compilation_level', 'SIMPLE_OPTIMIZATIONS'),
        ('use_closure_library', 'true'),
        ('output_format', 'json'),
        ('output_info', 'compiled_code'),
        ('output_info', 'warnings'),
        ('output_info', 'errors'),
        ('output_info', 'statistics'),
      ]

    # Read in all the source files.
    filenames = calcdeps.CalculateDependencies(self.search_paths,
        ['core/blockly.js'])
    for filename in filenames:
      # Filter out the Closure files (the compiler will add them).
      if filename.startswith('../'):
        continue
      f = open(filename)
      params.append(('js_code', ''.join(f.readlines())))
      f.close()

    self.do_compile(params, target_filename, filenames)

  def gen_generator(self, language):
    target_filename = language + '_compressed.js'
    # Define the parameters for the POST request.
    params = [
        ('compilation_level', 'SIMPLE_OPTIMIZATIONS'),
        ('output_format', 'json'),
        ('output_info', 'compiled_code'),
        ('output_info', 'warnings'),
        ('output_info', 'errors'),
        ('output_info', 'statistics'),
      ]

    # Read in all the source files.
    filenames = glob.glob('./generators/%s/*.js' % language)
    filenames.insert(0, './generators/%s.js' % language)
    for filename in filenames:
      f = open(filename)
      params.append(('js_code', ''.join(f.readlines())))
      f.close()

    self.do_compile(params, target_filename, filenames)

  def gen_language(self, language):
    target_filename = language + '_compressed.js'
    # Define the parameters for the POST request.
    params = [
        ('compilation_level', 'SIMPLE_OPTIMIZATIONS'),
        ('output_format', 'json'),
        ('output_info', 'compiled_code'),
        ('output_info', 'warnings'),
        ('output_info', 'errors'),
        ('output_info', 'statistics'),
      ]

    # Read in all the source files.
    filenames = glob.glob('./language/common/*.js')
    filenames += glob.glob('./language/%s/*.js' % language)
    filenames.remove('./language/%s/_messages.js' % language)
    filenames.insert(0, './language/%s/_messages.js' % language)
    for filename in filenames:
      f = open(filename)
      params.append(('js_code', ''.join(f.readlines())))
      f.close()

    self.do_compile(params, target_filename, filenames)

  def do_compile(self, params, target_filename, filenames):
    # Send the request to Google.
    headers = { "Content-type": "application/x-www-form-urlencoded" }
    conn = httplib.HTTPConnection('closure-compiler.appspot.com')
    conn.request('POST', '/compile', urllib.urlencode(params), headers)
    response = conn.getresponse()
    json_str = response.read()
    conn.close()

    # Parse the JSON response.
    json_data = json.loads(json_str)

    def file_lookup(name):
      if not name.startswith('Input_'):
        return '???'
      n = int(name[6:])
      return filenames[n]

    if json_data.has_key('serverErrors'):
      errors = json_data['serverErrors']
      for error in errors:
        print 'SERVER ERROR: %s' % target_filename
        print error['error']
    elif json_data.has_key('errors'):
      errors = json_data['errors']
      for error in errors:
        print('FATAL ERROR')
        print(error['error'])
        print('%s at line %d:' % (
            file_lookup(error['file']), error['lineno']))
        print(error['line'])
        print((' ' * error['charno']) + '^')
        sys.exit(1)
    else:
      if json_data.has_key('warnings'):
        warnings = json_data['warnings']
        for warning in warnings:
          print('WARNING')
          print(warning['warning'])
          print('%s at line %d:' % (
              file_lookup(warning['file']), warning['lineno']))
          print(warning['line'])
          print((' ' * warning['charno']) + '^')
        print()

      if not json_data.has_key('compiledCode'):
        print('FATAL ERROR: Compiler did not return compiledCode.')
        sys.exit(1)

      code = HEADER + '\n' + json_data['compiledCode']

      stats = json_data['statistics']
      original_b = stats['originalSize']
      compressed_b = stats['compressedSize']
      if original_b > 0 and compressed_b > 0:
        f = open(target_filename, 'w')
        f.write(code)
        f.close()

        original_kb = int(original_b / 1024 + 0.5)
        compressed_kb = int(compressed_b / 1024 + 0.5)
        ratio = int(float(compressed_b) / float(original_b) * 100 + 0.5)
        print('SUCCESS: ' + target_filename)
        print('Size changed from %d KB to %d KB (%d%%).' % (
            original_kb, compressed_kb, ratio))
      else:
        print 'UNKNOWN ERROR'

if __name__ == '__main__':
  try:
    calcdeps = import_path(
          '../closure-library-read-only/closure/bin/calcdeps.py')
  except ImportError:
    print("""Error: Closure not found.  Read this:
http://code.google.com/p/blockly/wiki/Closure""")
    sys.exit(1)
  search_paths = calcdeps.ExpandDirectories(
      ['core/', '../closure-library-read-only/'])

  # Run both tasks in parallel threads.
  # Uncompressed is limited by processor speed.
  # Compressed is limited by network and server speed.
  Gen_uncompressed(search_paths).start()
  Gen_compressed(search_paths).start()
<commit_msg>Fix build.py to run on Windows (Issue 159).<commit_after>#!/usr/bin/python
# Compresses the core Blockly files into a single JavaScript file.
#
# Copyright 2012 Google Inc.
# http://blockly.googlecode.com/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script generates two files:
#   blockly_compressed.js
#   blockly_uncompressed.js
# The compressed file is a concatenation of all of Blockly's core files which
# have been run through Google's Closure Compiler.  This is done using the
# online API (which takes a few seconds and requires an Internet connection).
# The uncompressed file is a script that loads in each of Blockly's core files
# one by one.  This takes much longer for a browser to load, but is useful
# when debugging code since line numbers are meaningful and variables haven't
# been renamed.  The uncompressed file also allows for a faster developement
# cycle since there is no need to rebuild or recompile, just reload.

import glob, httplib, json, os, re, sys, threading, urllib

def import_path(fullpath):
  """Import a file with full path specification.
  Allows one to import from any directory, something __import__ does not do.

  Args:
      fullpath:  Path and filename of import.

  Returns:
      An imported module.
  """
  path, filename = os.path.split(fullpath)
  filename, ext = os.path.splitext(filename)
  sys.path.append(path)
  module = __import__(filename)
  reload(module) # Might be out of date
  del sys.path[-1]
  return module

HEADER = ('// Do not edit this file; automatically generated by build.py.\n'
          '"use strict";\n')

class Gen_uncompressed(threading.Thread):
  """Generate a JavaScript file that loads Blockly's raw files.
  Runs in a separate thread.
  """
  def __init__(self, search_paths):
    threading.Thread.__init__(self)
    self.search_paths = search_paths

  def run(self):
    target_filename = 'blockly_uncompressed.js'
    f = open(target_filename, 'w')
    f.write(HEADER)
    f.write("""
window.BLOCKLY_DIR = (function() {
  // Find name of current directory.
  var scripts = document.getElementsByTagName('script');
  var re = new RegExp('(.+)[\/]blockly_uncompressed\.js$');
  for (var x = 0, script; script = scripts[x]; x++) {
    var match = re.exec(script.src);
    if (match) {
      return match[1];
    }
  }
  alert('Could not detect Blockly\\'s directory name.');
  return '';
})();

window.BLOCKLY_BOOT = function() {
// Execute after Closure has loaded.
if (!window.goog) {
  alert('Error: Closure not found.  Read this:\\n' +
        'http://code.google.com/p/blockly/wiki/Closure\\n');
}

// Build map of all dependencies (used and unused).
var dir = window.BLOCKLY_DIR.match(/[^\\/]+$/)[0];
""")
    add_dependency = []
    base_path = calcdeps.FindClosureBasePath(self.search_paths)
    for dep in calcdeps.BuildDependenciesFromFiles(self.search_paths):
      add_dependency.append(calcdeps.GetDepsLine(dep, base_path))
    add_dependency = '\n'.join(add_dependency)
    # Find the Blockly directory name and replace it with a JS variable.
    # This allows blockly_uncompressed.js to be compiled on one computer and be
    # used on another, even if the directory name differs.
    m = re.search('[\\/]([^\\/]+)[\\/]core[\\/]blockly.js', add_dependency)
    add_dependency = re.sub('([\\/])' + re.escape(m.group(1)) +
        '([\\/]core[\\/])', '\\1" + dir + "\\2', add_dependency)
    f.write(add_dependency + '\n')

    provides = []
    for dep in calcdeps.BuildDependenciesFromFiles(self.search_paths):
      if not dep.filename.startswith(os.pardir + os.sep):  # '../'
        provides.extend(dep.provides)
    provides.sort()
    f.write('\n')
    f.write('// Load Blockly.\n')
    for provide in provides:
      f.write('goog.require(\'%s\');\n' % provide)

    f.write("""
delete window.BLOCKLY_DIR;
delete window.BLOCKLY_BOOT;
};

document.write('<script type="text/javascript" src="' + window.BLOCKLY_DIR +
    '/../closure-library-read-only/closure/goog/base.js"></script>');
document.write('<script type="text/javascript">window.BLOCKLY_BOOT()</script>');
""")
    f.close()
    print('SUCCESS: ' + target_filename)

class Gen_compressed(threading.Thread):
  """Generate a JavaScript file that contains all of Blockly's core and all
  required parts of Closure, compiled together.
  Uses the Closure Compiler's online API.
  Runs in a separate thread.
  """
  def __init__(self, search_paths):
    threading.Thread.__init__(self)
    self.search_paths = search_paths

  def run(self):
    self.gen_core()
    self.gen_generator('javascript')
    self.gen_generator('python')
    self.gen_language('de')
    self.gen_language('en')
    self.gen_language('pt_br')
    self.gen_language('vi')
    self.gen_language('zh_tw')

  def gen_core(self):
    target_filename = 'blockly_compressed.js'
    # Define the parameters for the POST request.
    params = [
        ('compilation_level', 'SIMPLE_OPTIMIZATIONS'),
        ('use_closure_library', 'true'),
        ('output_format', 'json'),
        ('output_info', 'compiled_code'),
        ('output_info', 'warnings'),
        ('output_info', 'errors'),
        ('output_info', 'statistics'),
      ]

    # Read in all the source files.
    filenames = calcdeps.CalculateDependencies(self.search_paths,
        [os.path.join('core', 'blockly.js')])
    for filename in filenames:
      # Filter out the Closure files (the compiler will add them).
      if filename.startswith(os.pardir + os.sep):  # '../'
        continue
      f = open(filename)
      params.append(('js_code', ''.join(f.readlines())))
      f.close()

    self.do_compile(params, target_filename, filenames)

  def gen_generator(self, language):
    target_filename = language + '_compressed.js'
    # Define the parameters for the POST request.
    params = [
        ('compilation_level', 'SIMPLE_OPTIMIZATIONS'),
        ('output_format', 'json'),
        ('output_info', 'compiled_code'),
        ('output_info', 'warnings'),
        ('output_info', 'errors'),
        ('output_info', 'statistics'),
      ]

    # Read in all the source files.
    filenames = glob.glob(
        os.path.join('generators', language, '*.js'))
    filenames.insert(0, os.path.join('generators', language + '.js'))
    for filename in filenames:
      f = open(filename)
      params.append(('js_code', ''.join(f.readlines())))
      f.close()

    self.do_compile(params, target_filename, filenames)

  def gen_language(self, language):
    target_filename = language + '_compressed.js'
    # Define the parameters for the POST request.
    params = [
        ('compilation_level', 'SIMPLE_OPTIMIZATIONS'),
        ('output_format', 'json'),
        ('output_info', 'compiled_code'),
        ('output_info', 'warnings'),
        ('output_info', 'errors'),
        ('output_info', 'statistics'),
      ]

    # Read in all the source files.
    filenames = glob.glob(os.path.join('language', 'common', '*.js'))
    filenames += glob.glob(os.path.join('language', language, '*.js'))
    filenames.remove(os.path.join('language', language, '_messages.js'))
    filenames.insert(0, os.path.join('language', language, '_messages.js'))
    for filename in filenames:
      f = open(filename)
      params.append(('js_code', ''.join(f.readlines())))
      f.close()

    self.do_compile(params, target_filename, filenames)

  def do_compile(self, params, target_filename, filenames):
    # Send the request to Google.
    headers = { "Content-type": "application/x-www-form-urlencoded" }
    conn = httplib.HTTPConnection('closure-compiler.appspot.com')
    conn.request('POST', '/compile', urllib.urlencode(params), headers)
    response = conn.getresponse()
    json_str = response.read()
    conn.close()

    # Parse the JSON response.
    json_data = json.loads(json_str)

    def file_lookup(name):
      if not name.startswith('Input_'):
        return '???'
      n = int(name[6:])
      return filenames[n]

    if json_data.has_key('serverErrors'):
      errors = json_data['serverErrors']
      for error in errors:
        print 'SERVER ERROR: %s' % target_filename
        print error['error']
    elif json_data.has_key('errors'):
      errors = json_data['errors']
      for error in errors:
        print('FATAL ERROR')
        print(error['error'])
        print('%s at line %d:' % (
            file_lookup(error['file']), error['lineno']))
        print(error['line'])
        print((' ' * error['charno']) + '^')
        sys.exit(1)
    else:
      if json_data.has_key('warnings'):
        warnings = json_data['warnings']
        for warning in warnings:
          print('WARNING')
          print(warning['warning'])
          print('%s at line %d:' % (
              file_lookup(warning['file']), warning['lineno']))
          print(warning['line'])
          print((' ' * warning['charno']) + '^')
        print()

      if not json_data.has_key('compiledCode'):
        print('FATAL ERROR: Compiler did not return compiledCode.')
        sys.exit(1)

      code = HEADER + '\n' + json_data['compiledCode']

      stats = json_data['statistics']
      original_b = stats['originalSize']
      compressed_b = stats['compressedSize']
      if original_b > 0 and compressed_b > 0:
        f = open(target_filename, 'w')
        f.write(code)
        f.close()

        original_kb = int(original_b / 1024 + 0.5)
        compressed_kb = int(compressed_b / 1024 + 0.5)
        ratio = int(float(compressed_b) / float(original_b) * 100 + 0.5)
        print('SUCCESS: ' + target_filename)
        print('Size changed from %d KB to %d KB (%d%%).' % (
            original_kb, compressed_kb, ratio))
      else:
        print 'UNKNOWN ERROR'

if __name__ == '__main__':
  try:
    calcdeps = import_path(os.path.join(os.path.pardir,
          'closure-library-read-only', 'closure', 'bin', 'calcdeps.py'))
  except ImportError:
    print("""Error: Closure not found.  Read this:
http://code.google.com/p/blockly/wiki/Closure""")
    sys.exit(1)
  search_paths = calcdeps.ExpandDirectories(
      ['core', os.path.join(os.path.pardir, 'closure-library-read-only')])

  # Run both tasks in parallel threads.
  # Uncompressed is limited by processor speed.
  # Compressed is limited by network and server speed.
  Gen_uncompressed(search_paths).start()
  Gen_compressed(search_paths).start()
<|endoftext|>


<commit_before>from mcapp import app
from decorators import crossdomain, apikey, jsonp
from flask import request, g
import rethinkdb as r
import error
import dmutil
import json
import access

@app.route('/processes/<process_id>', methods=['GET'])
@jsonp
def get_process(process_id):
    return dmutil.get_single_from_table('processes', process_id)

@app.route('/processes', methods=['GET'])
@jsonp
def get_all_processes():
    return dmutil.get_all_from_table('processes')

@app.route('/processes/new', methods=['POST'])
@apikey
@crossdomain(origin='*')
def create_process():
    p = dict()
    j = request.get_json()
    p['name'] = dmutil.get_required('name', j)
    user = access.get_user()
    p['owner'] = user
    p['project'] = dmutil.get_required('project', j)
    if not dmutil.item_exists('projects', p['project']):
        return error.not_acceptable("Unknown project id: %s" % (p['project']))
    p['birthtime'] = r.now()
    p['mtime'] = p['birthtime']
    p['machine'] = dmutil.get_optional('machine', j)
    p['process_type'] = dmutil.get_required('process_type', j)
    p['description'] = dmutil.get_required('description', j)
    p['version'] = dmutil.get_optional('version', j)
    p['notes'] = dmutil.get_optional('notes', j, [])
    p['input_conditions'] = dmutil.get_optional('input_conditions', j, [])
    p['input_files'] = dmutil.get_optional('input_files', j, [])
    p['output_conditions'] = dmutil.get_optional('output_conditions', j, [])
    p['output_files'] = dmutil.get_optional('output_files', j, [])
    p['runs'] = dmutil.get_optional('runs', j, [])
    p['citations'] = dmutil.get_optional('citations', j, [])
    p['status'] = dmutil.get_optional('status', j)
    return dmutil.insert_entry('processes', p)

@app.route('/processes/from_template', methods=['POST'])
@apikey
@crossdomain(origin='*')
def create_process_from_template():
    p = dict()
    j = request.get_json()
    p['template'] = dmutil.get_required('id', j)
    project_id = dmutil.get_required('project', j)
    p['project'] = project_id
    if not dmutil.item_exists('projects', project_id):
        return error.not_acceptable("Unknown project id: %s" % (project_id))
    m = j['model']
    p['name'] = dmutil.get_required_prop('name', m)
    p['birthtime'] = r.now()
    p['mtime'] = p['birthtime']
    p['machine'] = dmutil.get_optional_prop('machine', m)
    p['process_type'] = dmutil.get_required_prop('process_type', m)
    p['description'] = dmutil.get_required_prop('description', m)
    p['version'] = dmutil.get_optional_prop('version', m)
    p['notes'] = dmutil.get_optional_prop('notes', m, [])
    p['input_conditions'] = dmutil.get_optional_prop('input_conditions', m, [])
    p['input_files'] = dmutil.get_optional_prop('input_files', m, [])
    p['output_conditions'] = dmutil.get_optional_prop('output_conditions', m, [])
    p['output_files'] = dmutil.get_optional_prop('output_files', m, [])
    p['runs'] = dmutil.get_optional_prop('runs', m, [])
    p['citations'] = dmutil.get_optional_prop('citations', m, [])
    p['status'] = dmutil.get_optional_prop('status', m)
    process_id = dmutil.insert_entry_id('processes', p)
    dmutil.insert_join_entry('project2processes',\
                           {'project_id': project_id, 'process_id': process_id})
    return json.dumps({'id': process_id})

@app.route('/processes/<process_id>/update', methods=['PUT'])
@apikey
@crossdomain(origin='*')
def update_process(process_id):
    process = r.table('processes').get(process_id).run(g.conn)
    if process is None:
        return error.bad_request("No such process: " + process_id)
    j = request.get_json()
    for id in j['input_files']:
        process['input_files'].append(id)
    input_files = process['input_files']
    r.table('processes').get(process_id).update({'input_files':input_files}).run(g.conn)
    return json.dumps({"success": True})
<commit_msg>Remove calls that were moved into the upload state building.<commit_after>from mcapp import app
from decorators import crossdomain, apikey, jsonp
from flask import request
import rethinkdb as r
import error
import dmutil
import access

@app.route('/processes/<process_id>', methods=['GET'])
@jsonp
def get_process(process_id):
    return dmutil.get_single_from_table('processes', process_id)

@app.route('/processes', methods=['GET'])
@jsonp
def get_all_processes():
    return dmutil.get_all_from_table('processes')

@app.route('/processes/new', methods=['POST'])
@apikey
@crossdomain(origin='*')
def create_process():
    p = dict()
    j = request.get_json()
    p['name'] = dmutil.get_required('name', j)
    user = access.get_user()
    p['owner'] = user
    p['project'] = dmutil.get_required('project', j)
    if not dmutil.item_exists('projects', p['project']):
        return error.not_acceptable("Unknown project id: %s" % (p['project']))
    p['birthtime'] = r.now()
    p['mtime'] = p['birthtime']
    p['machine'] = dmutil.get_optional('machine', j)
    p['process_type'] = dmutil.get_required('process_type', j)
    p['description'] = dmutil.get_required('description', j)
    p['version'] = dmutil.get_optional('version', j)
    p['notes'] = dmutil.get_optional('notes', j, [])
    p['input_conditions'] = dmutil.get_optional('input_conditions', j, [])
    p['input_files'] = dmutil.get_optional('input_files', j, [])
    p['output_conditions'] = dmutil.get_optional('output_conditions', j, [])
    p['output_files'] = dmutil.get_optional('output_files', j, [])
    p['runs'] = dmutil.get_optional('runs', j, [])
    p['citations'] = dmutil.get_optional('citations', j, [])
    p['status'] = dmutil.get_optional('status', j)
    return dmutil.insert_entry('processes', p)
<|endoftext|>


<commit_before>from mcapp import app
from decorators import crossdomain, apikey, jsonp
from flask import request, g
import rethinkdb as r
import error
import dmutil
import resp
import access
import args
import sys

def msg(s):
    print s
    sys.stdout.flush()

@app.route('/processes/project/<project_id>', methods=['GET'])
@apikey
@jsonp
def get_processes_for_project(project_id):
    processes = get_processes(project_id)
    return resp.to_json(processes)

def get_process_information(project_id):
    processes = list(r.table('project2process')
                     .get_all(project_id, index="project_id")
                     .eq_join("process_id", r.table("processes"))
                     .zip().run(g.conn, time_format="raw"))
    by_ids = {p['id']: p for p in processes}
    get_setup(project_id, by_ids)
    get_samples(project_id, by_ids)
    get_files(project_id, by_ids)
    return processes

def get_setup(project_id, processes):
    msg(processes.keys())
    setup = list(r.table('process2setup')
                 .get_all(*processes.keys(), index="process_id")
                 .merge(lambda row:
                        {
                            'setup_properties': r.table('setups')
                            .filter({"id": row['setup_id']})
                            .eq_join("id", r.table("setupproperties"),
                                     index="setup_id").zip()
                            .coerce_to('array'),
                            'samples': r.table('samples')
                        })
                 .run(g.conn, time_format="raw"))
    msg("setup....")
    msg(setup)

def get_samples(project_id, by_ids):
    pass

def get_files(project_id, by_ids):
    pass

def get_processes(project_id):
    processes = list(r.table("processes")
                     .get_all(project_id, index="project_id")
                     .run(g.conn, time_format="raw"))
    # For fast lookup put processes in a map
    by_ids = {p['id']: p for p in processes}

    # Add additional fields to fill in for the properties
    # and upstream/downstream processes
    for id, process in by_ids.iteritems():
        process['inputs'] = {}
        process['outputs'] = {}
        process['upstream'] = {}
        process['downstream'] = {}

    if len(by_ids) != 0:
        all_property_sets = get_property_sets(by_ids)
        add_process_properties(all_property_sets, by_ids)
        connect_feeders(all_property_sets, by_ids)
    return processes

def get_property_sets(processes):
    # Get all properties for all of the processes
    all = list(r.table("property_sets")
               .get_all(*processes.keys(), index="item_id")
               .eq_join("id", r.table("properties"), index="item_id")
               .map(lambda row: row.merge({
                   "left": {
                       "ps_id": row["left"]["id"],
                       "ps_name": row["left"]["name"],
                       "ps_process_id": row["left"]["item_id"],
                       "ps_item_type": row["left"]["item_type"]
                   }
               }))
               .without({
                   "left": {
                       "id": True,
                       "name": True,
                       "item_id": True,
                       "item_type": True
                   }
               })
               .zip().run(g.conn, time_format="raw"))
    return all

def add_process_properties(property_sets, processes):
    # Now go through each property set and add the property set
    # and its properties to the relevant process
    for property_set in property_sets:
        process_id = property_set["ps_process_id"]
        process = processes[process_id]
        add_property_set(property_set, process)

def add_property_set(property_set, process):
    ps_name = property_set["ps_name"]
    stype = property_set["stype"]
    if ps_name not in process[stype]:
        process[stype][ps_name] = []
    process[stype][ps_name].append({
        "unit": property_set["unit"],
        "other": property_set["other"],
        "display_name": property_set["display_name"],
        "name": property_set["name"],
        "ptype": property_set["ptype"],
        "value": property_set["value"]
    })

def connect_feeders(property_sets, processes):
    # Get the ids of all input/output properties that
    # could go into or come from another process
    inputs2processes, outputs2processes = get_feeders(property_sets)
    attach_feeders(inputs2processes, outputs2processes, processes)

def get_feeders(property_sets):
    feeder_ids = [prop['value'] for prop in property_sets
                  if prop["ptype"] == "file" or prop["ptype"] == "sample"]
    query = r.table("properties")\
             .get_all(*feeder_ids, index="value")\
             .eq_join("item_id", r.table("property_sets"))\
             .zip()\
             .eq_join("item_id", r.table("processes"))\
             .zip()
    i2p = {}
    o2p = {}
    for p in query.run(g.conn):
        if p["stype"] == "inputs":
            use = i2p
        else:
            use = o2p
        id = p["value"]
        if p["value"] not in use:
            use[id] = {
                "ptype": p["ptype"],
                "name": p["other"]["name"],
                "processes": {}
            }
        pid = p["id"]
        use[id]["processes"][pid] = p["name"]
    return i2p, o2p

def attach_feeders(inputs2processes, outputs2processes, processes):
    attach_feeders_for_type(outputs2processes, "downstream", "inputs",
                            processes)
    attach_feeders_for_type(inputs2processes, "upstream", "outputs",
                            processes)

# TODO: This shold probably be decomposed further. There is a lot
# of if and loop nesting.
def attach_feeders_for_type(io2process, feeder_name, io_name, processes):
    for key, process in processes.iteritems():
        for key, io_item in process[io_name].iteritems():
            # Only files and samples can go into a process
            # so skip other properties
            if key != "files" and key != "sample":
                continue
            # We now have an input or output set of item. Go through
            # each. The reason for the loop is we may have multiple
            # files in a single item (but only one sample, but we can
            # treat each as if there are multiple)
            for item in io_item:
                item_id = item["value"]
                # Check if this item is a potential feeder
                if item_id in io2process:
                    # It is a a feeder, now check the processes it goes
                    # to/from. We ignore our process.
                    for process_id in io2process[item_id]["processes"]:
                        if process_id != process["id"]:
                            # Not our process so add it. Check to see if
                            # our process entry exists or not and create
                            # if it doesn't
                            p_item = io2process[item_id]
                            if process_id not in process[feeder_name]:
                                process[feeder_name][process_id] = {
                                    "name": p_item["processes"][process_id],
                                    "items": []
                                }
                            process[feeder_name][process_id]["items"].append({
                                "item_name": p_item["name"],
                                "item_type": p_item["ptype"],
                            })

@app.route('/processes/<process_id>', methods=['GET'])
@apikey(shared=True)
@jsonp
def get_process(process_id):
    process = dmutil.get_single_from_table('processes', process_id, raw=True)
    if process is None:
        return error.bad_request("Unknown process")
    result = build_sample_file_objects(process, '')
    return args.json_as_format_arg(result)

@app.route('/processes/template/<template_id>', methods=['GET'])
@apikey
@jsonp
def get_all_processes_for_template(template_id):
    rr = r.table('processes').filter({'template': template_id})
    selection = list(rr.run(g.conn, time_format='raw'))
    return args.json_as_format_arg(selection)

@app.route('/processes/sample/<sample_id>', methods=['GET'])
@apikey
@jsonp
def get_processes_by_sample(sample_id):
    rr = r.table('samples_processes_denorm').get_all(sample_id,
                                                     index='sample_id')
    selection = list(rr.run(g.conn, time_format='raw'))
    return args.json_as_format_arg(selection)

@app.route('/processes/file/<file_id>', methods=['GET'])
@jsonp
def get_processes_by_file(file_id):
    rv = r.table('datafiles_denorm')\
          .filter({'df_id': file_id})\
          .eq_join('process_id', r.table('processes')).zip()
    selection = list(rv.run(g.conn, time_format='raw'))
    result = build_sample_file_objects(selection, 'array')
    return args.json_as_format_arg(result)

def build_sample_file_objects(selection, type):
    sample_list = []
    file_list = []
    file_objs = []
    if type == "array":
        for s in selection:
            inputs = s['inputs']
            outputs = s['outputs']
            for i in inputs:
                if i['attribute'] == 'sample':
                    sample_list.append(i['properties']['id']['value'])
                elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
            for o in outputs:
                if o['attribute'] == 'sample':
                    sample_list.append(o['properties']['id']['value'])
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
        if len(sample_list) != 0:
            rr = r.table('samples').get_all(*sample_list)
            sample_objs = list(rr.run(g.conn, time_format='raw'))
        if len(file_list) != 0:
            rr = r.table('datafiles').get_all(*file_list)
            file_objs = list(rr.run(g.conn, time_format='raw'))
        # stick the objects
        for s in selection:
            # get_all_sample_ids
            for i in inputs:
                item = i["properties"]["id"]["value"]
                if i['attribute'] == 'sample':
                    i['properties']['obj'] = get_an_item(item, sample_objs)
                elif i['attribute'] == 'file':
                    file_list.append(item)
                    i['properties']['obj'] = get_an_item(item, file_objs)
            for o in outputs:
                item = o["properties"]["id"]["value"]
                if o['attribute'] == 'sample':
                    o['properties']['obj'] = get_an_item(item, sample_objs)
                elif o['attribute'] == 'file':
                    file_list.append(item)
                    o['properties']['obj'] = get_an_item(item, file_objs)
        return selection
    else:
        s = selection
        inputs = s['inputs']
        outputs = s['outputs']
        for i in inputs:
            if i['attribute'] == 'sample':
                sample_list.append(i['properties']['id']['value'])
            elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
            for o in outputs:
                if o['attribute'] == 'sample':
                    sample_list.append(o['properties']['id']['value'])
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
        if len(sample_list) != 0:
            rr = r.table('samples').get_all(*sample_list)
            sample_objs = list(rr.run(g.conn, time_format='raw'))
        if len(file_list) != 0:
            rr = r.table('datafiles').get_all(*file_list)
            file_objs = list(rr.run(g.conn, time_format='raw'))
        # stick the objects
        for s in selection:
            # get_all_sample_ids
            for i in inputs:
                item = i["properties"]["id"]["value"]
                if i['attribute'] == 'sample':
                    i['properties']['obj'] = get_an_item(item, sample_objs)
                elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
                    i['properties']['obj'] = get_an_item(item, file_objs)
            for o in outputs:
                item = o["properties"]["id"]["value"]
                if o['attribute'] == 'sample':
                    o['properties']['obj'] = get_an_item(item, sample_objs)
                elif o['attribute'] == 'file':
                    file_list.append(item)
                    o['properties']['obj'] = get_an_item(item, file_objs)
        return selection

def get_an_item(id, obj_list):
    for obj in obj_list:
        if obj['id'] == id:
            return obj

@app.route('/processes/new', methods=['POST'])
@apikey
@crossdomain(origin='*')
def create_process():
    p = dict()
    j = request.get_json()
    p['name'] = dmutil.get_required('name', j)
    user = access.get_user()
    p['owner'] = user
    p['project'] = dmutil.get_required('project', j)
    if not dmutil.item_exists('projects', p['project']):
        return error.not_acceptable("Unknown project id: %s" % (p['project']))
    p['birthtime'] = r.now()
    p['mtime'] = p['birthtime']
    p['machine'] = dmutil.get_optional('machine', j)
    p['process_type'] = dmutil.get_required('process_type', j)
    p['description'] = dmutil.get_required('description', j)
    p['version'] = dmutil.get_optional('version', j)
    p['notes'] = dmutil.get_optional('notes', j, [])
    p['input_conditions'] = dmutil.get_optional('input_conditions', j, [])
    p['input_files'] = dmutil.get_optional('input_files', j, [])
    p['output_conditions'] = dmutil.get_optional('output_conditions', j, [])
    p['output_files'] = dmutil.get_optional('output_files', j, [])
    p['runs'] = dmutil.get_optional('runs', j, [])
    p['citations'] = dmutil.get_optional('citations', j, [])
    p['status'] = dmutil.get_optional('status', j)
    id = dmutil.insert_entry('processes', p)
    return resp.to_json_id(id)

@app.route('/process/update/<path:processid>', methods=['PUT'])
@apikey
@crossdomain(origin='*')
def update_process(processid):
    rv = r.table('processes').get(processid).update(request.json).run(g.conn)
    if (rv['replaced'] == 1 or rv['unchanged'] == 1):
        return ''
    else:
        error.update_conflict("Unable to update process: " + processid)
<commit_msg>update<commit_after>from mcapp import app
from decorators import crossdomain, apikey, jsonp
from flask import request, g
import rethinkdb as r
import error
import dmutil
import resp
import access
import args
import sys

def msg(s):
    print s
    sys.stdout.flush()

@app.route('/processes/project/<project_id>', methods=['GET'])
@apikey
@jsonp
def get_processes_for_project(project_id):
    processes = get_processes(project_id)
    return resp.to_json(processes)

def get_process_information(project_id):
    processes = list(r.table('project2process')
                     .get_all(project_id, index="project_id")
                     .eq_join("process_id", r.table("processes"))
                     .zip().run(g.conn, time_format="raw"))
    by_ids = {p['id']: p for p in processes}
    get_setup(project_id, by_ids)
    get_samples(project_id, by_ids)
    get_files(project_id, by_ids)
    return processes

def get_setup(project_id, processes):
    msg(processes.keys())
    setup = list(r.table('process2setup')
                 .get_all(*processes.keys(), index="process_id")
                 .merge(lambda row:
                        {
                            'setup_properties': r.table('setups')
                            .filter({"id": row['setup_id']})
                            .eq_join("id", r.table("setupproperties"),
                                     index="setup_id").zip()
                            .coerce_to('array')
                        })
                 .run(g.conn, time_format="raw"))
    msg("setup....")
    msg(setup)

def get_samples(project_id, by_ids):
    pass

def get_files(project_id, by_ids):
    pass

def get_processes(project_id):
    processes = list(r.table("processes")
                     .get_all(project_id, index="project_id")
                     .run(g.conn, time_format="raw"))
    # For fast lookup put processes in a map
    by_ids = {p['id']: p for p in processes}

    # Add additional fields to fill in for the properties
    # and upstream/downstream processes
    for id, process in by_ids.iteritems():
        process['inputs'] = {}
        process['outputs'] = {}
        process['upstream'] = {}
        process['downstream'] = {}

    if len(by_ids) != 0:
        all_property_sets = get_property_sets(by_ids)
        add_process_properties(all_property_sets, by_ids)
        connect_feeders(all_property_sets, by_ids)
    return processes

def get_property_sets(processes):
    # Get all properties for all of the processes
    all = list(r.table("property_sets")
               .get_all(*processes.keys(), index="item_id")
               .eq_join("id", r.table("properties"), index="item_id")
               .map(lambda row: row.merge({
                   "left": {
                       "ps_id": row["left"]["id"],
                       "ps_name": row["left"]["name"],
                       "ps_process_id": row["left"]["item_id"],
                       "ps_item_type": row["left"]["item_type"]
                   }
               }))
               .without({
                   "left": {
                       "id": True,
                       "name": True,
                       "item_id": True,
                       "item_type": True
                   }
               })
               .zip().run(g.conn, time_format="raw"))
    return all

def add_process_properties(property_sets, processes):
    # Now go through each property set and add the property set
    # and its properties to the relevant process
    for property_set in property_sets:
        process_id = property_set["ps_process_id"]
        process = processes[process_id]
        add_property_set(property_set, process)

def add_property_set(property_set, process):
    ps_name = property_set["ps_name"]
    stype = property_set["stype"]
    if ps_name not in process[stype]:
        process[stype][ps_name] = []
    process[stype][ps_name].append({
        "unit": property_set["unit"],
        "other": property_set["other"],
        "display_name": property_set["display_name"],
        "name": property_set["name"],
        "ptype": property_set["ptype"],
        "value": property_set["value"]
    })

def connect_feeders(property_sets, processes):
    # Get the ids of all input/output properties that
    # could go into or come from another process
    inputs2processes, outputs2processes = get_feeders(property_sets)
    attach_feeders(inputs2processes, outputs2processes, processes)

def get_feeders(property_sets):
    feeder_ids = [prop['value'] for prop in property_sets
                  if prop["ptype"] == "file" or prop["ptype"] == "sample"]
    query = r.table("properties")\
             .get_all(*feeder_ids, index="value")\
             .eq_join("item_id", r.table("property_sets"))\
             .zip()\
             .eq_join("item_id", r.table("processes"))\
             .zip()
    i2p = {}
    o2p = {}
    for p in query.run(g.conn):
        if p["stype"] == "inputs":
            use = i2p
        else:
            use = o2p
        id = p["value"]
        if p["value"] not in use:
            use[id] = {
                "ptype": p["ptype"],
                "name": p["other"]["name"],
                "processes": {}
            }
        pid = p["id"]
        use[id]["processes"][pid] = p["name"]
    return i2p, o2p

def attach_feeders(inputs2processes, outputs2processes, processes):
    attach_feeders_for_type(outputs2processes, "downstream", "inputs",
                            processes)
    attach_feeders_for_type(inputs2processes, "upstream", "outputs",
                            processes)

# TODO: This shold probably be decomposed further. There is a lot
# of if and loop nesting.
def attach_feeders_for_type(io2process, feeder_name, io_name, processes):
    for key, process in processes.iteritems():
        for key, io_item in process[io_name].iteritems():
            # Only files and samples can go into a process
            # so skip other properties
            if key != "files" and key != "sample":
                continue
            # We now have an input or output set of item. Go through
            # each. The reason for the loop is we may have multiple
            # files in a single item (but only one sample, but we can
            # treat each as if there are multiple)
            for item in io_item:
                item_id = item["value"]
                # Check if this item is a potential feeder
                if item_id in io2process:
                    # It is a a feeder, now check the processes it goes
                    # to/from. We ignore our process.
                    for process_id in io2process[item_id]["processes"]:
                        if process_id != process["id"]:
                            # Not our process so add it. Check to see if
                            # our process entry exists or not and create
                            # if it doesn't
                            p_item = io2process[item_id]
                            if process_id not in process[feeder_name]:
                                process[feeder_name][process_id] = {
                                    "name": p_item["processes"][process_id],
                                    "items": []
                                }
                            process[feeder_name][process_id]["items"].append({
                                "item_name": p_item["name"],
                                "item_type": p_item["ptype"],
                            })

@app.route('/processes/<process_id>', methods=['GET'])
@apikey(shared=True)
@jsonp
def get_process(process_id):
    process = dmutil.get_single_from_table('processes', process_id, raw=True)
    if process is None:
        return error.bad_request("Unknown process")
    result = build_sample_file_objects(process, '')
    return args.json_as_format_arg(result)

@app.route('/processes/template/<template_id>', methods=['GET'])
@apikey
@jsonp
def get_all_processes_for_template(template_id):
    rr = r.table('processes').filter({'template': template_id})
    selection = list(rr.run(g.conn, time_format='raw'))
    return args.json_as_format_arg(selection)

@app.route('/processes/sample/<sample_id>', methods=['GET'])
@apikey
@jsonp
def get_processes_by_sample(sample_id):
    rr = r.table('samples_processes_denorm').get_all(sample_id,
                                                     index='sample_id')
    selection = list(rr.run(g.conn, time_format='raw'))
    return args.json_as_format_arg(selection)

@app.route('/processes/file/<file_id>', methods=['GET'])
@jsonp
def get_processes_by_file(file_id):
    rv = r.table('datafiles_denorm')\
          .filter({'df_id': file_id})\
          .eq_join('process_id', r.table('processes')).zip()
    selection = list(rv.run(g.conn, time_format='raw'))
    result = build_sample_file_objects(selection, 'array')
    return args.json_as_format_arg(result)

def build_sample_file_objects(selection, type):
    sample_list = []
    file_list = []
    file_objs = []
    if type == "array":
        for s in selection:
            inputs = s['inputs']
            outputs = s['outputs']
            for i in inputs:
                if i['attribute'] == 'sample':
                    sample_list.append(i['properties']['id']['value'])
                elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
            for o in outputs:
                if o['attribute'] == 'sample':
                    sample_list.append(o['properties']['id']['value'])
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
        if len(sample_list) != 0:
            rr = r.table('samples').get_all(*sample_list)
            sample_objs = list(rr.run(g.conn, time_format='raw'))
        if len(file_list) != 0:
            rr = r.table('datafiles').get_all(*file_list)
            file_objs = list(rr.run(g.conn, time_format='raw'))
        # stick the objects
        for s in selection:
            # get_all_sample_ids
            for i in inputs:
                item = i["properties"]["id"]["value"]
                if i['attribute'] == 'sample':
                    i['properties']['obj'] = get_an_item(item, sample_objs)
                elif i['attribute'] == 'file':
                    file_list.append(item)
                    i['properties']['obj'] = get_an_item(item, file_objs)
            for o in outputs:
                item = o["properties"]["id"]["value"]
                if o['attribute'] == 'sample':
                    o['properties']['obj'] = get_an_item(item, sample_objs)
                elif o['attribute'] == 'file':
                    file_list.append(item)
                    o['properties']['obj'] = get_an_item(item, file_objs)
        return selection
    else:
        s = selection
        inputs = s['inputs']
        outputs = s['outputs']
        for i in inputs:
            if i['attribute'] == 'sample':
                sample_list.append(i['properties']['id']['value'])
            elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
            for o in outputs:
                if o['attribute'] == 'sample':
                    sample_list.append(o['properties']['id']['value'])
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
        if len(sample_list) != 0:
            rr = r.table('samples').get_all(*sample_list)
            sample_objs = list(rr.run(g.conn, time_format='raw'))
        if len(file_list) != 0:
            rr = r.table('datafiles').get_all(*file_list)
            file_objs = list(rr.run(g.conn, time_format='raw'))
        # stick the objects
        for s in selection:
            # get_all_sample_ids
            for i in inputs:
                item = i["properties"]["id"]["value"]
                if i['attribute'] == 'sample':
                    i['properties']['obj'] = get_an_item(item, sample_objs)
                elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
                    i['properties']['obj'] = get_an_item(item, file_objs)
            for o in outputs:
                item = o["properties"]["id"]["value"]
                if o['attribute'] == 'sample':
                    o['properties']['obj'] = get_an_item(item, sample_objs)
                elif o['attribute'] == 'file':
                    file_list.append(item)
                    o['properties']['obj'] = get_an_item(item, file_objs)
        return selection

def get_an_item(id, obj_list):
    for obj in obj_list:
        if obj['id'] == id:
            return obj

@app.route('/processes/new', methods=['POST'])
@apikey
@crossdomain(origin='*')
def create_process():
    p = dict()
    j = request.get_json()
    p['name'] = dmutil.get_required('name', j)
    user = access.get_user()
    p['owner'] = user
    p['project'] = dmutil.get_required('project', j)
    if not dmutil.item_exists('projects', p['project']):
        return error.not_acceptable("Unknown project id: %s" % (p['project']))
    p['birthtime'] = r.now()
    p['mtime'] = p['birthtime']
    p['machine'] = dmutil.get_optional('machine', j)
    p['process_type'] = dmutil.get_required('process_type', j)
    p['description'] = dmutil.get_required('description', j)
    p['version'] = dmutil.get_optional('version', j)
    p['notes'] = dmutil.get_optional('notes', j, [])
    p['input_conditions'] = dmutil.get_optional('input_conditions', j, [])
    p['input_files'] = dmutil.get_optional('input_files', j, [])
    p['output_conditions'] = dmutil.get_optional('output_conditions', j, [])
    p['output_files'] = dmutil.get_optional('output_files', j, [])
    p['runs'] = dmutil.get_optional('runs', j, [])
    p['citations'] = dmutil.get_optional('citations', j, [])
    p['status'] = dmutil.get_optional('status', j)
    id = dmutil.insert_entry('processes', p)
    return resp.to_json_id(id)

@app.route('/process/update/<path:processid>', methods=['PUT'])
@apikey
@crossdomain(origin='*')
def update_process(processid):
    rv = r.table('processes').get(processid).update(request.json).run(g.conn)
    if (rv['replaced'] == 1 or rv['unchanged'] == 1):
        return ''
    else:
        error.update_conflict("Unable to update process: " + processid)
<|endoftext|>


<commit_before>import os
import sys

from flask.ext.frozen import Freezer
import webassets

from content import app, assets

def build_assets():
    for bundle in assets:
        assert isinstance(bundle, webassets.Bundle)
        print("Building bundle {}".format(bundle.output))
        bundle.build(force=True, disable_cache=True)

# This is a copy of Freezer.no_argument_rules() modified to ignore certain paths
def no_argument_rules_urls_with_ignore():
    """URL generator for URL rules that take no arguments."""
    ignored = app.config.get('FREEZER_IGNORE_ENDPOINTS', [])
    for rule in app.url_map.iter_rules():
        if rule.endpoint not in ignored and not rule.arguments and 'GET' in rule.methods:
            yield rule.endpoint, {}

def build_pages():
    ignored = [bundle.output for bundle in assets]
    app.config['FREEZER_DESTINATION'] = os.path.abspath("static")
    app.config['FREEZER_BASE_URL'] = "http://dabo.guru/"
    app.config['FREEZER_DESTINATION_IGNORE'] = ignored
    app.config['FREEZER_IGNORE_ENDPOINTS'] = ['oauth_respond', 'mc_api_name', 'mc_api_uuid', 'serve_markdown']
    freezer = Freezer(app=app, with_static_files=False, with_no_argument_rules=False, log_url_for=True)
    freezer.register_generator(no_argument_rules_urls_with_ignore)

    print("Freezing")
    freezer.freeze()

def main():
    if sys.argv:
        is_assets = False
        is_pages = False
        for arg in sys.argv:
            if arg == '--assets' or arg == '-a':
                is_assets = True
            elif arg == "--pages" or arg == "-p":
                is_pages = True
    else:
        is_assets = True
        is_pages = True
    if is_assets:
        build_assets()
    if is_pages:
        build_pages()

main()
<commit_msg>Fix build.py so that it *does* build stuff without arguments<commit_after>import os
import sys

from flask.ext.frozen import Freezer
import webassets

from content import app, assets

def build_assets():
    for bundle in assets:
        assert isinstance(bundle, webassets.Bundle)
        print("Building bundle {}".format(bundle.output))
        bundle.build(force=True, disable_cache=True)

# This is a copy of Freezer.no_argument_rules() modified to ignore certain paths
def no_argument_rules_urls_with_ignore():
    """URL generator for URL rules that take no arguments."""
    ignored = app.config.get('FREEZER_IGNORE_ENDPOINTS', [])
    for rule in app.url_map.iter_rules():
        if rule.endpoint not in ignored and not rule.arguments and 'GET' in rule.methods:
            yield rule.endpoint, {}

def build_pages():
    ignored = [bundle.output for bundle in assets]
    app.config['FREEZER_DESTINATION'] = os.path.abspath("static")
    app.config['FREEZER_BASE_URL'] = "http://dabo.guru/"
    app.config['FREEZER_DESTINATION_IGNORE'] = ignored
    app.config['FREEZER_IGNORE_ENDPOINTS'] = ['oauth_respond', 'mc_api_name', 'mc_api_uuid', 'serve_markdown']
    freezer = Freezer(app=app, with_static_files=False, with_no_argument_rules=False, log_url_for=True)
    freezer.register_generator(no_argument_rules_urls_with_ignore)

    print("Freezing")
    freezer.freeze()

def main():
    args = sys.argv[1:]
    if args:
        is_assets = False
        is_pages = False
        for arg in args:
            if arg == '--assets' or arg == '-a':
                is_assets = True
            elif arg == "--pages" or arg == "-p":
                is_pages = True
            else:
                print("Unknown argument '{}'".format(arg))
                print("Valid arguments: --assets -a --pages -p")
                return
    else:
        is_assets = True
        is_pages = True

    if is_assets:
        build_assets()
    if is_pages:
        build_pages()

main()
<|endoftext|>


<commit_before>from ..mcapp import app
from ..decorators import crossdomain, apikey, jsonp
from flask import jsonify, g, request, send_from_directory
from ..utils import mkdirp
import rethinkdb as r
import os.path
import os
from ..args import json_as_format_arg
import tempfile
from .. import access
from .. import error
from .. import dmutil
from .. import validate
from .. import mcdir
from loader.model import datafile
from mcapi import mcexceptions
import traceback

class StateCreateSaver(object):
    def __init__(self):
        self.objects = {}

    def insert(self, table, entry):
        rv = r.table('saver').insert(entry).run(g.conn)
        id = rv['generated_keys'][0]
        self.objects[id] = table
        return id

    def insert_newval(self, table, entry):
        rv = r.table('saver').insert(entry, return_vals=True).run(g.conn)
        id = rv['generated_keys'][0]
        self.objects[id] = table
        return rv

    def move_to_tables(self):
        for key in self.objects:
            table_name = self.objects[key]
            o = r.table('saver').get(key).run(g.conn, time_format='raw')
            r.table(table_name).insert(o).run(g.conn)

    def delete_tables(self):
        for key in self.objects:
            r.table('saver').get(key).delete().run(g.conn)
        self.objects.clear()

@app.route('/udqueue')
@apikey
@jsonp
def get_udqueue():
    user = access.get_user()
    selection = list(r.table('udqueue').filter({'owner': user}).run(g.conn))
    return json_as_format_arg(selection)

@app.route('/upload', methods=['POST'])
@apikey
@crossdomain(origin='*')
def upload_state():
    user = access.get_user()
    j = request.get_json()
    state_id = dmutil.get_required('state_id', j)
    process_id = load_data_dir(user, state_id)
    if (process_id):
        return jsonify({'success': True, 'process': process_id})
    else:
        return error.bad_request('unable to create process')

def load_data_dir(user, state_id):
    state_saver = StateCreateSaver()
    try:
        process_id = load_data(user, state_id,state_saver)
        return process_id
    except mcexceptions.RequiredAttributeException as rae:
        traceback.print_exc()
        state_saver.delete_tables()
        print "Missing attribute: %s" % (rae.attr)
    except Exception as exc:
        traceback.print_exc()
        state_saver.delete_tables()
        raise load_data_dir.retry(exc=exc)
    finally:
        state_saver.delete_tables()

def load_data(user, state_id, state_saver):
    process_id = load_provenance_from_state(state_id, state_saver)
    r.table('drafts').get(state_id).delete().run(g.conn)
    state_saver.move_to_tables()
    state_saver.delete_tables()
    return process_id

def load_provenance_from_state(state_id, saver):
    state = r.table('drafts').get(state_id).run(g.conn)
    attributes = state['attributes']
    user = state['owner']
    project_id = attributes['project_id']
    saver.project_id = project_id
    create_process_from_template(attributes['process'], saver)
    process_id = saver.process_id
    if 'input_files' in attributes:
        r.table('saver').get(process_id).update({'input_files': attributes['input_files']}).run(g.conn)
    if 'output_files' in attributes:
        r.table('saver').get(process_id).update({'output_files': attributes['output_files']}).run(g.conn)
    input_conditions = dmutil.get_optional('input_conditions', attributes, [])
    output_conditions = dmutil.get_optional('output_conditions', attributes, [])
    create_conditions_from_templates_modified(process_id, user, input_conditions, output_conditions, saver)
    return process_id

def create_process_from_template(j, saver):
    project_id = saver.project_id
    p = dict()
    p['project'] = project_id
    p['name'] = dmutil.get_required('name', j)
    p['birthtime'] = r.now()
    p['mtime'] = p['birthtime']
    p['machine'] = dmutil.get_optional('machine', j)
    p['process_type'] = dmutil.get_optional('process_type', j)
    p['description'] = dmutil.get_optional('description', j)
    p['version'] = dmutil.get_optional('version', j)
    p['template'] = dmutil.get_required('template', j)
    p['notes'] = dmutil.get_optional('notes', j, [])
    p['input_conditions'] = dmutil.get_optional('input_conditions', j, [])
    p['input_files'] = dmutil.get_optional('input_files', j, [])
    p['output_conditions'] = dmutil.get_optional('output_conditions', j, [])
    p['output_files'] = dmutil.get_optional('output_files', j, [])
    p['runs'] = dmutil.get_optional('runs', j, [])
    p['citations'] = dmutil.get_optional('citations', j, [])
    p['status'] = dmutil.get_optional('status', j)
    process_id = saver.insert('processes', p)
    saver.process_id = process_id
    saver.insert('project2processes', {'project_id': project_id, 'process_id': process_id})

def create_conditions_from_templates(process_id, user, input_conditions, output_conditions, saver):
    for condition_name in input_conditions:
        condition = input_conditions[condition_name]
        condition[u'condition_type'] = 'input_conditions'
        create_condition_from_template_modified(process_id, user, condition, saver)
    for condition_name in output_conditions:
        condition = output_conditions[condition_name]
        condition[u'condition_type'] = 'output_conditions'
        create_condition_from_template_modified(process_id, user, condition, saver)

def create_condition_from_template(process_id, user, j, saver):
    c = dict()
    m = j['model']
    type_of_condition = dmutil.get_required('condition_type', j)
    c['owner'] = user
    c['material'] = dmutil.get_optional('material', j)
    c['template'] = dmutil.get_required('id', j)
    c['name'] = dmutil.get_required('name', j) #dmutil.get_required('template_name', j) = every condition instance should have its own name
    c['description'] = dmutil.get_optional('description', j)
    c['model'] = list()
    for attr in m:
        add_model_item(c, attr['name'],attr['value'], attr['unit'], attr['value_choice'],attr['unit_choice'],attr['type'])
        #c[attr['name']] = attr['value']
    c_id = saver.insert('conditions', c)
    new_conditions = r.table('saver').get(process_id)[type_of_condition].append(c_id).run(g.conn)
    r.table('saver').get(process_id).update({type_of_condition:new_conditions}).run(g.conn)

def add_model_item(c, name, value,unit, value_choice, unit_choice,value_type):
    c['model'].append({'name':name, 'value':value,'unit':unit, 'value_choice':value_choice, 'unit_choice':unit_choice, 'type':value_type})

def create_conditions_from_templates_modified(process_id, user, input_conditions, output_conditions, saver):
    for key  in input_conditions:
        values = input_conditions[key]
        values['condition_type'] = 'input_conditions'
        create_condition_from_template_modified(process_id, user, values, saver)
    for key  in output_conditions:
        values = output_conditions[key]
        values['condition_type'] = 'output_conditions'
        create_condition_from_template_modified(process_id, user, values, saver)

def create_condition_from_template_modified(process_id, user, j, saver):
    c = dict()
    type_of_condition = dmutil.get_required('condition_type', j)
    c['owner'] = dmutil.get_optional('owner', j)
    c['material'] = dmutil.get_optional('material', j)
    c['model'] = dmutil.get_optional('model', j)
    c['template'] = dmutil.get_required('template_name', j)
    c_id = saver.insert('conditions', c)
    new_conditions = r.table('saver').get(process_id)[type_of_condition].append(c_id).run(g.conn)
    r.table('saver').get(process_id).update({type_of_condition:new_conditions}).run(g.conn)

@app.route('/import', methods=['POST'])
@apikey
@crossdomain(origin='*')
def import_file():
    user = access.get_user()
    if 'file' not in request.files:
        return error.bad_request("No files to import")
    elif 'project' not in request.form:
        return error.bad_request('No project specified')
    elif 'datadir' not in request.form:
        return error.bad_request('No datadir specified')
    project = request.form['project']
    datadir = request.form['datadir']
    file = request.files['file']
    proj = validate.project_id_exists(project, user)
    if proj is None:
        return error.bad_request("Project doesn't exist %s" % (project))
    ddir = validate.datadir_id_exists(datadir, user)
    if ddir is None:
        return error.bad_request(
            "Datadir doesn't exist %s" % (datadir))
    if not datadir_in_project(ddir, proj):
        return error.bad_request(
            "Datadir %s is not in project %s" % (datadir, project))
    if filename_in_datadir(ddir, file.filename):
        return error.bad_request(
            "File %s already exists in datadir %s" % (file.filename, datadir))
    dfid = make_datafile(datadir, user, file.filename)
    filepath = os.path.join(mcdir.for_uid(dfid), dfid)
    file.save(filepath)
    #load_data_file.delay(df, project, datadir)
    return jsonify({'id': dfid})

def datadir_in_project(ddir, proj):
    filter_by = {'project_id': proj['id'], 'datadir_id': ddir['id']}
    selection = list(r.table('project2datadir').filter(filter_by).run(g.conn))
    if selection:
        return True
    return False

def filename_in_datadir(ddir, filename):
    files = list(r.table('datafiles').filter({'name': filename}).run(g.conn))
    if not files:
        return False
    for file in files:
        for ddir_id in file['datadirs']:
            if ddir_id == ddir['id']:
                return True
    return False

def make_datafile(datadir, user, filename):
    df = datafile.DataFile(os.path.basename(filename), "private", user)
    df.datadirs.append(datadir)
    dfid = dmutil.insert_entry_id('datafiles', df.__dict__)
    ddir = r.table('datadirs').get(datadir).run(g.conn)
    dfiles = ddir['datafiles']
    dfiles.append(dfid)
    r.table('datadirs').get(datadir).update({'datafiles': dfiles}).run(g.conn)
    return dfid

@app.route('/download/<path:datafile>')
#@apikey
def download_file(datafile):
    #user = access.get_user()
    return send_from_directory('/tmp', 'ReviewQueue.png', as_attachment=True)
    #df = r.table('datafiles').get(datafile).run(g.conn)
    #if not checkAccess(user, df):
    #   return error_not_found_response()
    #return None

@app.route('/upload12', methods=['POST'])
@apikey
@crossdomain(origin='*')
def upload_file():
    user = access.get_user()
    state_id = request.form['state_id']
    mkdirp('/tmp/uploads')
    tdir = tempfile.mkdtemp(dir='/tmp/uploads')
    for key in request.files.keys():
        datadir = request.form[key + "_datadir"]
        file = request.files[key]
        dir = os.path.join(tdir, datadir)
        mkdirp(dir)
        filepath = os.path.join(dir, file.filename)
        file.save(filepath)
    chain(load_data_dir.si(user, tdir, state_id)\
          | import_data_dir_to_repo.si(tdir))()
    return jsonify({'success': True})

<commit_msg>updated templates to use new sample code<commit_after>from ..mcapp import app
from ..decorators import crossdomain, apikey, jsonp
from flask import jsonify, g, request, send_from_directory
from ..utils import mkdirp
import rethinkdb as r
import os.path
import os
from ..args import json_as_format_arg
import tempfile
from .. import access
from .. import error
from .. import dmutil
from .. import validate
from .. import mcdir
from loader.model import datafile
from mcapi import mcexceptions
import traceback

class StateCreateSaver(object):
    def __init__(self):
        self.objects = {}

    def insert(self, table, entry):
        rv = r.table('saver').insert(entry).run(g.conn)
        id = rv['generated_keys'][0]
        self.objects[id] = table
        return id

    def insert_newval(self, table, entry):
        rv = r.table('saver').insert(entry, return_vals=True).run(g.conn)
        id = rv['generated_keys'][0]
        self.objects[id] = table
        return rv

    def move_to_tables(self):
        for key in self.objects:
            table_name = self.objects[key]
            o = r.table('saver').get(key).run(g.conn, time_format='raw')
            r.table(table_name).insert(o).run(g.conn)

    def delete_tables(self):
        for key in self.objects:
            r.table('saver').get(key).delete().run(g.conn)
        self.objects.clear()

@app.route('/udqueue')
@apikey
@jsonp
def get_udqueue():
    user = access.get_user()
    selection = list(r.table('udqueue').filter({'owner': user}).run(g.conn))
    return json_as_format_arg(selection)

@app.route('/upload', methods=['POST'])
@apikey
@crossdomain(origin='*')
def upload_state():
    user = access.get_user()
    j = request.get_json()
    state_id = dmutil.get_required('state_id', j)
    process_id = load_data_dir(user, state_id)
    if (process_id):
        return jsonify({'success': True, 'process': process_id})
    else:
        return error.bad_request('unable to create process')

def load_data_dir(user, state_id):
    state_saver = StateCreateSaver()
    try:
        process_id = load_data(user, state_id,state_saver)
        return process_id
    except mcexceptions.RequiredAttributeException as rae:
        traceback.print_exc()
        state_saver.delete_tables()
        print "Missing attribute: %s" % (rae.attr)
    except Exception as exc:
        traceback.print_exc()
        state_saver.delete_tables()
        raise load_data_dir.retry(exc=exc)
    finally:
        state_saver.delete_tables()

def load_data(user, state_id, state_saver):
    process_id = load_provenance_from_state(state_id, state_saver)
    r.table('drafts').get(state_id).delete().run(g.conn)
    state_saver.move_to_tables()
    state_saver.delete_tables()
    return process_id

def load_provenance_from_state(state_id, saver):
    state = r.table('drafts').get(state_id).run(g.conn)
    attributes = state['attributes']
    user = state['owner']
    project_id = attributes['project_id']
    saver.project_id = project_id
    create_process_from_template(attributes['process'], saver)
    process_id = saver.process_id
    if 'input_files' in attributes:
        r.table('saver').get(process_id).update({'input_files': attributes['input_files']}).run(g.conn)
    if 'output_files' in attributes:
        r.table('saver').get(process_id).update({'output_files': attributes['output_files']}).run(g.conn)
    input_conditions = dmutil.get_optional('input_conditions', attributes, [])
    output_conditions = dmutil.get_optional('output_conditions', attributes, [])
    create_conditions_from_templates_modified(process_id, user, input_conditions, output_conditions, saver)
    return process_id

def create_process_from_template(j, saver):
    project_id = saver.project_id
    p = dict()
    p['project'] = project_id
    p['name'] = dmutil.get_required('name', j)
    p['birthtime'] = r.now()
    p['mtime'] = p['birthtime']
    p['machine'] = dmutil.get_optional('machine', j)
    p['process_type'] = dmutil.get_optional('process_type', j)
    p['description'] = dmutil.get_optional('description', j)
    p['version'] = dmutil.get_optional('version', j)
    p['template'] = dmutil.get_required('template', j)
    p['notes'] = dmutil.get_optional('notes', j, [])
    p['input_conditions'] = dmutil.get_optional('input_conditions', j, [])
    p['input_files'] = dmutil.get_optional('input_files', j, [])
    p['output_conditions'] = dmutil.get_optional('output_conditions', j, [])
    p['output_files'] = dmutil.get_optional('output_files', j, [])
    p['runs'] = dmutil.get_optional('runs', j, [])
    p['citations'] = dmutil.get_optional('citations', j, [])
    p['status'] = dmutil.get_optional('status', j)
    process_id = saver.insert('processes', p)
    saver.process_id = process_id
    saver.insert('project2processes', {'project_id': project_id, 'process_id': process_id})

def create_conditions_from_templates(process_id, user, input_conditions, output_conditions, saver):
    for condition_name in input_conditions:
        condition = input_conditions[condition_name]
        condition[u'condition_type'] = 'input_conditions'
        create_condition_from_template_modified(process_id, user, condition, saver)
    for condition_name in output_conditions:
        condition = output_conditions[condition_name]
        condition[u'condition_type'] = 'output_conditions'
        create_condition_from_template_modified(process_id, user, condition, saver)

def create_condition_from_template(process_id, user, j, saver):
    c = dict()
    m = j['model']
    type_of_condition = dmutil.get_required('condition_type', j)
    c['owner'] = user
    c['material'] = dmutil.get_optional('material', j)
    c['template'] = dmutil.get_required('id', j)
    c['name'] = dmutil.get_required('name', j) #dmutil.get_required('template_name', j) = every condition instance should have its own name
    c['description'] = dmutil.get_optional('description', j)
    c['model'] = list()
    for attr in m:
        add_model_item(c, attr['name'],attr['value'], attr['unit'], attr['value_choice'],attr['unit_choice'],attr['type'])
        #c[attr['name']] = attr['value']
    c_id = saver.insert('conditions', c)
    new_conditions = r.table('saver').get(process_id)[type_of_condition].append(c_id).run(g.conn)
    r.table('saver').get(process_id).update({type_of_condition:new_conditions}).run(g.conn)

def add_model_item(c, name, value,unit, value_choice, unit_choice,value_type):
    c['model'].append({'name':name, 'value':value,'unit':unit, 'value_choice':value_choice, 'unit_choice':unit_choice, 'type':value_type})

def create_conditions_from_templates_modified(process_id, user, input_conditions, output_conditions, saver):
    for key  in input_conditions:
        values = input_conditions[key]
        values['condition_type'] = 'input_conditions'
        create_condition_from_template_modified(process_id, user, values, saver)
    for key  in output_conditions:
        values = output_conditions[key]
        values['condition_type'] = 'output_conditions'
        create_condition_from_template_modified(process_id, user, values, saver)

def create_condition_from_template_modified(process_id, user, j, saver):
    c = dict()
    type_of_condition = dmutil.get_required('condition_type', j)
    c['owner'] = dmutil.get_optional('owner', j)
    c['material'] = dmutil.get_optional('material', j)
    c['model'] = dmutil.get_optional('model', j)
    c['template'] = dmutil.get_required('template_name', j)
    c['sample_id'] = dmutil.get_optional('sample_id', j)
    c_id = saver.insert('conditions', c)
    new_conditions = r.table('saver').get(process_id)[type_of_condition].append(c_id).run(g.conn)
    r.table('saver').get(process_id).update({type_of_condition:new_conditions}).run(g.conn)

@app.route('/import', methods=['POST'])
@apikey
@crossdomain(origin='*')
def import_file():
    user = access.get_user()
    if 'file' not in request.files:
        return error.bad_request("No files to import")
    elif 'project' not in request.form:
        return error.bad_request('No project specified')
    elif 'datadir' not in request.form:
        return error.bad_request('No datadir specified')
    project = request.form['project']
    datadir = request.form['datadir']
    file = request.files['file']
    proj = validate.project_id_exists(project, user)
    if proj is None:
        return error.bad_request("Project doesn't exist %s" % (project))
    ddir = validate.datadir_id_exists(datadir, user)
    if ddir is None:
        return error.bad_request(
            "Datadir doesn't exist %s" % (datadir))
    if not datadir_in_project(ddir, proj):
        return error.bad_request(
            "Datadir %s is not in project %s" % (datadir, project))
    if filename_in_datadir(ddir, file.filename):
        return error.bad_request(
            "File %s already exists in datadir %s" % (file.filename, datadir))
    dfid = make_datafile(datadir, user, file.filename)
    filepath = os.path.join(mcdir.for_uid(dfid), dfid)
    file.save(filepath)
    #load_data_file.delay(df, project, datadir)
    return jsonify({'id': dfid})

def datadir_in_project(ddir, proj):
    filter_by = {'project_id': proj['id'], 'datadir_id': ddir['id']}
    selection = list(r.table('project2datadir').filter(filter_by).run(g.conn))
    if selection:
        return True
    return False

def filename_in_datadir(ddir, filename):
    files = list(r.table('datafiles').filter({'name': filename}).run(g.conn))
    if not files:
        return False
    for file in files:
        for ddir_id in file['datadirs']:
            if ddir_id == ddir['id']:
                return True
    return False

def make_datafile(datadir, user, filename):
    df = datafile.DataFile(os.path.basename(filename), "private", user)
    df.datadirs.append(datadir)
    dfid = dmutil.insert_entry_id('datafiles', df.__dict__)
    ddir = r.table('datadirs').get(datadir).run(g.conn)
    dfiles = ddir['datafiles']
    dfiles.append(dfid)
    r.table('datadirs').get(datadir).update({'datafiles': dfiles}).run(g.conn)
    return dfid

@app.route('/download/<path:datafile>')
#@apikey
def download_file(datafile):
    #user = access.get_user()
    return send_from_directory('/tmp', 'ReviewQueue.png', as_attachment=True)
    #df = r.table('datafiles').get(datafile).run(g.conn)
    #if not checkAccess(user, df):
    #   return error_not_found_response()
    #return None

@app.route('/upload12', methods=['POST'])
@apikey
@crossdomain(origin='*')
def upload_file():
    user = access.get_user()
    state_id = request.form['state_id']
    mkdirp('/tmp/uploads')
    tdir = tempfile.mkdtemp(dir='/tmp/uploads')
    for key in request.files.keys():
        datadir = request.form[key + "_datadir"]
        file = request.files[key]
        dir = os.path.join(tdir, datadir)
        mkdirp(dir)
        filepath = os.path.join(dir, file.filename)
        file.save(filepath)
    chain(load_data_dir.si(user, tdir, state_id)\
          | import_data_dir_to_repo.si(tdir))()
    return jsonify({'success': True})

<|endoftext|>


<commit_before>#!/usr/bin/env python
import os, sys, subprocess, tempfile, shutil
import getopt
import json, ConfigParser
import xml.etree.ElementTree as ET
import zipfile

NAME = "jsonview"
VERSION = json.load(open("package.json"))["version"]
XPI_NAME = "{}-{}.xpi".format(NAME, VERSION)

## Helpers ##

# Search for profile dir
def getProfileDir(profileName):
  if os.name == "nt":
    homeDir = os.path.join(os.getenv("APPDATA"), "Mozilla", "Firefox")
  else:
    homeDir = os.path.join(os.path.expanduser("~"), ".mozilla", "firefox")

  profilesPath = os.path.join(homeDir, "profiles.ini")
  config = ConfigParser.ConfigParser()
  config.read(profilesPath)

  for section in config.sections():
    if section.startswith("Profile") \
        and config.get(section, "Name") == profileName:
      return os.path.join(homeDir, config.get(section, "Path"))
  return None

# Copy all localized description tags
def copyLocalizedDescription(source, target):
  rdfNamespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
  emNamespace = "http://www.mozilla.org/2004/em-rdf#"

  targetTree = ET.parse(target)
  targetDesc = targetTree.getroot().find("./{{{}}}Description" \
      .format(rdfNamespace))

  sourceTree = ET.parse(source)
  sourceTagName = "./{{{}}}Description/{{{}}}localized" \
      .format(rdfNamespace, emNamespace)

  for a in sourceTree.getroot().findall(sourceTagName):
    targetDesc.append(a)

  ET.register_namespace("", rdfNamespace)
  ET.register_namespace("em", emNamespace)
  targetTree.write(target, "utf-8", True)

# Unpack a xpi to a folder
def unpackXpi(source, target):
  xpi = zipfile.ZipFile(source, "r")
  xpi.extractall(target)
  xpi.close()

# Recreate a xpi file from a folder
def packXpi(source, target):
  xpi = zipfile.ZipFile(target, "w", zipfile.ZIP_DEFLATED)

  for dirPath, dirNames, fileNames in os.walk(source):
    for name in fileNames:
      filePath = os.path.join(dirPath, name)
      relPath = os.path.relpath(filePath, source)
      xpi.write(filePath, relPath)

  xpi.close()

# Generate XPI file
def createXpi(xpiName):
  args = ["cfx", "xpi", "--output-file=" + xpiName]
  return subprocess.call(args)

# Run in browser
def runBrowser(profileDir):
  args = ["cfx", "run"]
  if profileDir is not None:
    args.append("-p")
    args.append(profileDir)

  return subprocess.call(args)

# Mozilla bug 661083
# Addon metadata can not be localized. It requires unpacking the xpi,
# updating the install.rdf and compressing everything again.
def fixLocalizedDescription(xpiName):
  if not os.path.isfile(xpiName):
    print("File {} must be created before running this task")
    return 1

  targetDir = tempfile.mkdtemp()
  targetFile = "install.rdf"

  unpackXpi(XPI_NAME, targetDir)
  copyLocalizedDescription(os.path.join("src", targetFile), \
      os.path.join(targetDir, targetFile))
  packXpi(targetDir, "fixed-" + xpiName)

  # Cleanup
  shutil.rmtree(targetDir)
  return 0

# Print usage
def printUsage():
  print("Usage: {} [OPTIONS...] COMMAND".format(sys.argv[0]))
  print("""
COMMANDS:
  run     Run the addon
  xpi     Create xpi file
  fix     Fix localized description in generated xpi

OPTIONS:
  -p PROFILE, --profile=PROFILE
          Used with command 'run'. Open the addon with the specific browser
          profile. PROFILE can be either an absolute path or a profile name
          (i.e. 'dev'), which then be translated to profile path in
          ~/.mozilla/firefox
""")

## Tasks ##
def help(opts):
  printUsage()

def run(opts):
  profileDir = None

  for o, a in opts:
    if o in ("-p", "--profile"):
      profileDir = a
    else:
      print("Unhandled option: {}".format(o))
      printUsage()
      return 1

  # If profile is just a name, search for full path
  if profileDir is not None \
      and "/" not in profileDir \
      and "\\" not in profileDir:
    profileName = profileDir
    profileDir = getProfileDir(profileName)
    if profileDir is None:
      print("Profile '{}' does not exist.".format(profileName))
      return 1

  return runBrowser(profileDir)

def xpi(opts):
  return createXpi(XPI_NAME)

def fix(opts):
  targetDir = tempfile.mkdtemp()
  targetFile = "install.rdf"

  unpackXpi(XPI_NAME, targetDir)
  copyLocalizedDescription(os.path.join("src", targetFile), \
      os.path.join(targetDir, targetFile))
  packXpi(targetDir, "fixed-" + XPI_NAME)

  # Cleanup
  shutil.rmtree(targetDir)
  return 0

if __name__ == "__main__":
  try:
    opts, args = getopt.getopt(sys.argv[1:], "p:", ["profile="])
  except getopt.GetoptError as err:
    print(str(err))
    printUsage()
    sys.exit(1)

  if not args:
    printUsage()
    sys.exit(1)

  for cmd in args:
    if cmd in locals():
      print("Running task: {}...".format(cmd))
      locals()[cmd](opts)
    else:
      print("Command '{}' is not supported".format(cmd))
      printUsage()
      sys.exit(1)
  sys.exit(0)
<commit_msg>Docstrings<commit_after>#!/usr/bin/env python
import os, sys, subprocess, tempfile, shutil
import getopt
import json, ConfigParser
import xml.etree.ElementTree as ET
import zipfile

NAME = "jsonview"
VERSION = json.load(open("package.json"))["version"]
XPI_NAME = "{}-{}.xpi".format(NAME, VERSION)

## Helpers ##

def getProfileDir(profileName):
  """
  Search for profile dir
  """
  if os.name == "nt":
    homeDir = os.path.join(os.getenv("APPDATA"), "Mozilla", "Firefox")
  else:
    homeDir = os.path.join(os.path.expanduser("~"), ".mozilla", "firefox")

  profilesPath = os.path.join(homeDir, "profiles.ini")
  config = ConfigParser.ConfigParser()
  config.read(profilesPath)

  for section in config.sections():
    if section.startswith("Profile") \
        and config.get(section, "Name") == profileName:
      return os.path.join(homeDir, config.get(section, "Path"))
  return None

def copyLocalizedDescription(source, target):
  """
  Copy all localized description tags
  """
  rdfNamespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
  emNamespace = "http://www.mozilla.org/2004/em-rdf#"

  targetTree = ET.parse(target)
  targetDesc = targetTree.getroot().find("./{{{}}}Description" \
      .format(rdfNamespace))

  sourceTree = ET.parse(source)
  sourceTagName = "./{{{}}}Description/{{{}}}localized" \
      .format(rdfNamespace, emNamespace)

  for a in sourceTree.getroot().findall(sourceTagName):
    targetDesc.append(a)

  ET.register_namespace("", rdfNamespace)
  ET.register_namespace("em", emNamespace)
  targetTree.write(target, "utf-8", True)

def unpackXpi(source, target):
  """
  Unpack a xpi to a folder
  """
  xpi = zipfile.ZipFile(source, "r")
  xpi.extractall(target)
  xpi.close()

def packXpi(source, target):
  """
  Recreate a xpi file from a folder
  """
  xpi = zipfile.ZipFile(target, "w", zipfile.ZIP_DEFLATED)

  for dirPath, dirNames, fileNames in os.walk(source):
    for name in fileNames:
      filePath = os.path.join(dirPath, name)
      relPath = os.path.relpath(filePath, source)
      xpi.write(filePath, relPath)

  xpi.close()

def createXpi(xpiName):
  """
  Generate XPI file
  """
  args = ["cfx", "xpi", "--output-file=" + xpiName]
  return subprocess.call(args)

def runBrowser(profileDir):
  """
  Run in browser
  """
  args = ["cfx", "run"]
  if profileDir is not None:
    args.append("-p")
    args.append(profileDir)

  return subprocess.call(args)

def fixLocalizedDescription(xpiName):
  """
  Mozilla bug 661083
  Addon metadata can not be localized. It requires unpacking the xpi,
  updating the install.rdf and compressing everything again.
  """
  if not os.path.isfile(xpiName):
    print("File {} must be created before running this task")
    return 1

  targetDir = tempfile.mkdtemp()
  targetFile = "install.rdf"

  unpackXpi(XPI_NAME, targetDir)
  copyLocalizedDescription(os.path.join("src", targetFile), \
      os.path.join(targetDir, targetFile))
  packXpi(targetDir, "fixed-" + xpiName)

  # Cleanup
  shutil.rmtree(targetDir)
  return 0

def printUsage():
  """
  Print usage
  """
  print("Usage: {} [OPTIONS...] COMMAND".format(sys.argv[0]))
  print("""
COMMANDS:
  run     Run the addon
  xpi     Create xpi file
  fix     Fix localized description in generated xpi

OPTIONS:
  -p PROFILE, --profile=PROFILE
          Used with command 'run'. Open the addon with the specific browser
          profile. PROFILE can be either an absolute path or a profile name
          (i.e. 'dev'), which then be translated to profile path in
          ~/.mozilla/firefox
""")

## Tasks ##
def help(opts):
  printUsage()

def run(opts):
  profileDir = None

  for o, a in opts:
    if o in ("-p", "--profile"):
      profileDir = a
    else:
      print("Unhandled option: {}".format(o))
      printUsage()
      return 1

  # If profile is just a name, search for full path
  if profileDir is not None \
      and "/" not in profileDir \
      and "\\" not in profileDir:
    profileName = profileDir
    profileDir = getProfileDir(profileName)
    if profileDir is None:
      print("Profile '{}' does not exist.".format(profileName))
      return 1

  return runBrowser(profileDir)

def xpi(opts):
  return createXpi(XPI_NAME)

def fix(opts):
  targetDir = tempfile.mkdtemp()
  targetFile = "install.rdf"

  unpackXpi(XPI_NAME, targetDir)
  copyLocalizedDescription(os.path.join("src", targetFile), \
      os.path.join(targetDir, targetFile))
  packXpi(targetDir, "fixed-" + XPI_NAME)

  # Cleanup
  shutil.rmtree(targetDir)
  return 0

if __name__ == "__main__":
  try:
    opts, args = getopt.getopt(sys.argv[1:], "p:", ["profile="])
  except getopt.GetoptError as err:
    print(str(err))
    printUsage()
    sys.exit(1)

  if not args:
    printUsage()
    sys.exit(1)

  for cmd in args:
    if cmd in locals():
      print("Running task: {}...".format(cmd))
      locals()[cmd](opts)
    else:
      print("Command '{}' is not supported".format(cmd))
      printUsage()
      sys.exit(1)
  sys.exit(0)
<|endoftext|>


<commit_before>import shutil
from codecs import open
import json

__author__ = 'Samuel Gratzl'

def _git_head(cwd):
  import subprocess
  try:
    output = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD'], cwd=cwd)
    return output.strip()
  except subprocess.CalledProcessError:
    return 'error'

def _resolve_plugin(repo, version):
  import os.path
  if os.path.isdir('.git') and repo:
    if repo.endswith('.git'):
      repo = repo[0:-4]
    return repo + '/commit/' + _git_head('.')
  # not a git repo
  return version

def to_version(v):
  import datetime
  now = datetime.datetime.utcnow()
  return v.replace('SNAPSHOT', now.strftime('%Y%m%d-%H%M%S'))

with open('package.json', 'r', encoding='utf-8') as f:
  pkg = json.load(f)

name = pkg['name']
version = to_version(pkg['version'])
resolved = _resolve_plugin(pkg.get('repository', {}).get('url'), version)

# copy source code
shutil.rmtree('build/source/' + name, ignore_errors=True)
shutil.copytree(name, 'build/source/' + name, symlinks=False, ignore=shutil.ignore_patterns('*.pyc'))

# create buildInfo.json
build_info = dict(name=name, version=version, resolved=resolved, description=pkg['description'],
                  homepage=pkg.get('homepage'), repository=pkg.get('repository', {}).get('url'))

# TODO create build Info

with open('build/source/'+ name +'/buildInfo.json', 'w', encoding='utf-8') as f:
  json.dump(build_info, f, indent=2)
<commit_msg>fix flake8<commit_after>import shutil
from codecs import open
import json

__author__ = 'Samuel Gratzl'

def _git_head(cwd):
  import subprocess
  try:
    output = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD'], cwd=cwd)
    return output.strip()
  except subprocess.CalledProcessError:
    return 'error'

def _resolve_plugin(repo, version):
  import os.path
  if os.path.isdir('.git') and repo:
    if repo.endswith('.git'):
      repo = repo[0:-4]
    return repo + '/commit/' + _git_head('.')
  # not a git repo
  return version

def to_version(v):
  import datetime
  now = datetime.datetime.utcnow()
  return v.replace('SNAPSHOT', now.strftime('%Y%m%d-%H%M%S'))

with open('package.json', 'r', encoding='utf-8') as f:
  pkg = json.load(f)

name = pkg['name']
version = to_version(pkg['version'])
resolved = _resolve_plugin(pkg.get('repository', {}).get('url'), version)

# copy source code
shutil.rmtree('build/source/' + name, ignore_errors=True)
shutil.copytree(name, 'build/source/' + name, symlinks=False, ignore=shutil.ignore_patterns('*.pyc'))

# create buildInfo.json
build_info = dict(name=name, version=version, resolved=resolved, description=pkg['description'],
                  homepage=pkg.get('homepage'), repository=pkg.get('repository', {}).get('url'))

# TODO create build Info

with open('build/source/' + name + '/buildInfo.json', 'w', encoding='utf-8') as f:
  json.dump(build_info, f, indent=2)
<|endoftext|>