<commit_before>#!/usr/bin/env python
"""
Fab file to help with managing project.  For docs on Fab file, please see: http://docs.fabfile.org/
"""
import sys
import os
import warnings
import json
import re
from fabric.api import *

"""
Base configuration
"""
env.project_name = 'minnpost-basemaps'
env.pg_host = 'localhost'
env.pg_dbname = 'minnpost_fec'
env.pg_user = 'postgres'
env.pg_pass = ''
env.labels = None

# Tilemill paths.  For Ubuntu
if os.path.exists('/usr/share/tilemill'):
  env.tilemill_path = '/usr/share/tilemill'
  env.tilemill_projects = '/usr/share/mapbox/project'
  env.node_path = '/usr/bin/node'
# OSX
else:
  env.tilemill_path = '/Applications/TileMill.app/Contents/Resources'
  env.tilemill_projects = '~/Documents/MapBox/project'
  env.node_path = '%(tilemill_path)s/node' % env

"""
Environments
"""
def production():
  """
  Work on production environment
  """
  env.settings = 'production'
  env.s3_buckets = ['a.tiles.minnpost', 'b.tiles.minnpost', 'c.tiles.minnpost', 'd.tiles.minnpost']
  env.s3_template = 'http://{s}.tiles.minnpost.s3.amazonaws.com'
  env.acl = 'acl-public'

def staging():
  """
  Work on staging environment
  """
  env.settings = 'staging'
  env.s3_buckets = ['testing.tiles.minnpost']
  env.s3_template = 'http://testing.tiles.minnpost.s3.amazonaws.com'
  env.acl = 'acl-public'

def map(name):
  """
  Select map to work on.
  """
  env.map = name

def deploy_to_s3(concurrency):
  """
  Deploy tiles to S3.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])
  env.concurrency = concurrency
  _create_map_suffix()

  # Deploy to many buckets (multi-dns-head mode)
  for bucket in env.s3_buckets:
    env.s3_bucket = bucket
    local('ivs3 %(map)s/tiles %(s3_bucket)s/%(project_name)s/%(map)s%(map_suffix)s --%(acl)s -c %(concurrency)s' % env)

def export_deploy(concurrency=32, minzoom=None, maxzoom=None):
  """
  Deploy a map. Optionally takes a concurrency parameter indicating how many files to upload simultaneously.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])

  create_exports()
  cleanup_exports()
  generate_mbtile(minzoom, maxzoom)
  generate_tiles_from_mbtile()
  generate_tilejson()
  deploy_to_s3(concurrency)
  reset_labels()

def link_caches():
  """
  Link local map cache to Mapbox cache to speed up mapnik conversion.
  """
  require('map', provided_by=[map])

  env.base_path = os.getcwd()
  exists = os.path.exists('%(base_path)s/%(map)s/cache' % env)
  if exists:
    print "There already a linked cache directory."
  else:
    env.tilemill_cache = os.path.expanduser('%(tilemill_projects)s/../cache/' % env)
    local(('ln -s %(tilemill_cache)s %(base_path)s/%(map)s/cache') % env)

def carto_to_mapnik():
  """
  Convert carto styles to mapnik configuration.
  """
  require('map', provided_by=[map])
  link_caches()
  local('%(tilemill_path)s/node_modules/carto/bin/carto %(map)s/project.mml > %(map)s/%(map)s.xml' % env)

def generate_mbtile(minzoom=None, maxzoom=None):
  """
  Generate MBtile.
  """
  require('map', provided_by=[map])

  # Read data from project mml
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)

    # Define config values
    env.minzoom = config['minzoom'] if minzoom == None else minzoom
    env.maxzoom = config['maxzoom'] if maxzoom == None else maxzoom
    env.bbox = '%f,%f,%f,%f' % (config['bounds'][0], config['bounds'][1], config['bounds'][2], config['bounds'][3])

    # Export
    local('%(tilemill_path)s/node %(tilemill_path)s/index.js export --format=mbtiles --minzoom=%(minzoom)s --maxzoom=%(maxzoom)s --bbox=%(bbox)s %(map)s %(map)s/exports/%(map)s.mbtiles' % env)

def generate_tiles_from_mbtile():
  """
  Generate MBtile.
  """
  require('map', provided_by=[map])
  read_project()

  exists = os.path.exists('%(map)s/exports/%(map)s.mbtiles' % env)
  if exists:
    with settings(warn_only=True):
      local('rm -rf %(map)s/tiles-tmp' % env)
      local('rm -rf %(map)s/tiles' % env)
      local('mb-util --scheme=tms %(map)s/exports/%(map)s.mbtiles %(map)s/tiles-tmp' % env)
      local('mv "%(map)s/tiles-tmp/%(map_version)s/%(map_title)s" %(map)s/tiles' % env)
      local('mv %(map)s/tiles-tmp/metadata.json %(map)s/tiles/metadata.json' % env)
  else:
    print 'No MBTile file found in exports.'

def generate_tilejson():
  """
  Generate valid tilejson file.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])
  _create_map_suffix()

  # Utilize project data
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)
    tilejson = {}

    # Base values
    tilejson['scheme'] = 'tms'
    tilejson['tilejson'] = '2.0.0'

    # Attempt to get values from config
    try:
      tilejson['name'] = config['name'] if config.has_key('name') else ''
      tilejson['description'] = config['description'] if config.has_key('description') else ''
      tilejson['version'] = config['version'] if config.has_key('version') else '1.0.0'
      tilejson['attribution'] = config['attribution'] if config.has_key('attribution') else ''
      tilejson['legend'] = config['legend'] if config.has_key('legend') else ''
      tilejson['template'] = config['template'] if config.has_key('template') else ''
      tilejson['minzoom'] = config['minzoom'] if config.has_key('minzoom') else 0
      tilejson['maxzoom'] = config['maxzoom'] if config.has_key('maxzoom') else 22
      tilejson['bounds'] = config['bounds'] if config.has_key('bounds') else [-180, -90, 180, 90]
      tilejson['center'] = config['center'] if config.has_key('center') else null
    except KeyError:
      print 'Key error'

    # Figure out template
    tilejson['grids'] = []
    tilejson['tiles'] = []
    for bucket in env.s3_buckets:
      env.s3_bucket = bucket
      tilejson['grids'].append('http://%(s3_bucket)s.s3.amazonaws.com/%(project_name)s/%(map)s%(map_suffix)s/{z}/{x}/{y}.grid.json' % env)
      tilejson['tiles'].append('http://%(s3_bucket)s.s3.amazonaws.com/%(project_name)s/%(map)s%(map_suffix)s/{z}/{x}/{y}.png' % env)

    print tilejson

    # Write regular and jsonp tilejson files
    tilejson_file = open('%(map)s/tiles/tilejson.json' % env, 'w')
    tilejson_file.write(json.dumps(tilejson, sort_keys = True, indent = 2))
    tilejson_file.close()
    tilejsonp_file = open('%(map)s/tiles/tilejson.jsonp' % env, 'w')
    tilejsonp_file.write('grid(%s)' % json.dumps(tilejson, sort_keys = True, indent = 2))
    tilejsonp_file.close()

def generate_tiles_mapnik(process_count, minzoom=None, maxzoom=None):
  """
  Render tile from mapnik configuration
  """
  env.process_count = process_count

  # Read data from project mml
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)

    # Define config values
    env.minzoom = config['minzoom'] if minzoom == None else minzoom
    env.maxzoom = config['maxzoom'] if maxzoom == None else maxzoom
    env.minlon = config['bounds'][0]
    env.minlat = config['bounds'][1]
    env.maxlon = config['bounds'][2]
    env.maxlat = config['bounds'][3]

    # Cleanup tiles
    local('rm -rf %(map)s/tiles/*' % env)

    # Render tiles
    command = 'ivtile %(map)s/%(map)s.xml %(map)s/tiles %(maxlat)s %(minlon)s %(minlat)s %(maxlon)s %(minzoom)s %(maxzoom)s -p %(process_count)s'
    if 'buffer' in env:
      command += ' -b %(buffer)s'
    local(command % env)

def no_labels():
  """
  Updates tilemill project to no use labels
  """
  require('map', provided_by=[map])
  env.labels = False

  # Create a backup
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists != True:
    local('cp %(map)s/project.mml %(map)s/project.mml.orig' % env);

  # From the original, load the json, find any label style
  # and remove.
  overwrite = open('%(map)s/project.mml' % env, 'w')
  with open('%(map)s/project.mml.orig' % env, 'r') as f:
    mml = json.load(f)

    # Process styles
    styles = []
    for index, item in enumerate(mml['Stylesheet']):
      if item != 'labels.mss':
        styles.append(item)
    mml['Stylesheet'] = styles

    overwrite.write(json.dumps(mml, sort_keys = True, indent = 2))
    overwrite.close()

def only_labels():
  """
  Updates tilemill project to use only labels
  """
  require('map', provided_by=[map])
  env.labels = True

  # Create a backup
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists != True:
    local('cp %(map)s/project.mml %(map)s/project.mml.orig' % env);

  # From the original, load the json, keep palette and labels
  overwrite = open('%(map)s/project.mml' % env, 'w')
  with open('%(map)s/project.mml.orig' % env, 'r') as f:
    mml = json.load(f)

    # Process styles
    styles = []
    for index, item in enumerate(mml['Stylesheet']):
      if item == 'labels.mss' or item == 'palette.mss':
        styles.append(item)
    mml['Stylesheet'] = styles

    overwrite.write(json.dumps(mml, sort_keys = True, indent = 2))
    overwrite.close()

def reset_labels():
  """
  Resets any label changes process
  """
  require('map', provided_by=[map])

  # Check for orig
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists:
    local('rm -f %(map)s/project.mml' % env);
    local('mv %(map)s/project.mml.orig %(map)s/project.mml' % env);
  else:
    print 'No label processing to reset.'

def _create_map_suffix():
  """
  Creates map suffix for deploying
  """
  if env.labels == None:
    env.map_suffix = ''
  if env.labels == True:
    env.map_suffix = '-labels'
  if env.labels == False:
    env.map_suffix = '-no-labels'

def read_project():
  """
  Get data from the TileMill project, to be used in other
  commands.
  """
  require('map', provided_by=[map])

  # Read JSON from file
  with open('%(map)s/project.mml' % env, 'r') as f:
    mml = json.load(f)
    env.map_title = mml['name']

    # Version is not always defined
    try:
      env.map_version = mml['version']
    except KeyError:
      env.map_version = '1.0.0'

def create_exports():
  """
  Create export directories
  """
  require('map', provided_by=[map])
  local('mkdir -p %(map)s/tiles' % env)
  local('mkdir -p %(map)s/exports' % env)

def cleanup_exports():
  """
  Cleanup export directories
  """
  require('map', provided_by=[map])
  local('rm -rf %(map)s/tiles-tmp' % env)
  local('rm -rf %(map)s/tiles/*' % env)
  local('rm -rf %(map)s/exports/*' % env)

def link():
  """
  Link a map into the MapBox directory
  """
  require('map', provided_by=[map])

  exists = os.path.exists(os.path.expanduser('%(tilemill_projects)s/%(map)s' % env))
  if exists:
    print "A directory with that name already exists in your MapBox projects."
  else:
    env.base_path = os.getcwd()
    local(('ln -s %(base_path)s/%(map)s/ %(tilemill_projects)s/%(map)s') % env)

def unlink():
  """
  unLink a map into the MapBox directory
  """
  require('map', provided_by=[map])

  exists = os.path.exists(os.path.expanduser('%(tilemill_projects)s/%(map)s' % env))
  if exists:
    local(('unlink %(tilemill_projects)s/%(map)s') % env)
  else:
    print "There is no directpry with that name in your MapBox projects."

def clone(name):
  """
  Clone a map to work on.
  """
  require('map', provided_by=[map])
  env.clone = name

  exists = os.path.exists('%(clone)s' % env)
  if exists:
    print "A directory with that name already exists in this project."
  else:
    local('cp -r %(map)s %(clone)s' % env)

    # Now link it
    env.map = env.clone
    link()

def remove_from_s3():
  """
  Remove map from S3
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])

  with settings(warn_only=True):
    for bucket in env.s3_buckets:
      env.s3_bucket = bucket
      local('s3cmd del --recursive s3://%(s3_bucket)s/%(project_name)s/%(map)s' % env)<commit_msg>Scheme support, template support, and just using one bucket.<commit_after>#!/usr/bin/env python
"""
Fab file to help with managing project.  For docs on Fab file, please see: http://docs.fabfile.org/
"""
import sys
import os
import warnings
import json
import re
from fabric.api import *

"""
Base configuration
"""
env.project_name = 'minnpost-basemaps'
env.pg_host = 'localhost'
env.pg_dbname = 'minnpost_fec'
env.pg_user = 'postgres'
env.pg_pass = ''
env.labels = None
env.tile_scheme = 'tms'
env.tile_template = '{z}/{x}/{y}'

# Tilemill paths.  For Ubuntu
if os.path.exists('/usr/share/tilemill'):
  env.tilemill_path = '/usr/share/tilemill'
  env.tilemill_projects = '/usr/share/mapbox/project'
  env.node_path = '/usr/bin/node'
# OSX
else:
  env.tilemill_path = '/Applications/TileMill.app/Contents/Resources'
  env.tilemill_projects = '~/Documents/MapBox/project'
  env.node_path = '%(tilemill_path)s/node' % env

"""
Environments
"""
def production():
  """
  Work on production environment
  """
  env.settings = 'production'
  #env.s3_buckets = ['a.tiles.minnpost', 'b.tiles.minnpost', 'c.tiles.minnpost', 'd.tiles.minnpost']
  env.s3_buckets = ['a.tiles.minnpost']
  env.s3_template = 'http://{s}.tiles.minnpost.s3.amazonaws.com'
  env.acl = 'acl-public'

def staging():
  """
  Work on staging environment
  """
  env.settings = 'staging'
  env.s3_buckets = ['testing.tiles.minnpost']
  env.s3_template = 'http://testing.tiles.minnpost.s3.amazonaws.com'
  env.acl = 'acl-public'

def map(name):
  """
  Select map to work on.
  """
  env.map = name

def tile_scheme(scheme='tms'):
  """
  Change tilescheme.
  """
  env.tile_scheme = scheme

def deploy_to_s3(concurrency):
  """
  Deploy tiles to S3.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])
  env.concurrency = concurrency
  _create_map_suffix()

  # Deploy to many buckets (multi-dns-head mode)
  for bucket in env.s3_buckets:
    env.s3_bucket = bucket
    local('ivs3 %(map)s/tiles %(s3_bucket)s/%(project_name)s/%(map)s%(map_suffix)s --%(acl)s -c %(concurrency)s' % env)

def export_deploy(concurrency=32, minzoom=None, maxzoom=None):
  """
  Deploy a map. Optionally takes a concurrency parameter indicating how many files to upload simultaneously.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])

  create_exports()
  cleanup_exports()
  generate_mbtile(minzoom, maxzoom)
  generate_tiles_from_mbtile()
  generate_tilejson()
  deploy_to_s3(concurrency)
  reset_labels()

def link_caches():
  """
  Link local map cache to Mapbox cache to speed up mapnik conversion.
  """
  require('map', provided_by=[map])

  env.base_path = os.getcwd()
  exists = os.path.exists('%(base_path)s/%(map)s/cache' % env)
  if exists:
    print "There already a linked cache directory."
  else:
    env.tilemill_cache = os.path.expanduser('%(tilemill_projects)s/../cache/' % env)
    local(('ln -s %(tilemill_cache)s %(base_path)s/%(map)s/cache') % env)

def carto_to_mapnik():
  """
  Convert carto styles to mapnik configuration.
  """
  require('map', provided_by=[map])
  link_caches()
  local('%(tilemill_path)s/node_modules/carto/bin/carto %(map)s/project.mml > %(map)s/%(map)s.xml' % env)

def generate_mbtile(minzoom=None, maxzoom=None):
  """
  Generate MBtile.
  """
  require('map', provided_by=[map])

  # Read data from project mml
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)

    # Define config values
    env.minzoom = config['minzoom'] if minzoom == None else minzoom
    env.maxzoom = config['maxzoom'] if maxzoom == None else maxzoom
    env.bbox = '%f,%f,%f,%f' % (config['bounds'][0], config['bounds'][1], config['bounds'][2], config['bounds'][3])

    # Export
    local('%(tilemill_path)s/node %(tilemill_path)s/index.js export --format=mbtiles --minzoom=%(minzoom)s --maxzoom=%(maxzoom)s --bbox=%(bbox)s %(map)s %(map)s/exports/%(map)s.mbtiles' % env)

def generate_tiles_from_mbtile():
  """
  Generate MBtile.
  """
  require('map', provided_by=[map])
  read_project()

  exists = os.path.exists('%(map)s/exports/%(map)s.mbtiles' % env)
  if exists:
    with settings(warn_only=True):
      local('rm -rf %(map)s/tiles-tmp' % env)
      local('rm -rf %(map)s/tiles' % env)
      local('mb-util --scheme=%(tile_scheme)s %(map)s/exports/%(map)s.mbtiles %(map)s/tiles-tmp' % env)
      local('mv "%(map)s/tiles-tmp/%(map_version)s/%(map_title)s" %(map)s/tiles' % env)
      local('mv %(map)s/tiles-tmp/metadata.json %(map)s/tiles/metadata.json' % env)
  else:
    print 'No MBTile file found in exports.'

def generate_tilejson():
  """
  Generate valid tilejson file.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])
  _create_map_suffix()

  # Utilize project data
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)
    tilejson = {}

    # Base values
    tilejson['scheme'] = env.tile_scheme
    tilejson['tilejson'] = '2.0.0'
    tilejson['id'] = env.map

    # Attempt to get values from config
    try:
      tilejson['name'] = config['name'] if config.has_key('name') else ''
      tilejson['description'] = config['description'] if config.has_key('description') else ''
      tilejson['version'] = config['version'] if config.has_key('version') else '1.0.0'
      tilejson['attribution'] = config['attribution'] if config.has_key('attribution') else ''
      tilejson['legend'] = config['legend'] if config.has_key('legend') else ''
      tilejson['minzoom'] = config['minzoom'] if config.has_key('minzoom') else 0
      tilejson['maxzoom'] = config['maxzoom'] if config.has_key('maxzoom') else 22
      tilejson['bounds'] = config['bounds'] if config.has_key('bounds') else [-180, -90, 180, 90]
      tilejson['center'] = config['center'] if config.has_key('center') else null
    except KeyError:
      print 'Key error'

    # Template is in the metadata.json file
    with open('%(map)s/tiles/metadata.json' % env, 'r') as j:
      metadata = json.load(j)
      try:
        tilejson['template'] = metadata['template']
      except KeyError:
        tilejson['template'] = ''

    # Figure out template
    tilejson['grids'] = []
    tilejson['tiles'] = []
    for bucket in env.s3_buckets:
      env.s3_bucket = bucket
      tilejson['grids'].append('http://%(s3_bucket)s.s3.amazonaws.com/%(project_name)s/%(map)s%(map_suffix)s/%(tile_template)s.grid.json' % env)
      tilejson['tiles'].append('http://%(s3_bucket)s.s3.amazonaws.com/%(project_name)s/%(map)s%(map_suffix)s/%(tile_template)s.png' % env)

    # Write regular and jsonp tilejson files
    tilejson_file = open('%(map)s/tiles/tilejson.json' % env, 'w')
    tilejson_file.write(json.dumps(tilejson, sort_keys = True, indent = 2))
    tilejson_file.close()
    tilejsonp_file = open('%(map)s/tiles/tilejson.jsonp' % env, 'w')
    tilejsonp_file.write('grid(%s)' % json.dumps(tilejson, sort_keys = True, indent = 2))
    tilejsonp_file.close()

def generate_tiles_mapnik(process_count, minzoom=None, maxzoom=None):
  """
  Render tile from mapnik configuration
  """
  env.process_count = process_count

  # Read data from project mml
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)

    # Define config values
    env.minzoom = config['minzoom'] if minzoom == None else minzoom
    env.maxzoom = config['maxzoom'] if maxzoom == None else maxzoom
    env.minlon = config['bounds'][0]
    env.minlat = config['bounds'][1]
    env.maxlon = config['bounds'][2]
    env.maxlat = config['bounds'][3]

    # Cleanup tiles
    local('rm -rf %(map)s/tiles/*' % env)

    # Render tiles
    command = 'ivtile %(map)s/%(map)s.xml %(map)s/tiles %(maxlat)s %(minlon)s %(minlat)s %(maxlon)s %(minzoom)s %(maxzoom)s -p %(process_count)s'
    if 'buffer' in env:
      command += ' -b %(buffer)s'
    local(command % env)

def no_labels():
  """
  Updates tilemill project to no use labels
  """
  require('map', provided_by=[map])
  env.labels = False

  # Create a backup
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists != True:
    local('cp %(map)s/project.mml %(map)s/project.mml.orig' % env);

  # From the original, load the json, find any label style
  # and remove.
  overwrite = open('%(map)s/project.mml' % env, 'w')
  with open('%(map)s/project.mml.orig' % env, 'r') as f:
    mml = json.load(f)

    # Process styles
    styles = []
    for index, item in enumerate(mml['Stylesheet']):
      if item != 'labels.mss':
        styles.append(item)
    mml['Stylesheet'] = styles

    overwrite.write(json.dumps(mml, sort_keys = True, indent = 2))
    overwrite.close()

def only_labels():
  """
  Updates tilemill project to use only labels
  """
  require('map', provided_by=[map])
  env.labels = True

  # Create a backup
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists != True:
    local('cp %(map)s/project.mml %(map)s/project.mml.orig' % env);

  # From the original, load the json, keep palette and labels
  overwrite = open('%(map)s/project.mml' % env, 'w')
  with open('%(map)s/project.mml.orig' % env, 'r') as f:
    mml = json.load(f)

    # Process styles
    styles = []
    for index, item in enumerate(mml['Stylesheet']):
      if item == 'labels.mss' or item == 'palette.mss':
        styles.append(item)
    mml['Stylesheet'] = styles

    overwrite.write(json.dumps(mml, sort_keys = True, indent = 2))
    overwrite.close()

def reset_labels():
  """
  Resets any label changes process
  """
  require('map', provided_by=[map])

  # Check for orig
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists:
    local('rm -f %(map)s/project.mml' % env);
    local('mv %(map)s/project.mml.orig %(map)s/project.mml' % env);
  else:
    print 'No label processing to reset.'

def _create_map_suffix():
  """
  Creates map suffix for deploying
  """
  if env.labels == None:
    env.map_suffix = ''
  if env.labels == True:
    env.map_suffix = '-labels'
  if env.labels == False:
    env.map_suffix = '-no-labels'

def read_project():
  """
  Get data from the TileMill project, to be used in other
  commands.
  """
  require('map', provided_by=[map])

  # Read JSON from file
  with open('%(map)s/project.mml' % env, 'r') as f:
    mml = json.load(f)
    env.map_title = mml['name']

    # Version is not always defined
    try:
      env.map_version = mml['version']
    except KeyError:
      env.map_version = '1.0.0'

def create_exports():
  """
  Create export directories
  """
  require('map', provided_by=[map])
  local('mkdir -p %(map)s/tiles' % env)
  local('mkdir -p %(map)s/exports' % env)

def cleanup_exports():
  """
  Cleanup export directories
  """
  require('map', provided_by=[map])
  local('rm -rf %(map)s/tiles-tmp' % env)
  local('rm -rf %(map)s/tiles/*' % env)
  local('rm -rf %(map)s/exports/*' % env)

def link():
  """
  Link a map into the MapBox directory
  """
  require('map', provided_by=[map])

  exists = os.path.exists(os.path.expanduser('%(tilemill_projects)s/%(map)s' % env))
  if exists:
    print "A directory with that name already exists in your MapBox projects."
  else:
    env.base_path = os.getcwd()
    local(('ln -s %(base_path)s/%(map)s/ %(tilemill_projects)s/%(map)s') % env)

def unlink():
  """
  unLink a map into the MapBox directory
  """
  require('map', provided_by=[map])

  exists = os.path.exists(os.path.expanduser('%(tilemill_projects)s/%(map)s' % env))
  if exists:
    local(('unlink %(tilemill_projects)s/%(map)s') % env)
  else:
    print "There is no directpry with that name in your MapBox projects."

def clone(name):
  """
  Clone a map to work on.
  """
  require('map', provided_by=[map])
  env.clone = name

  exists = os.path.exists('%(clone)s' % env)
  if exists:
    print "A directory with that name already exists in this project."
  else:
    local('cp -r %(map)s %(clone)s' % env)

    # Now link it
    env.map = env.clone
    link()

def remove_from_s3():
  """
  Remove map from S3
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])

  with settings(warn_only=True):
    for bucket in env.s3_buckets:
      env.s3_bucket = bucket
      local('s3cmd del --recursive s3://%(s3_bucket)s/%(project_name)s/%(map)s' % env)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

"""
    Basic deployment script for openquake.
"""

import getpass
import sys

from fabric.api import env, run, sudo, cd
from fabric.state import output as fabric_output
from time import gmtime, strftime

# We really don't care about warnings.
fabric_output.warnings = False

def development():
    """ Specify development hosts """
    env.user = getpass.getuser()
    env.hosts = ["localhost"] #"host1", "host2", "host3"]

def production():
    """ Specify production hosts """
    env.user = "deploy"
    env.hosts = ["productionhost1", "productionhost2",
                 "productionhost3", "productionhost4"]

def bootstrap(deploy_dir="/tmp/openquake/"):
    """ Bootstraps an environment for runtime """
    if not ls("%s/releases" % deploy_dir):
        run("mkdir -p %s/releases" % deploy_dir)

def deploy(git_url="git@github.com:gem/openquake.git",
           deploy_dir="/tmp/openquake/"):
    """ Deploy openquake to a series of hosts """

    if not env.hosts:
        usage()
        sys.exit()

    deploy_fmt = "%Y%m%d%H%M%S"
    deploy_timestamp = strftime(deploy_fmt, gmtime())

    with cd("%s/releases" % deploy_dir):
        # Clone into a timestamp file.
        run("git clone %s %s" % (git_url, deploy_timestamp))
        with cd(".."):
            if ls("current"):
                sudo("unlink current")
            sudo("ln -s releases/%s current" % deploy_timestamp)

def usage():
    print "OpenGEM deployment."
    print "To bootstrap run: fab [environment] bootstrap"
    print "To deploy run: fab [environment] deploy"
    print
    print "available environments:"
    print "development"
    print "production"

def ls(file):
    env.warn_only = True
    res = run("ls %s" % file)
    env.warn_only = False

    print type(res)

    return res
<commit_msg>Squashed commit of the following:<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

"""
    Basic deployment script for openquake.
"""

import getpass
import sys

from fabric.api import env, run, sudo, cd
from fabric.state import output as fabric_output
from time import gmtime, strftime

# We really don't care about warnings.
fabric_output.warnings = False

# temporary
OPENQUAKE_DEMO_PKG_URL = "http://gemsun04.ethz.ch/geonode-client/temp/\
openquake-0.11.tar.gz"

@hosts('gemsun04.ethz.ch')
def server():
    """Restart redis-server and rabbitmq-server.

    Note: This function assumes rabbitmq and redis are already installed."""
    # Note: the target host of this deploy script should configure sudoer
    # privileges of env.user such that the following commands (such as
    # rabbitmqctl, apt-get, etc.) can be run as sudo without a password.
    env.user = 'gemadmin'
    if not _pip_is_installed():
        _install_pip()

    redis = 'redis-server'
    rabbitmq = 'rabbitmq-server'

    # stop server processes
    _stop(redis)
    _stop(rabbitmq)

    # update the openquake package
    _install_openquake()

    # restart server processes
    _start(redis)
    _start(rabbitmq)

@hosts('gemsun03.ethz.ch', 'gemsun04.ethz.ch')
def worker():
    """Update and restart celery.

    Note: This function assumes that celery is already installed and configured
    on the client."""

    env.user = 'gemadmin'
    if not _pip_is_installed():
        _install_pip()
    # kill celeryd processes
    _stop('celeryd')

    # update openquake package
    _install_openquake()

    # restart celery
    _start('celeryd')

def _pip_is_installed():
    return _warn_only(run, 'which pip')

def _install_pip():
    _sudo_no_shell('apt-get install python-pip -y')

def _install_openquake():
    _sudo_no_shell('pip install openquake -f %s' % OPENQUAKE_DEMO_PKG_URL)

def _sudo_no_shell(command):
    sudo(command, shell=False)

def _start(service):
    """Start a service (as sudo) using its init script."""
    _sudo_no_shell('/etc/init.d/%s start' % service)

def _stop(service):
    """Stop a service (as sudo) using its init script."""
    _sudo_no_shell('/etc/init.d/%s stop' % service)

def development():
    """ Specify development hosts """
    env.user = getpass.getuser()
    env.hosts = ["localhost"] #"host1", "host2", "host3"]

def production():
    """ Specify production hosts """
    env.user = "deploy"
    env.hosts = ["productionhost1", "productionhost2",
                 "productionhost3", "productionhost4"]

def bootstrap(deploy_dir="/tmp/openquake/"):
    """ Bootstraps an environment for runtime """
    if not ls("%s/releases" % deploy_dir):
        run("mkdir -p %s/releases" % deploy_dir)

def deploy(git_url="git@github.com:gem/openquake.git",
           deploy_dir="/tmp/openquake/"):
    """ Deploy openquake to a series of hosts """

    if not env.hosts:
        usage()
        sys.exit()

    deploy_fmt = "%Y%m%d%H%M%S"
    deploy_timestamp = strftime(deploy_fmt, gmtime())

    with cd("%s/releases" % deploy_dir):
        # Clone into a timestamp file.
        run("git clone %s %s" % (git_url, deploy_timestamp))
        with cd(".."):
            if ls("current"):
                sudo("unlink current")
            sudo("ln -s releases/%s current" % deploy_timestamp)

def usage():
    print "OpenGEM deployment."
    print "To bootstrap run: fab [environment] bootstrap"
    print "To deploy run: fab [environment] deploy"
    print
    print "available environments:"
    print "development"
    print "production"

def ls(file):
    env.warn_only = True
    res = run("ls %s" % file)
    env.warn_only = False

    print type(res)

    return res
<|endoftext|>


<commit_before># coding: utf-8

import glob
import json
import os.path
import re
import shutil
import sys

from mpienv.mpi import BrokenMPI
from mpienv.mpi import MPI
from mpienv.py import MPI4Py

try:
    import __builtin__
except ImportError:
    import builtins

class UnknownMPI(RuntimeError):
    pass

def yes_no_input(msg):
    if hasattr(__builtin__, 'raw_input'):
        input = __builtin__.raw_input
    else:
        input = builtins.input

    try:
        choice = input("{} [y/N]: ".format(msg)).lower()
        while True:
            if choice in ['y', 'ye', 'yes']:
                return True
            elif choice in ['n', 'no']:
                return False
            else:
                choice = input(
                    "Please respond with 'yes' or 'no' [y/N]: ").lower()
    except (EOFError, KeyboardInterrupt):
        return False

def mkdir_p(path):
    if not os.path.exists(path):
        os.makedirs(path)

DefaultConf = {
    'mpich': {
    },
    'mvapich': {
    },
    'openmpi': {
    },
}

class Mpienv(object):
    def __init__(self, root_dir):
        self._root_dir = root_dir
        self._vers_dir = os.path.join(os.environ.get("MPIENV_VERSIONS_DIR") or
                                      os.path.join(root_dir, 'versions'))
        self._shims_dir = os.path.join(self._vers_dir, 'shims')
        pybin = os.path.realpath(sys.executable)
        pybin_enc = re.sub(r'[^a-zA-Z0-9.]', '_', re.sub('^/', '', pybin))

        self._mpi_dir = os.path.join(self._vers_dir, 'mpi')
        self._pylib_dir = os.path.join(self._vers_dir, 'pylib', pybin_enc)
        self._pybuild_dir = os.path.join(self._vers_dir, 'pybuild', pybin_enc)
        self._cache_dir = os.environ.get("MPIENV_CACHE_DIR",
                                         os.path.join(root_dir, 'cache'))
        self._build_dir = os.environ.get("MPIENV_BUILD_DIR",
                                         os.path.join(root_dir, 'builds'))

        mkdir_p(self._vers_dir)
        mkdir_p(self._mpi_dir)
        mkdir_p(self._pylib_dir)
        mkdir_p(self._pybuild_dir)
        mkdir_p(self._cache_dir)
        mkdir_p(self._build_dir)

        self._load_config()
        self._load_mpi_info()

        self._conf['root_dir'] = self._root_dir
        self._conf['vers_dir'] = self._vers_dir
        self._conf['mpi_dir'] = self._mpi_dir
        self._conf['pylib_dir'] = self._pylib_dir
        self._conf['pybuild_dir'] = self._pybuild_dir
        self._conf['cache_dir'] = self._cache_dir
        self._conf['build_dir'] = self._build_dir
        self._conf['shims_dir'] = self._shims_dir

    def root_dir(self):
        return self._root_dir

    def build_dir(self):
        return self._build_dir

    def cache_dir(self):
        return self._cache_dir

    def mpi_dir(self):
        return self._mpi_dir

    def pylib_dir(self):
        return self._pylib_dir

    def pybuild_dir(self):
        return self._pybuild_dir

    def _load_mpi_info(self):
        # Get the current status of the MPI environment.
        self._installed = {}
        for mpiexec_link in glob.glob(os.path.join(self._mpi_dir, '*')):
            assert(os.path.islink(mpiexec_link))
            name = os.path.split(mpiexec_link)[-1]
            try:
                mpiexec = os.readlink(mpiexec_link)
            except EnvironmentError:
                raise RuntimeError("Internal Error: {} is not a file or link"
                                   .format(mpiexec_link))
            try:
                mpi = self.get_mpi_from_mpiexec(mpiexec)
            except RuntimeError:
                # If the directory exists but MPI is not found,
                # then it's a broken MPI.
                if os.path.exists(mpiexec):
                    mpi = BrokenMPI(mpiexec)
                else:
                    sys.stderr.write("mpienv: [Warning] Directory '{}' "
                                     "is registered as {} but no mpiexec "
                                     "is found.\n".format(
                                         mpiexec,
                                         name))
                    continue  # skip
            mpi.name = name
            self._installed[name] = mpi

    def _load_config(self):
        conf_json = os.path.join(self._root_dir, "config.json")
        if os.path.exists(conf_json):
            with open(conf_json) as f:
                pass  # sys.stderr.write(f.read())
            with open(conf_json) as f:
                conf = json.load(f)
        else:
            sys.stderr.write("Warning: Cannot find config file\n")
            conf = {}

        self._conf = DefaultConf.copy()
        self._conf.update(conf)

    def config(self):
        return self._conf

    def get_mpi_from_mpiexec(self, mpiexec):
        mpi_class = MPI(self, mpiexec)
        return mpi_class(mpiexec, self._conf)

    def prefix(self, name):
        return os.path.join(self._mpi_dir, name)

    def get_mpi_from_name(self, name):
        if name not in self:
            sys.stderr.write("mpienv: Error: "
                             "unknown MPI installation: "
                             "'{}'\n".format(name))
            exit(-1)

        mpiexec = os.readlink(os.path.join(self._mpi_dir, name))
        mpi_class = MPI(self, mpiexec)
        return mpi_class(mpiexec, self._conf, name)

    def items(self):
        return self._installed.items()

    def keys(self):
        return self._installed.keys()

    def __getitem__(self, key):
        try:
            return next(mpi for name, mpi in self._installed.items()
                        if name == key)
        except StopIteration:
            raise KeyError()

    def __contains__(self, key):
        return key in self._installed

    def mpiexec(self, name):
        return os.path.realpath(os.path.join(
            self._mpi_dir, name, 'bin', 'mpiexec'))

    def is_installed(self, mpiexec):
        # Find mpiexec in the path or something and check if it is already
        # under our control.
        prefix = os.path.abspath(os.path.join(os.path.dirname(mpiexec),
                                              os.path.pardir))
        assert os.path.isdir(prefix)

        for name, mpi in self.items():
            if os.path.realpath(mpi.mpiexec) == mpiexec:
                return name

        return None

    def get_current_name(self):
        try:
            return next(name for name, mpi in self.items() if mpi.is_active)
        except StopIteration:
            raise UnknownMPI()

    def add(self, target, name=None):
        # `target` is expected to be an mpiexec command or its prefix
        if os.path.isdir(target):
            # target seems to be prefix
            target = os.path.join(target, 'bin', 'mpiexec')

        mpi = self.get_mpi_from_mpiexec(target)

        if isinstance(mpi, BrokenMPI):
            sys.stderr.write("Cannot find MPI in {}\n".format(target))
            exit(-1)

        n = self.is_installed(target)
        if n is not None:
            raise RuntimeError("{} is already managed "
                               "as '{}'".format(target, n))

        if self._installed.get(name) is not None:
            raise RuntimeError("Specifed name '{}' is "
                               "already taken".format(name))
        else:
            name = mpi.default_name
            if name in self:
                sys.stderr.write("Error: "
                                 "Recommended name for {} is {}, "
                                 "but the name is "
                                 "already used. "
                                 "Try -n option.\n".format(target, name))
                exit(-1)

        # dst -> src
        dst = os.path.join(self._mpi_dir, name)
        src = target

        os.symlink(src, dst)

        return name

    def rm(self, name, prompt=False):
        mpi = self.get_mpi_from_name(name)

        if not mpi.is_broken and mpi.is_active:
            sys.stderr.write("You cannot remove active MPI: "
                             "'{}'\n".format(name))
            exit(-1)

        if (not prompt) or yes_no_input("Remove '{}' ?".format(name)):
            mpi.remove()

        mpi4py = MPI4Py(self._conf, name)
        if mpi4py.is_installed():
            mpi4py.rm()

    def rename(self, name_from, name_to):
        if name_from not in self:
            raise RuntimeError("No such MPI: '{}'".format(name_from))

        if name_to in self:
            raise RuntimeError("Name '{}' already exists".format(name_to))

        path_from = os.path.join(self._mpi_dir, name_from)
        path_to = os.path.join(self._mpi_dir, name_to)

        shutil.move(path_from, path_to)

        mpi4py = MPI4Py(self._conf, name_from)
        if mpi4py.is_installed():
            mpi4py.rename(name_to)

    def use(self, name, mpi4py=False):
        mpi = self.get_mpi_from_name(name)

        if isinstance(mpi, BrokenMPI):
            print(mpi)
            sys.stderr.write("mpienv-use: Error: "
                             "'{}' seems to be broken. Maybe it is removed.\n"
                             "".format(name))
            exit(-1)

        if mpi.is_broken:
            sys.stderr.write("mpienv-use: Error: "
                             "'{}' seems to be broken. Maybe it is removed.\n"
                             "".format(name))
            exit(-1)

        mpi.use(name, mpi4py=mpi4py)

    def exec_(self, cmds):
        name = self.get_current_name()
        mpi = self.get_mpi_from_name(name)
        mpi.exec_(cmds)

_root_dir = (os.environ.get("MPIENV_ROOT", None) or
             os.path.join(os.path.expanduser('~'), '.mpienv'))
mpienv = Mpienv(_root_dir)
<commit_msg>bugfix<commit_after># coding: utf-8

import glob
import json
import os.path
import re
import shutil
import sys

from mpienv.mpi import BrokenMPI
from mpienv.mpi import MPI
from mpienv.py import MPI4Py

try:
    import __builtin__
except ImportError:
    import builtins

class UnknownMPI(RuntimeError):
    pass

def yes_no_input(msg):
    if hasattr(__builtin__, 'raw_input'):
        input = __builtin__.raw_input
    else:
        input = builtins.input

    try:
        choice = input("{} [y/N]: ".format(msg)).lower()
        while True:
            if choice in ['y', 'ye', 'yes']:
                return True
            elif choice in ['n', 'no']:
                return False
            else:
                choice = input(
                    "Please respond with 'yes' or 'no' [y/N]: ").lower()
    except (EOFError, KeyboardInterrupt):
        return False

def mkdir_p(path):
    if not os.path.exists(path):
        os.makedirs(path)

DefaultConf = {
    'mpich': {
    },
    'mvapich': {
    },
    'openmpi': {
    },
}

class Mpienv(object):
    def __init__(self, root_dir):
        self._root_dir = root_dir
        self._vers_dir = os.path.join(os.environ.get("MPIENV_VERSIONS_DIR") or
                                      os.path.join(root_dir, 'versions'))
        self._shims_dir = os.path.join(self._vers_dir, 'shims')
        pybin = os.path.realpath(sys.executable)
        pybin_enc = re.sub(r'[^a-zA-Z0-9.]', '_', re.sub('^/', '', pybin))

        self._mpi_dir = os.path.join(self._vers_dir, 'mpi')
        self._pylib_dir = os.path.join(self._vers_dir, 'pylib', pybin_enc)
        self._pybuild_dir = os.path.join(self._vers_dir, 'pybuild', pybin_enc)
        self._cache_dir = os.environ.get("MPIENV_CACHE_DIR",
                                         os.path.join(root_dir, 'cache'))
        self._build_dir = os.environ.get("MPIENV_BUILD_DIR",
                                         os.path.join(root_dir, 'builds'))

        mkdir_p(self._vers_dir)
        mkdir_p(self._mpi_dir)
        mkdir_p(self._pylib_dir)
        mkdir_p(self._pybuild_dir)
        mkdir_p(self._cache_dir)
        mkdir_p(self._build_dir)

        self._load_config()
        self._load_mpi_info()

        self._conf['root_dir'] = self._root_dir
        self._conf['vers_dir'] = self._vers_dir
        self._conf['mpi_dir'] = self._mpi_dir
        self._conf['pylib_dir'] = self._pylib_dir
        self._conf['pybuild_dir'] = self._pybuild_dir
        self._conf['cache_dir'] = self._cache_dir
        self._conf['build_dir'] = self._build_dir
        self._conf['shims_dir'] = self._shims_dir

    def root_dir(self):
        return self._root_dir

    def build_dir(self):
        return self._build_dir

    def cache_dir(self):
        return self._cache_dir

    def mpi_dir(self):
        return self._mpi_dir

    def pylib_dir(self):
        return self._pylib_dir

    def pybuild_dir(self):
        return self._pybuild_dir

    def _load_mpi_info(self):
        # Get the current status of the MPI environment.
        self._installed = {}
        for mpiexec_link in glob.glob(os.path.join(self._mpi_dir, '*')):
            assert(os.path.islink(mpiexec_link))
            name = os.path.split(mpiexec_link)[-1]
            try:
                mpiexec = os.readlink(mpiexec_link)
            except EnvironmentError:
                raise RuntimeError("Internal Error: {} is not a file or link"
                                   .format(mpiexec_link))
            try:
                mpi = self.get_mpi_from_mpiexec(mpiexec)
            except RuntimeError:
                # If the directory exists but MPI is not found,
                # then it's a broken MPI.
                if os.path.exists(mpiexec):
                    mpi = BrokenMPI(mpiexec)
                else:
                    sys.stderr.write("mpienv: [Warning] Directory '{}' "
                                     "is registered as {} but no mpiexec "
                                     "is found.\n".format(
                                         mpiexec,
                                         name))
                    continue  # skip
            mpi.name = name
            self._installed[name] = mpi

    def _load_config(self):
        conf_json = os.path.join(self._root_dir, "config.json")
        if os.path.exists(conf_json):
            with open(conf_json) as f:
                pass  # sys.stderr.write(f.read())
            with open(conf_json) as f:
                conf = json.load(f)
        else:
            sys.stderr.write("Warning: Cannot find config file\n")
            conf = {}

        self._conf = DefaultConf.copy()
        self._conf.update(conf)

    def config(self):
        return self._conf

    def get_mpi_from_mpiexec(self, mpiexec):
        mpi_class = MPI(self, mpiexec)
        return mpi_class(mpiexec, self._conf)

    def prefix(self, name):
        return os.path.join(self._mpi_dir, name)

    def get_mpi_from_name(self, name):
        if name not in self:
            sys.stderr.write("mpienv: Error: "
                             "unknown MPI installation: "
                             "'{}'\n".format(name))
            exit(-1)

        mpiexec = os.readlink(os.path.join(self._mpi_dir, name))
        mpi_class = MPI(self, mpiexec)
        return mpi_class(mpiexec, self._conf, name)

    def items(self):
        return self._installed.items()

    def keys(self):
        return self._installed.keys()

    def __getitem__(self, key):
        try:
            return next(mpi for name, mpi in self._installed.items()
                        if name == key)
        except StopIteration:
            raise KeyError()

    def __contains__(self, key):
        return key in self._installed

    def mpiexec(self, name):
        return os.path.realpath(os.path.join(
            self._mpi_dir, name, 'bin', 'mpiexec'))

    def is_installed(self, mpiexec):
        # Find mpiexec in the path or something and check if it is already
        # under our control.
        prefix = os.path.abspath(os.path.join(os.path.dirname(mpiexec),
                                              os.path.pardir))
        assert os.path.isdir(prefix)

        for name, mpi in self.items():
            if os.path.realpath(mpi.mpiexec) == mpiexec:
                return name

        return None

    def get_current_name(self):
        try:
            return next(name for name, mpi in self.items() if mpi.is_active)
        except StopIteration:
            raise UnknownMPI()

    def add(self, target, name=None):
        # `target` is expected to be an mpiexec command or its prefix
        if os.path.isdir(target):
            # target seems to be prefix
            target = os.path.join(target, 'bin', 'mpiexec')

        mpi = self.get_mpi_from_mpiexec(target)

        if isinstance(mpi, BrokenMPI):
            sys.stderr.write("Cannot find MPI in {}\n".format(target))
            exit(-1)

        n = self.is_installed(target)
        if n is not None:
            raise RuntimeError("{} is already managed "
                               "as '{}'".format(target, n))

        if self._installed.get(name) is not None:
            raise RuntimeError("Specifed name '{}' is "
                               "already taken".format(name))
        elif name is None:
            name = mpi.default_name
            if name in self:
                sys.stderr.write("Error: "
                                 "Recommended name for {} is {}, "
                                 "but the name is "
                                 "already used. "
                                 "Try -n option.\n".format(target, name))
                exit(-1)

        # dst -> src
        dst = os.path.join(self._mpi_dir, name)
        src = target

        os.symlink(src, dst)

        return name

    def rm(self, name, prompt=False):
        mpi = self.get_mpi_from_name(name)

        if not mpi.is_broken and mpi.is_active:
            sys.stderr.write("You cannot remove active MPI: "
                             "'{}'\n".format(name))
            exit(-1)

        if (not prompt) or yes_no_input("Remove '{}' ?".format(name)):
            mpi.remove()

        mpi4py = MPI4Py(self._conf, name)
        if mpi4py.is_installed():
            mpi4py.rm()

    def rename(self, name_from, name_to):
        if name_from not in self:
            raise RuntimeError("No such MPI: '{}'".format(name_from))

        if name_to in self:
            raise RuntimeError("Name '{}' already exists".format(name_to))

        path_from = os.path.join(self._mpi_dir, name_from)
        path_to = os.path.join(self._mpi_dir, name_to)

        shutil.move(path_from, path_to)

        mpi4py = MPI4Py(self._conf, name_from)
        if mpi4py.is_installed():
            mpi4py.rename(name_to)

    def use(self, name, mpi4py=False):
        mpi = self.get_mpi_from_name(name)

        if isinstance(mpi, BrokenMPI):
            print(mpi)
            sys.stderr.write("mpienv-use: Error: "
                             "'{}' seems to be broken. Maybe it is removed.\n"
                             "".format(name))
            exit(-1)

        if mpi.is_broken:
            sys.stderr.write("mpienv-use: Error: "
                             "'{}' seems to be broken. Maybe it is removed.\n"
                             "".format(name))
            exit(-1)

        mpi.use(name, mpi4py=mpi4py)

    def exec_(self, cmds):
        name = self.get_current_name()
        mpi = self.get_mpi_from_name(name)
        mpi.exec_(cmds)

_root_dir = (os.environ.get("MPIENV_ROOT", None) or
             os.path.join(os.path.expanduser('~'), '.mpienv'))
mpienv = Mpienv(_root_dir)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Run 'fab --list' to see list of available commands.

References:
# http://docs.fabfile.org/en/1.0.1/usage/execution.html#how-host-lists-are-constructed
"""

from __future__ import with_statement
import platform
assert ('2', '6') <= platform.python_version_tuple() < ('3', '0')

import time
import logging
import os
import sh
import sys
import datetime as dt

from fabric.api import local, task

APP_NAME = "Westminster Daily"
PROJ_DIR = os.path.dirname(os.path.abspath(__file__))
SITE_NAME = "reformedconfessions.com"

sys.path.append(PROJ_DIR)

log = logging.getLogger()
log.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
log.addHandler(ch)

@task
def console():
    """Load the application in an interactive console.
    """
    local('env DEV=yes python -i runserver.py', capture=False)

@task(alias='preview')
def server():
    """Run the dev server"""
    os.chdir(PROJ_DIR)
    local('env DEV=yes python runserver.py', capture=False)

@task
def test():
    """Run the test suite"""
    local('py.test --verbose   --full-trace -ff', capture=False)

@task
def build_images(month):

    image_path = 'flask_application/static/images/docs/'
    server = sh.python('runserver.py', _bg=True)
    print "starting server with pid {}".format(server.pid)
    time.sleep(5)
    for date in (dt.datetime(2004, int(month), 01) + dt.timedelta(n) for n in range(1)):
        webkit2png = sh.Command("/usr/local/bin/webkit2png")
        p = webkit2png(date.strftime('http://0.0.0.0:8080/i/%m/%d'),
                       selector="#content",
                       filename=date.strftime('%m%d'),
                       fullsize=True,
                       dir=image_path,
                       **{"ignore-ssl-check": True}
                       )
        print p.ran
        pngquant = sh.Command('/usr/local/bin/pngquant')
        pngquant("--force",
                 "--ext", ".png",
                 "--quality", "65-80",
                 "{}{}-full.png".format(image_path, date.strftime('%m%d')))

@task
def images_to_s3():
    local('s3cmd put --acl-public images/* s3://reformedconfessions')

@task
def clean():
    """Clear the cached .pyc files."""
    local("find . \( -iname '*.pyc' -o -name '*~' \) -exec rm -v {} \;", capture=False)
    local("rm -rf htmlcov", capture=False)

@task
def configure_tweet():
    """Tweet today's confession post
    """
    import twitter as tw

    cred = {
        "consumer_key": os.environ['TW_CONSUMER_KEY'],
        "consumer_secret": os.environ['TW_CONSUMER_SECRET'],
        "app_name": "Reformed Confessions"
    }
    oauth_token, oauth_token_secret = tw.oauth_dance(**cred)
    print "oauth_token", oauth_token
    print "oauth_token_secret", oauth_token_secret

def make_facebook_string(content):
    c_strings = []
    for c in content:
        if c['type'] == 'confession':
            c_strings.append("""{}

{}

""".format(c['long_citation'], c['body']))
        elif c['type'] == 'catechism':
            c_strings.append("""{}

Q. {}
A. {}

""".format(c['long_citation'], c['question'], c['answer']))
    return ''.join(c_strings)

def ascii_encode_dict(data):
    ascii_encode = lambda x: x.encode('ascii') if isinstance(x, unicode) else x
    return dict(map(ascii_encode, pair) for pair in data.items())

@task
def catechism_to_yaml(name="wlc"):
    """Generate YAML file for catechism
    """
    import re
    import json
    import yaml

    with open('./flask_application/static/confessions/{}.json'.format(name), 'r') as f:
        catechism = json.load(f, object_hook=ascii_encode_dict)

    for k, v in catechism.items():
        del v['prooftext_verses']

    catechism2 = {}
    for k, v in catechism.items():
        catechism2[int(k)] = v
        for pk in list(v['prooftexts'].keys()):
            v['prooftexts'][int(pk)] = v['prooftexts'][pk]
            del v['prooftexts'][(pk)]
        a = v['answer']
        pattern = r"(.*?)"
        v['answer'] = re.sub(pattern, r"\2[\1]", a)

        #
        v['1question'] = v['question']
        v['2answer'] = v['answer']
        v['3prooftexts'] = v['prooftexts']
        del v['question']
        del v['answer']
        del v['prooftexts']

    print (yaml.dump(catechism2, default_flow_style=False, width=120)
               .replace("1question", "Q")
               .replace("2answer", "A")
               .replace("3prooftexts", "proofs")
           )

@task
def confession_to_yaml():
    """Generate YAML file for WCF"""
    import re
    import json
    import yaml

    with open('../westminster-daily/flask_application/static/confessions/wcf.json', 'r') as f:
        catechism = json.load(f, object_hook=ascii_encode_dict)

    for k, v in catechism.items():
        del v['prooftext_verses']

    catechism2 = {}
    for k, v in catechism.items():
        catechism2[int(k)] = v
        for pk in list(v['prooftexts'].keys()):
            v['prooftexts'][int(pk)] = v['prooftexts'][pk]
            del v['prooftexts'][(pk)]
        body = {int(k): v for k, v in v['body'].items()}
        for k2, v2 in body.items():
            pattern = r"(.*?)"

            body[k2] = re.sub(pattern, r"\2[\1]", v2)

        #
        v['body'] = body
        v['1title'] = v['title']
        del v['title']

    print (yaml.dump(catechism2, default_flow_style=False, width=120)
               .replace("1title", "title")
           )
<commit_msg>remove unused code<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Run 'fab --list' to see list of available commands.

References:
# http://docs.fabfile.org/en/1.0.1/usage/execution.html#how-host-lists-are-constructed
"""

from __future__ import with_statement
import platform
assert ('2', '6') <= platform.python_version_tuple() < ('3', '0')

import time
import logging
import os
import sh
import sys
import datetime as dt

from fabric.api import local, task

APP_NAME = "Westminster Daily"
PROJ_DIR = os.path.dirname(os.path.abspath(__file__))
SITE_NAME = "reformedconfessions.com"

sys.path.append(PROJ_DIR)

log = logging.getLogger()
log.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
log.addHandler(ch)

@task
def build_images(month):

    image_path = 'flask_application/static/images/docs/'
    server = sh.python('runserver.py', _bg=True)
    print "starting server with pid {}".format(server.pid)
    time.sleep(5)
    for date in (dt.datetime(2004, int(month), 01) + dt.timedelta(n) for n in range(1)):
        webkit2png = sh.Command("/usr/local/bin/webkit2png")
        p = webkit2png(date.strftime('http://0.0.0.0:8080/i/%m/%d'),
                       selector="#content",
                       filename=date.strftime('%m%d'),
                       fullsize=True,
                       dir=image_path,
                       **{"ignore-ssl-check": True}
                       )
        print p.ran
        pngquant = sh.Command('/usr/local/bin/pngquant')
        pngquant("--force",
                 "--ext", ".png",
                 "--quality", "65-80",
                 "{}{}-full.png".format(image_path, date.strftime('%m%d')))

@task
def images_to_s3():
    local('s3cmd put --acl-public images/* s3://reformedconfessions')

@task
def catechism_to_yaml(name="wlc"):
    """Generate YAML file for catechism
    """
    import re
    import json
    import yaml

    with open('./flask_application/static/confessions/{}.json'.format(name), 'r') as f:
        catechism = json.load(f, object_hook=ascii_encode_dict)

    for k, v in catechism.items():
        del v['prooftext_verses']

    catechism2 = {}
    for k, v in catechism.items():
        catechism2[int(k)] = v
        for pk in list(v['prooftexts'].keys()):
            v['prooftexts'][int(pk)] = v['prooftexts'][pk]
            del v['prooftexts'][(pk)]
        a = v['answer']
        pattern = r"(.*?)"
        v['answer'] = re.sub(pattern, r"\2[\1]", a)

        #
        v['1question'] = v['question']
        v['2answer'] = v['answer']
        v['3prooftexts'] = v['prooftexts']
        del v['question']
        del v['answer']
        del v['prooftexts']

    print (yaml.dump(catechism2, default_flow_style=False, width=120)
               .replace("1question", "Q")
               .replace("2answer", "A")
               .replace("3prooftexts", "proofs")
           )

@task
def confession_to_yaml():
    """Generate YAML file for WCF"""
    import re
    import json
    import yaml

    with open('../westminster-daily/flask_application/static/confessions/wcf.json', 'r') as f:
        catechism = json.load(f, object_hook=ascii_encode_dict)

    for k, v in catechism.items():
        del v['prooftext_verses']

    catechism2 = {}
    for k, v in catechism.items():
        catechism2[int(k)] = v
        for pk in list(v['prooftexts'].keys()):
            v['prooftexts'][int(pk)] = v['prooftexts'][pk]
            del v['prooftexts'][(pk)]
        body = {int(k): v for k, v in v['body'].items()}
        for k2, v2 in body.items():
            pattern = r"(.*?)"

            body[k2] = re.sub(pattern, r"\2[\1]", v2)

        #
        v['body'] = body
        v['1title'] = v['title']
        del v['title']

    print (yaml.dump(catechism2, default_flow_style=False, width=120)
               .replace("1title", "title")
           )
<|endoftext|>


<commit_before>"""This program handles the communication over I2C
between a Raspberry Pi and a MPU-6050 Gyroscope / Accelerometer combo.
Made by: MrTijn/Tijndagamer
Released under the MIT License
Copyright 2015, 2016
"""

import smbus

class mpu6050:

    # Global Variables
    GRAVITIY_MS2 = 9.80665
    address = None
    bus = smbus.SMBus(1)

    # Scale Modifiers
    ACCEL_SCALE_MODIFIER_2G = 16384.0
    ACCEL_SCALE_MODIFIER_4G = 8192.0
    ACCEL_SCALE_MODIFIER_8G = 4096.0
    ACCEL_SCALE_MODIFIER_16G = 2048.0

    GYRO_SCALE_MODIFIER_250DEG = 131.0
    GYRO_SCALE_MODIFIER_500DEG = 65.5
    GYRO_SCALE_MODIFIER_1000DEG = 32.8
    GYRO_SCALE_MODIFIER_2000DEG = 16.4

    # Pre-defined ranges
    ACCEL_RANGE_2G = 0x00
    ACCEL_RANGE_4G = 0x08
    ACCEL_RANGE_8G = 0x10
    ACCEL_RANGE_16G = 0x18

    GYRO_RANGE_250DEG = 0x00
    GYRO_RANGE_500DEG = 0x08
    GYRO_RANGE_1000DEG = 0x10
    GYRO_RANGE_2000DEG = 0x18

    # MPU-6050 Registers
    PWR_MGMT_1 = 0x6B
    PWR_MGMT_2 = 0x6C

    SELF_TEST_X = 0x0D
    SELF_TEST_Y = 0x0E
    SELF_TEST_Z = 0x0F
    SELF_TEST_A = 0x10

    ACCEL_XOUT0 = 0x3B
    ACCEL_XOUT1 = 0x3C
    ACCEL_YOUT0 = 0x3D
    ACCEL_YOUT1 = 0x3E
    ACCEL_ZOUT0 = 0x3F
    ACCEL_ZOUT1 = 0x40

    TEMP_OUT0 = 0x41
    TEMP_OUT1 = 0x42

    GYRO_XOUT0 = 0x43
    GYRO_XOUT1 = 0x44
    GYRO_YOUT0 = 0x45
    GYRO_YOUT1 = 0x46
    GYRO_ZOUT0 = 0x47
    GYRO_ZOUT1 = 0x48

    ACCEL_CONFIG = 0x1C
    GYRO_CONFIG = 0x1B

    def __init__(self, address):
        self.address = address

        # Wake up the MPU-6050 since it starts in sleep mode
        self.bus.write_byte_data(self.address, self.PWR_MGMT_1, 0x00)

    # I2C communication methods

    def read_i2c_word(self, register):
        """Read two i2c registers and combine them.

        register -- the first register to read from.
        Returns the combined read results.
        """
        # Read the data from the registers
        high = self.bus.read_byte_data(self.address, register)
        low = self.bus.read_byte_data(self.address, register + 1)

        value = (high << 8) + low

        if (value >= 0x8000):
            return -((65535 - value) + 1)
        else:
            return value

    # MPU-6050 Methods

    def get_temp(self):
        """Reads the temperature from the onboard temperature sensor of the MPU-6050.

        Returns the temperature in degrees Celcius.
        """
        raw_temp = self.read_i2c_word(self.TEMP_OUT0)

        # Get the actual temperature using the formule given in the
        # MPU-6050 Register Map and Descriptions revision 4.2, page 30
        actual_temp = (raw_temp / 340.0) + 36.53

        return actual_temp

    def set_accel_range(self, accel_range):
        """Sets the range of the accelerometer to range.

        accel_range -- the range to set the accelerometer to. Using a
        pre-defined range is advised.
        """
        # First change it to 0x00 to make sure we write the correct value later
        self.bus.write_byte_data(self.address, self.ACCEL_CONFIG, 0x00)

        # Write the new range to the ACCEL_CONFIG register
        self.bus.write_byte_data(self.address, self.ACCEL_CONFIG, accel_range)

    def read_accel_range(self, raw = False):
        """Reads the range the accelerometer is set to.

        If raw is True, it will return the raw value from the ACCEL_CONFIG
        register
        If raw is False, it will return an integer: -1, 2, 4, 8 or 16. When it
        returns -1 something went wrong.
        """
        raw_data = self.bus.read_byte_data(self.address, self.ACCEL_CONFIG)

        if raw is True:
            return raw_data
        elif raw is False:
            if raw_data == self.ACCEL_RANGE_2G:
                return 2
            elif raw_data == self.ACCEL_RANGE_4G:
                return 4
            elif raw_data == self.ACCEL_RANGE_8G:
                return 8
            elif raw_data == self.ACCEL_RANGE_16G:
                return 16
            else:
                return -1

    def get_accel_data(self, g = False):
        """Gets and returns the X, Y and Z values from the accelerometer.

        If g is True, it will return the data in g
        If g is False, it will return the data in m/s^2
        Returns a dictionary with the measurement results.
        """
        x = self.read_i2c_word(self.ACCEL_XOUT0)
        y = self.read_i2c_word(self.ACCEL_YOUT0)
        z = self.read_i2c_word(self.ACCEL_ZOUT0)

        accel_scale_modifier = None
        accel_range = self.read_accel_range(True)

        if accel_range == self.ACCEL_RANGE_2G:
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_2G
        elif accel_range == self.ACCEL_RANGE_4G:
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_4G
        elif accel_range == self.ACCEL_RANGE_8G:
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_8G
        elif accel_range == self.ACCEL_RANGE_16G:
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_16G
        else:
            print("Unkown range - accel_scale_modifier set to self.ACCEL_SCALE_MODIFIER_2G")
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_2G

        x = x / accel_scale_modifier
        y = y / accel_scale_modifier
        z = z / accel_scale_modifier

        if g is True:
            return {'x': x, 'y': y, 'z': z}
        elif g is False:
            x = x * self.GRAVITIY_MS2
            y = y * self.GRAVITIY_MS2
            z = z * self.GRAVITIY_MS2
            return {'x': x, 'y': y, 'z': z}

    def set_gyro_range(self, gyro_range):
        """Sets the range of the gyroscope to range.

        gyro_range -- the range to set the gyroscope to. Using a pre-defined
        range is advised.
        """
        # First change it to 0x00 to make sure we write the correct value later
        self.bus.write_byte_data(self.address, self.GYRO_CONFIG, 0x00)

        # Write the new range to the ACCEL_CONFIG register
        self.bus.write_byte_data(self.address, self.GYRO_CONFIG, gyro_range)

    def read_gyro_range(self, raw = False):
        """Reads the range the gyroscope is set to.

        If raw is True, it will return the raw value from the GYRO_CONFIG
        register.
        If raw is False, it will return 250, 500, 1000, 2000 or -1. If the
        returned value is equal to -1 something went wrong.
        """
        raw_data = self.bus.read_byte_data(self.address, self.GYRO_CONFIG)

        if raw is True:
            return raw_data
        elif raw is False:
            if raw_data == self.GYRO_RANGE_250DEG:
                return 250
            elif raw_data == self.GYRO_RANGE_500DEG:
                return 500
            elif raw_data == self.GYRO_RANGE_1000DEG:
                return 1000
            elif raw_data == self.GYRO_RANGE_2000DEG:
                return 2000
            else:
                return -1

    def get_gyro_data(self):
        """Gets and returns the X, Y and Z values from the gyroscope.

        Returns the read values in a dictionary.
        """
        x = self.read_i2c_word(self.GYRO_XOUT0)
        y = self.read_i2c_word(self.GYRO_YOUT0)
        z = self.read_i2c_word(self.GYRO_ZOUT0)

        gyro_scale_modifier = None
        gyro_range = self.read_gyro_range(True)

        if gyro_range == self.GYRO_RANGE_250DEG:
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_250DEG
        elif gyro_range == self.GYRO_RANGE_500DEG:
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_500DEG
        elif gyro_range == self.GYRO_RANGE_1000DEG:
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_1000DEG
        elif gyro_range == self.GYRO_RANGE_2000DEG:
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_2000DEG
        else:
            print("Unkown range - gyro_scale_modifier set to self.GYRO_SCALE_MODIFIER_250DEG")
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_250DEG

        x = x / gyro_scale_modifier
        y = y / gyro_scale_modifier
        z = z / gyro_scale_modifier

        return {'x': x, 'y': y, 'z': z}

    def get_all_data(self):
        """Reads and returns all the available data."""
        temp = get_temp()
        accel = get_accel_data()
        gyro = get_gyro_data()

        return [accel, gyro, temp]

if __name__ == "__main__":
    mpu = MPU6050(0x68)
    print(mpu.get_temp())
    accel_data = mpu.get_accel_data()
    print(accel_data['x'])
    print(accel_data['y'])
    print(accel_data['z'])
    gyro_data = mpu.get_gyro_data()
    print(gyro_data['x'])
    print(gyro_data['y'])
    print(gyro_data['z'])
<commit_msg>Remove unused constants<commit_after>"""This program handles the communication over I2C
between a Raspberry Pi and a MPU-6050 Gyroscope / Accelerometer combo.
Made by: MrTijn/Tijndagamer
Released under the MIT License
Copyright 2015, 2016
"""

import smbus

class mpu6050:

    # Global Variables
    GRAVITIY_MS2 = 9.80665
    address = None
    bus = smbus.SMBus(1)

    # Scale Modifiers
    ACCEL_SCALE_MODIFIER_2G = 16384.0
    ACCEL_SCALE_MODIFIER_4G = 8192.0
    ACCEL_SCALE_MODIFIER_8G = 4096.0
    ACCEL_SCALE_MODIFIER_16G = 2048.0

    GYRO_SCALE_MODIFIER_250DEG = 131.0
    GYRO_SCALE_MODIFIER_500DEG = 65.5
    GYRO_SCALE_MODIFIER_1000DEG = 32.8
    GYRO_SCALE_MODIFIER_2000DEG = 16.4

    # Pre-defined ranges
    ACCEL_RANGE_2G = 0x00
    ACCEL_RANGE_4G = 0x08
    ACCEL_RANGE_8G = 0x10
    ACCEL_RANGE_16G = 0x18

    GYRO_RANGE_250DEG = 0x00
    GYRO_RANGE_500DEG = 0x08
    GYRO_RANGE_1000DEG = 0x10
    GYRO_RANGE_2000DEG = 0x18

    # MPU-6050 Registers
    PWR_MGMT_1 = 0x6B
    PWR_MGMT_2 = 0x6C

    ACCEL_XOUT0 = 0x3B
    ACCEL_YOUT0 = 0x3D
    ACCEL_ZOUT0 = 0x3F

    TEMP_OUT0 = 0x41

    GYRO_XOUT0 = 0x43
    GYRO_YOUT0 = 0x45
    GYRO_ZOUT0 = 0x47

    ACCEL_CONFIG = 0x1C
    GYRO_CONFIG = 0x1B

    def __init__(self, address):
        self.address = address

        # Wake up the MPU-6050 since it starts in sleep mode
        self.bus.write_byte_data(self.address, self.PWR_MGMT_1, 0x00)

    # I2C communication methods

    def read_i2c_word(self, register):
        """Read two i2c registers and combine them.

        register -- the first register to read from.
        Returns the combined read results.
        """
        # Read the data from the registers
        high = self.bus.read_byte_data(self.address, register)
        low = self.bus.read_byte_data(self.address, register + 1)

        value = (high << 8) + low

        if (value >= 0x8000):
            return -((65535 - value) + 1)
        else:
            return value

    # MPU-6050 Methods

    def get_temp(self):
        """Reads the temperature from the onboard temperature sensor of the MPU-6050.

        Returns the temperature in degrees Celcius.
        """
        raw_temp = self.read_i2c_word(self.TEMP_OUT0)

        # Get the actual temperature using the formule given in the
        # MPU-6050 Register Map and Descriptions revision 4.2, page 30
        actual_temp = (raw_temp / 340.0) + 36.53

        return actual_temp

    def set_accel_range(self, accel_range):
        """Sets the range of the accelerometer to range.

        accel_range -- the range to set the accelerometer to. Using a
        pre-defined range is advised.
        """
        # First change it to 0x00 to make sure we write the correct value later
        self.bus.write_byte_data(self.address, self.ACCEL_CONFIG, 0x00)

        # Write the new range to the ACCEL_CONFIG register
        self.bus.write_byte_data(self.address, self.ACCEL_CONFIG, accel_range)

    def read_accel_range(self, raw = False):
        """Reads the range the accelerometer is set to.

        If raw is True, it will return the raw value from the ACCEL_CONFIG
        register
        If raw is False, it will return an integer: -1, 2, 4, 8 or 16. When it
        returns -1 something went wrong.
        """
        raw_data = self.bus.read_byte_data(self.address, self.ACCEL_CONFIG)

        if raw is True:
            return raw_data
        elif raw is False:
            if raw_data == self.ACCEL_RANGE_2G:
                return 2
            elif raw_data == self.ACCEL_RANGE_4G:
                return 4
            elif raw_data == self.ACCEL_RANGE_8G:
                return 8
            elif raw_data == self.ACCEL_RANGE_16G:
                return 16
            else:
                return -1

    def get_accel_data(self, g = False):
        """Gets and returns the X, Y and Z values from the accelerometer.

        If g is True, it will return the data in g
        If g is False, it will return the data in m/s^2
        Returns a dictionary with the measurement results.
        """
        x = self.read_i2c_word(self.ACCEL_XOUT0)
        y = self.read_i2c_word(self.ACCEL_YOUT0)
        z = self.read_i2c_word(self.ACCEL_ZOUT0)

        accel_scale_modifier = None
        accel_range = self.read_accel_range(True)

        if accel_range == self.ACCEL_RANGE_2G:
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_2G
        elif accel_range == self.ACCEL_RANGE_4G:
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_4G
        elif accel_range == self.ACCEL_RANGE_8G:
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_8G
        elif accel_range == self.ACCEL_RANGE_16G:
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_16G
        else:
            print("Unkown range - accel_scale_modifier set to self.ACCEL_SCALE_MODIFIER_2G")
            accel_scale_modifier = self.ACCEL_SCALE_MODIFIER_2G

        x = x / accel_scale_modifier
        y = y / accel_scale_modifier
        z = z / accel_scale_modifier

        if g is True:
            return {'x': x, 'y': y, 'z': z}
        elif g is False:
            x = x * self.GRAVITIY_MS2
            y = y * self.GRAVITIY_MS2
            z = z * self.GRAVITIY_MS2
            return {'x': x, 'y': y, 'z': z}

    def set_gyro_range(self, gyro_range):
        """Sets the range of the gyroscope to range.

        gyro_range -- the range to set the gyroscope to. Using a pre-defined
        range is advised.
        """
        # First change it to 0x00 to make sure we write the correct value later
        self.bus.write_byte_data(self.address, self.GYRO_CONFIG, 0x00)

        # Write the new range to the ACCEL_CONFIG register
        self.bus.write_byte_data(self.address, self.GYRO_CONFIG, gyro_range)

    def read_gyro_range(self, raw = False):
        """Reads the range the gyroscope is set to.

        If raw is True, it will return the raw value from the GYRO_CONFIG
        register.
        If raw is False, it will return 250, 500, 1000, 2000 or -1. If the
        returned value is equal to -1 something went wrong.
        """
        raw_data = self.bus.read_byte_data(self.address, self.GYRO_CONFIG)

        if raw is True:
            return raw_data
        elif raw is False:
            if raw_data == self.GYRO_RANGE_250DEG:
                return 250
            elif raw_data == self.GYRO_RANGE_500DEG:
                return 500
            elif raw_data == self.GYRO_RANGE_1000DEG:
                return 1000
            elif raw_data == self.GYRO_RANGE_2000DEG:
                return 2000
            else:
                return -1

    def get_gyro_data(self):
        """Gets and returns the X, Y and Z values from the gyroscope.

        Returns the read values in a dictionary.
        """
        x = self.read_i2c_word(self.GYRO_XOUT0)
        y = self.read_i2c_word(self.GYRO_YOUT0)
        z = self.read_i2c_word(self.GYRO_ZOUT0)

        gyro_scale_modifier = None
        gyro_range = self.read_gyro_range(True)

        if gyro_range == self.GYRO_RANGE_250DEG:
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_250DEG
        elif gyro_range == self.GYRO_RANGE_500DEG:
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_500DEG
        elif gyro_range == self.GYRO_RANGE_1000DEG:
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_1000DEG
        elif gyro_range == self.GYRO_RANGE_2000DEG:
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_2000DEG
        else:
            print("Unkown range - gyro_scale_modifier set to self.GYRO_SCALE_MODIFIER_250DEG")
            gyro_scale_modifier = self.GYRO_SCALE_MODIFIER_250DEG

        x = x / gyro_scale_modifier
        y = y / gyro_scale_modifier
        z = z / gyro_scale_modifier

        return {'x': x, 'y': y, 'z': z}

    def get_all_data(self):
        """Reads and returns all the available data."""
        temp = get_temp()
        accel = get_accel_data()
        gyro = get_gyro_data()

        return [accel, gyro, temp]

if __name__ == "__main__":
    mpu = MPU6050(0x68)
    print(mpu.get_temp())
    accel_data = mpu.get_accel_data()
    print(accel_data['x'])
    print(accel_data['y'])
    print(accel_data['z'])
    gyro_data = mpu.get_gyro_data()
    print(gyro_data['x'])
    print(gyro_data['y'])
    print(gyro_data['z'])
<|endoftext|>


<commit_before><commit_msg>Bugfixes, renaming of variables, comments<commit_after><|endoftext|>


<commit_before>"""Fabric file for Goldstone development."""
# Copyright 2015 Solinea, Inc.
#
# Licensed under the Solinea Software License Agreement (goldstone),
# Version 1.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at:
#
#     http://www.solinea.com/goldstone/LICENSE.pdf
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys

from contextlib import contextmanager
from fabric.api import task, local, warn, prompt

# Add the current directory to the module search path.
sys.path.append('')

# The Goldstone settings directory, relative to the Goldstone root where we're
# executing from.
SETTINGS_DIR = "goldstone.settings"

# The settings for running test locally in development.
DEV_SETTINGS = SETTINGS_DIR + ".test"

def _django_manage(command, target='', proj_settings=None, daemon=False):
    """Run manage.py <command>.

    :param command: The command to send to manage. E.g., test
    :type command: str
    :keyword target:A subcommand to sen d to manage. E.g., test user
    :type target: str
    :keyword proj_settings: The project settings to use
    :type proj_settings: str
    :keyword daemon: True if the command should be run in a background process
    :param daemon: bool

    """

    # Create the --settings argument, if requested.
    settings_opt = '' if proj_settings is None \
                   else " --settings=%s " % proj_settings

    # Run this command as a background process, if requested.
    daemon_opt = "&" if daemon else ''

    local("./manage.py %s %s %s %s" %
          (command, target, settings_opt, daemon_opt))

@contextmanager
def _django_env(proj_settings=DEV_SETTINGS):
    """Load a new context into DJANGO_SETTINGS_MODULE."""

    old_settings = os.environ.get('DJANGO_SETTINGS_MODULE')
    os.environ['DJANGO_SETTINGS_MODULE'] = proj_settings

    # Yield control.
    yield

    # Restore state.
    if old_settings is None:
        del os.environ['DJANGO_SETTINGS_MODULE']
    else:
        os.environ['DJANGO_SETTINGS_MODULE'] = old_settings

def _choose(candidates):
    """Return a user selection from a displayed list, or None.

    :param candidates: The selections, one per line
    :type candidates: str or None

    """

    # We haven't made a choice yet.
    choice = None

    # Make a list of the choices.
    choices = [x for x in candidates.split("\n")]

    # If there are choices from which to choose...
    if choices:
        choice_in = None

        # Tell the user what the nummeric selection range is.
        choice_range = \
            "0" if len(choices) == 1 else "0-%s" % (len(choices) - 1)

        # While we don't have a valid selection...
        while choice is None or choice < 0 or choice >= len(choices):
            if choice_in is not None:
                warn("Invalid choice: %s" % choice_in)

            # Display the choices.
            for i, entry in enumerate(choices):
                print "[%s] %s" % (i, entry)

            # Get the user's selection.
            try:
                choice_in = prompt("Choice (%s): " % (choice_range))
                choice = int(choice_in)
            except ValueError:
                pass

        return choices[choice]

    return None

@task
def syncmigrate(proj_settings=DEV_SETTINGS):
    """Do a syncdb and migrate.

    This is the last installation step before execution a load command."""

    print "doing a syncdb and migrate ..."
    print '(answer "no" to the, "create a superuser?" question.)'
    _django_manage("syncdb", proj_settings=proj_settings)
    _django_manage("migrate", proj_settings=proj_settings)

    # N.B. We must create the superuser separately because of an interaction
    # between DRF and Django signals. See
    # https://github.com/tomchristie/django-rest-framework/issues/987.
    print "please create a superuser account ..."
    _django_manage("createsuperuser", proj_settings=proj_settings)

@task
def load(proj_settings=DEV_SETTINGS):
    """Do an initialize_development().

    This is the last installation step before executing a runserver command."""

    print "initializing goldstone ..."
    with _django_env(proj_settings):
        # We have the desired Django settings now. Import the initialization
        # code.
        from initial_load import initialize_development

        # Initialize the world.
        initialize_development()

def _choose_runserver_settings():
    """Display the available settings files for a "runserver" command, ask the
    user to select one, and return a valid selection.

    :return: A filepath to a settings file
    :rtype: str

    """

    # Bash command to locate the candidate settings files, from results piped
    # in. Is there a simpler expression that'll do the job?
    CANDIDATES = 'egrep "dev|test|jstanford" | egrep -v "development|pyc"'

    # Make a list of all the candidate settings file.
    candidates = local("ls goldstone/settings | %s" % CANDIDATES, capture=True)

    # Return the user's selection
    print "\nchoose a settings file to use:"
    return _choose(candidates)

@task
def runserver():
    """Do runserver using a user-selected settings file."""

    # Get the user's desired settings file, strip off the trailing ".py", and
    # convert it into a Python path.
    settings = _choose_runserver_settings().replace(".py", '')
    settings = SETTINGS_DIR + '.' + settings

    _django_manage("runserver", proj_settings=settings)

@task
def test(target=''):
    """Run the unit tests.

    :keyword target: The app or test target. E.g., goldstone.user
    :type target: str

    """

    _django_manage("test", target=target, proj_settings=DEV_SETTINGS)
<commit_msg>GOLD-639: Upgraded fabfile to optionally display verbose output in the runserver command.<commit_after>"""Fabric file for Goldstone development."""
# Copyright 2015 Solinea, Inc.
#
# Licensed under the Solinea Software License Agreement (goldstone),
# Version 1.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at:
#
#     http://www.solinea.com/goldstone/LICENSE.pdf
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys

from contextlib import contextmanager
from fabric.api import task, local, warn, prompt

# Add the current directory to the module search path.
sys.path.append('')

# The Goldstone settings directory, relative to the Goldstone root where we're
# executing from.
SETTINGS_DIR = "goldstone.settings"

# The default settings are to run Elasticsearch and PostgreSQL locally.
DEV_SETTINGS = SETTINGS_DIR + ".test_oak_c2"

def _django_manage(command, target='', proj_settings=None, daemon=False):
    """Run manage.py <command>.

    :param command: The command to send to manage. E.g., test
    :type command: str
    :keyword target:A subcommand to sen d to manage. E.g., test user
    :type target: str
    :keyword proj_settings: The project settings to use
    :type proj_settings: str
    :keyword daemon: True if the command should be run in a background process
    :param daemon: bool

    """

    # Create the --settings argument, if requested.
    settings_opt = '' if proj_settings is None \
                   else " --settings=%s " % proj_settings

    # Run this command as a background process, if requested.
    daemon_opt = "&" if daemon else ''

    local("./manage.py %s %s %s %s" %
          (command, target, settings_opt, daemon_opt))

@contextmanager
def _django_env(proj_settings=DEV_SETTINGS):
    """Load a new context into DJANGO_SETTINGS_MODULE."""

    old_settings = os.environ.get('DJANGO_SETTINGS_MODULE')
    os.environ['DJANGO_SETTINGS_MODULE'] = proj_settings

    # Yield control.
    yield

    # Restore state.
    if old_settings is None:
        del os.environ['DJANGO_SETTINGS_MODULE']
    else:
        os.environ['DJANGO_SETTINGS_MODULE'] = old_settings

def _choose(candidates):
    """Return a user selection from a displayed list, or None.

    :param candidates: The selections, separated by '\n'. So the last entry is
                       *not* terminated by a '\n'!
    :type candidates: str or None

    """

    # We haven't made a choice yet.
    choice = None

    # Make a list of the choices.
    choices = [x for x in candidates.split("\n")]

    # If there are choices from which to choose...
    if choices:
        choice_in = None

        # Tell the user what the nummeric selection range is.
        choice_range = \
            "0" if len(choices) == 1 else "0-%s" % (len(choices) - 1)

        # While we don't have a valid selection...
        while choice is None or choice < 0 or choice >= len(choices):
            if choice_in is not None:
                warn("Invalid choice: %s" % choice_in)

            # Display the choices.
            for i, entry in enumerate(choices):
                print "[%s] %s" % (i, entry)

            # Get the user's selection.
            try:
                choice_in = prompt("Choice (%s): " % (choice_range))
                choice = int(choice_in)
            except ValueError:
                pass

        return choices[choice]

    return None

@task
def syncmigrate(proj_settings=DEV_SETTINGS):
    """Do a syncdb and migrate.

    This is the last installation step before execution a load command."""

    print "doing a syncdb and migrate ..."
    print '(answer "no" to the, "create a superuser?" question.)'
    _django_manage("syncdb", proj_settings=proj_settings)
    _django_manage("migrate", proj_settings=proj_settings)

    # N.B. We must create the superuser separately because of an interaction
    # between DRF and Django signals. See
    # https://github.com/tomchristie/django-rest-framework/issues/987.
    print "please create a superuser account ..."
    _django_manage("createsuperuser", proj_settings=proj_settings)

@task
def load(proj_settings=DEV_SETTINGS):
    """Do an initialize_development().

    This is the last installation step before executing a runserver command."""

    print "initializing goldstone ..."
    with _django_env(proj_settings):
        # We have the desired Django settings now. Import the initialization
        # code.
        from initial_load import initialize_development

        # Initialize the world.
        initialize_development()

def _choose_runserver_settings(verbose):
    """Display the available settings files for a "runserver" command, ask the
    user to select one, and return a valid selection.

    This displays as choices only those settings files that make sense as a
    choice.

    :param verbose: Display detail about each settings choice?
    :type verbose: bool
    :return: A filepath to a settings file
    :rtype: str

    """
    from importlib import import_module

    # Bash command to locate the candidate settings files, from results piped
    # in. The results will be in alphabetical order by default.
    CANDIDATES = 'egrep "dev_|test_|jstanford" | egrep -v "pyc"'

    # Make a list of all the candidate settings file.
    candidates = local("ls goldstone/settings | %s" % CANDIDATES, capture=True)

    # If the user wants verbose output, and each module's docstring to its
    # selection...
    if verbose:
        candidates_list = candidates.split('\n')

        result = []

        # For every settings filename...
        for entry in candidates_list:
            # Strip off the ".py" and import the module.
            filename = entry[:-3]
            module = import_module(SETTINGS_DIR + '.' + filename)

            if module.__doc__:
                # This module has a docstring. Remove embedded \n's from it and
                # add it to the result.
                docstring = module.__doc__.replace('\n', ' ')
                result.append("%s (%s)" % (entry, docstring))
            else:
                # No module docstring. This entry will be only the filename.
                result.append(entry)

        # Convert the verbose list into a string for the prompt function.
        candidates = '\n'.join(result)

    # Return the user's selection
    print "\nchoose a settings file to use:"
    return _choose(candidates)

@task
def runserver(verbose=False):
    """Do runserver using a user-selected settings file.

    :keyword verbose: Display detail about each settings choice?
    :type verbose: bool

    """

    # Get the user's desired settings file, strip off the trailing ".py", and
    # convert it into a Python path.
    settings = _choose_runserver_settings(verbose).replace(".py", '')
    settings = SETTINGS_DIR + '.' + settings

    _django_manage("runserver", proj_settings=settings)

@task
def test(target=''):
    """Run the unit tests.

    :keyword target: The app or test target. E.g., goldstone.user
    :type target: str

    """

    _django_manage("test", target=target, proj_settings=DEV_SETTINGS)
<|endoftext|>


<commit_before>import logging
logger = logging.getLogger('')

from os.path import exists
from urllib import urlopen
from bs4 import BeautifulSoup
from lxml import etree

from django.conf import settings

class Scraper(object):
    def __init__(self, sync=False):
        self.sync = sync

    def read_or_dl_with_nl(self, url, name, reset=False):
        soup = self.read_or_dl(url, name, reset=reset)
        suppe = self.read_or_dl(url.replace("&language=fr", "&language=nl", 1), name + " nl", reset=reset)
        return soup, suppe

    def read_or_dl(self, url, name, reset=False):
        logger.debug("\033[0;33mparsing %s --- %s\033[0m" % (url, name))
        if not reset and exists('dump/%s' % name) and settings.CACHE_SCRAPING:
            text = open('dump/%s' % name).read()
        else:
            text = urlopen(url).read()
            open('dump/%s' % name, "w").write(text)
        soup = BeautifulSoup(text, "html5lib", from_encoding="latin1")
        if soup.title.text == "404 Not Found":
            raise IndexError
        return soup

    def lxml_read_or_dl_with_nl(self, url, name, reset=False):
        soup = self.lxml_read_or_dl(url, name, reset)
        suppe = self.lxml_read_or_dl(url.replace("&language=fr", "&language=nl", 1), name + " nl", reset)
        return soup, suppe

    def lxml_read_or_dl(self, url, name, reset=False):
        logger.debug("LXML parsing %s --- %s" % (url, name))
        if not reset and exists('dump/%s' % name) and settings.CACHE_SCRAPING:
            text = open('dump/%s' % name)
        else:
            text = urlopen(url)
            open('dump/%s' % name, "w").write(urlopen(url).read())
        soup = etree.parse(text, etree.HTMLParser())
        return soup
<commit_msg>[mod] rename methods<commit_after>import logging
logger = logging.getLogger('')

from os.path import exists
from urllib import urlopen
from bs4 import BeautifulSoup
from lxml import etree

from django.conf import settings

class Scraper(object):
    def __init__(self, sync=False):
        self.sync = sync

    def get_with_nl(self, url, name, reset=False):
        soup = self.read_or_dl(url, name, reset=reset)
        suppe = self.read_or_dl(url.replace("&language=fr", "&language=nl", 1), name + " nl", reset=reset)
        return soup, suppe

    def get(self, url, name, reset=False):
        logger.debug("\033[0;33mparsing %s --- %s\033[0m" % (url, name))
        if not reset and exists('dump/%s' % name) and settings.CACHE_SCRAPING:
            text = open('dump/%s' % name).read()
        else:
            text = urlopen(url).read()
            open('dump/%s' % name, "w").write(text)
        soup = BeautifulSoup(text, "html5lib", from_encoding="latin1")
        if soup.title.text == "404 Not Found":
            raise IndexError
        return soup

    def lxml_get_with_nl(self, url, name, reset=False):
        soup = self.lxml_read_or_dl(url, name, reset)
        suppe = self.lxml_read_or_dl(url.replace("&language=fr", "&language=nl", 1), name + " nl", reset)
        return soup, suppe

    def lxml_get(self, url, name, reset=False):
        logger.debug("LXML parsing %s --- %s" % (url, name))
        if not reset and exists('dump/%s' % name) and settings.CACHE_SCRAPING:
            text = open('dump/%s' % name)
        else:
            text = urlopen(url)
            open('dump/%s' % name, "w").write(urlopen(url).read())
        soup = etree.parse(text, etree.HTMLParser())
        return soup
<|endoftext|>


<commit_before>__title__ = 'MT940'
__package_name__ = 'mt-940'
__author__ = 'Rick van Hattem (wolph)'
__description__ = ' '.join('''
A library to parse MT940 files and returns smart Python collections for
statistics and manipulation.
'''.strip().split())
__email__ = 'wolph@wol.ph'
__version__ = '4.15.0'
__license__ = 'BSD'
__copyright__ = 'Copyright 2015 Rick van Hattem (wolph)'
__url__ = 'https://github.com/WoLpH/mt940'
<commit_msg>Incrementing version to v4.16.0<commit_after>__title__ = 'MT940'
__package_name__ = 'mt-940'
__author__ = 'Rick van Hattem (wolph)'
__description__ = ' '.join('''
A library to parse MT940 files and returns smart Python collections for
statistics and manipulation.
'''.strip().split())
__email__ = 'wolph@wol.ph'
__version__ = '4.16.0'
__license__ = 'BSD'
__copyright__ = 'Copyright 2015 Rick van Hattem (wolph)'
__url__ = 'https://github.com/WoLpH/mt940'
<|endoftext|>


<commit_before>"""Version information."""
__version__ = '1.0.0rc2'
<commit_msg>We can call this version 1.0.0<commit_after>"""Version information."""
__version__ = '1.0.0'
<|endoftext|>