<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
#
# A script which will be invoked from gyp to create an SDK.
#
# Usage: create_sdk.py sdk_directory
#
# The SDK will be used either from the command-line or from the editor.
# Top structure is
#
# ..dart-sdk/
# ....bin/
# ......dart or dart.exe (executable)
# ......dart.lib (import library for VM native extensions on Windows)
# ......dart2js
# ......dart_analyzer
# ......pub
# ....include/
# ......dart_api.h
# ......dart_debugger_api.h
# ....lib/
# ......_internal/
# ......core/
# ......coreimpl/
# ......crypto/
# ......html/
# ......io/
# ......isolate/
# ......json/
# ......math/
# ......mirrors/
# ......uri/
# ......utf/
# ......scalarlist/
# ....pkg/
# ......args/
# ......compiler/
# ......dartdoc/
#.......htmlescape/
# ......intl/
# ......logging/
# ......meta/
# ......unittest/
# ......(more will come here)
# ....util/
# ......analyzer/
# ........dart_analyzer.jar
# ........(third-party libraries for dart_analyzer)
# ......pub/
# ......(more will come here)

import os
import re
import sys
import tempfile
import utils

# TODO(dgrove): Only import modules following Google style guide.
from os.path import basename, dirname, join, realpath, exists, isdir

# TODO(dgrove): Only import modules following Google style guide.
from shutil import copyfile, copymode, copytree, ignore_patterns, rmtree, move

def ReplaceInFiles(paths, subs):
  '''Reads a series of files, applies a series of substitutions to each, and
     saves them back out. subs should by a list of (pattern, replace) tuples.'''
  for path in paths:
    contents = open(path).read()
    for pattern, replace in subs:
      contents = re.sub(pattern, replace, contents)

    dest = open(path, 'w')
    dest.write(contents)
    dest.close()

def Copy(src, dest):
  copyfile(src, dest)
  copymode(src, dest)

# TODO(zundel): this excludes the analyzer from the sdk build until builders
# have all prerequisite software installed.  Also update dart.gyp.
def ShouldCopyAnalyzer():
  os = utils.GuessOS();
  return os == 'linux' or os == 'macos'

def CopyShellScript(src_file, dest_dir):
  '''Copies a shell/batch script to the given destination directory. Handles
     using the appropriate platform-specific file extension.'''
  file_extension = ''
  if utils.GuessOS() == 'win32':
    file_extension = '.bat'

  src = src_file + file_extension
  dest = join(dest_dir, basename(src_file) + file_extension)
  Copy(src, dest)

def CopyDart2Js(build_dir, sdk_root, revision):
  if revision:
    ReplaceInFiles([os.path.join(sdk_root, 'pkg', 'compiler',
                                 'implementation', 'compiler.dart')],
                   [(r"BUILD_ID = 'build number could not be determined'",
                     r"BUILD_ID = '%s'" % revision)])
  if utils.GuessOS() == 'win32':
    dart2js = os.path.join(sdk_root, 'bin', 'dart2js.bat')
    Copy(os.path.join(build_dir, 'dart2js.bat'), dart2js)
    dartdoc = os.path.join(sdk_root, 'bin', 'dartdoc.bat')
    Copy(os.path.join(build_dir, 'dartdoc.bat'), dartdoc)
    # TODO(dgrove) - fix this once issue 4788 is addressed.
    ReplaceInFiles([dart2js],
                   [(r'%SCRIPTPATH%\.\.\\\.\.\\lib', r'%SCRIPTPATH%..\\pkg')]);
    ReplaceInFiles([dartdoc],
                   [(r'%SCRIPTPATH%\.\.\\\.\.\\pkg', r'%SCRIPTPATH%..\\pkg')]);
  else:
    dart2js = os.path.join(sdk_root, 'bin', 'dart2js')
    Copy(os.path.join(build_dir, 'dart2js'), dart2js)
    dartdoc = os.path.join(sdk_root, 'bin', 'dartdoc')
    Copy(os.path.join(build_dir, 'dartdoc'), dartdoc)

    # TODO(dgrove) - fix this once issue 4788 is addressed.
    ReplaceInFiles([dart2js],
                   [(r'\$BIN_DIR/\.\./\.\./lib', r'$BIN_DIR/../pkg')])
    ReplaceInFiles([dartdoc],
                   [(r'\$BIN_DIR/\.\./\.\.', r'$BIN_DIR/..')])

def Main(argv):
  # Pull in all of the gpyi files which will be munged into the sdk.
  io_runtime_sources = \
    (eval(open("runtime/bin/io_sources.gypi").read()))['sources']

  HOME = dirname(dirname(realpath(__file__)))

  SDK_tmp = tempfile.mkdtemp()
  SDK = argv[1]

  # TODO(dgrove) - deal with architectures that are not ia32.

  if exists(SDK):
    rmtree(SDK)

  # Create and populate sdk/bin.
  BIN = join(SDK_tmp, 'bin')
  os.makedirs(BIN)

  # Copy the Dart VM binary and the Windows Dart VM link library
  # into sdk/bin.
  #
  # TODO(dgrove) - deal with architectures that are not ia32.
  build_dir = os.path.dirname(argv[1])
  dart_file_extension = ''
  analyzer_file_extension = ''
  if utils.GuessOS() == 'win32':
    dart_file_extension = '.exe'
    analyzer_file_extension = '.bat'  # TODO(zundel): test on Windows
    dart_import_lib_src = join(HOME, build_dir, 'dart.lib')
    dart_import_lib_dest = join(BIN, 'dart.lib')
    copyfile(dart_import_lib_src, dart_import_lib_dest)
  dart_src_binary = join(HOME, build_dir, 'dart' + dart_file_extension)
  dart_dest_binary = join(BIN, 'dart' + dart_file_extension)
  copyfile(dart_src_binary, dart_dest_binary)
  copymode(dart_src_binary, dart_dest_binary)

  if ShouldCopyAnalyzer():
    # Copy analyzer into sdk/bin
    ANALYZER_HOME = join(HOME, build_dir, 'analyzer')
    dart_analyzer_src_binary = join(ANALYZER_HOME, 'bin', 'dart_analyzer')
    dart_analyzer_dest_binary = join(BIN,
        'dart_analyzer' + analyzer_file_extension)
    copyfile(dart_analyzer_src_binary, dart_analyzer_dest_binary)
    copymode(dart_analyzer_src_binary, dart_analyzer_dest_binary)

  # Create pub shell script.
  pub_src_script = join(HOME, 'utils', 'pub', 'sdk', 'pub')
  CopyShellScript(pub_src_script, BIN)

  #
  # Create and populate sdk/include.
  #
  INCLUDE = join(SDK_tmp, 'include')
  os.makedirs(INCLUDE)
  copyfile(join(HOME, 'runtime', 'include', 'dart_api.h'),
           join(INCLUDE, 'dart_api.h'))
  copyfile(join(HOME, 'runtime', 'include', 'dart_debugger_api.h'),
           join(INCLUDE, 'dart_debugger_api.h'))

  #
  # Create and populate sdk/lib.
  #

  LIB = join(SDK_tmp, 'lib')
  os.makedirs(LIB)

  #
  # Create and populate lib/io.
  #
  io_dest_dir = join(LIB, 'io')
  os.makedirs(io_dest_dir)
  os.makedirs(join(io_dest_dir, 'runtime'))
  for filename in io_runtime_sources:
    assert filename.endswith('.dart')
    if filename == 'io.dart':
      copyfile(join(HOME, 'runtime', 'bin', filename),
               join(io_dest_dir, 'io_runtime.dart'))
    else:
      copyfile(join(HOME, 'runtime', 'bin', filename),
               join(io_dest_dir, 'runtime', filename))

  # Construct lib/io/io_runtime.dart from whole cloth.
  dest_file = open(join(io_dest_dir, 'io_runtime.dart'), 'a')
  for filename in io_runtime_sources:
    assert filename.endswith('.dart')
    if filename == 'io.dart':
      continue
    dest_file.write('#source("runtime/' + filename + '");\n')
  dest_file.close()

  #
  # Create and populate lib/{core, crypto, isolate, json, uri, utf, ...}.
  #

  os.makedirs(join(LIB, 'html'))
  for library in ['_internal', 'core', 'coreimpl', 'crypto', 'isolate',
                  join('html', 'dart2js'), join('html', 'dartium'), 'json',
                  'math', 'mirrors', 'scalarlist', 'uri', 'utf']:
    copytree(join(HOME, 'lib', library), join(LIB, library),
             ignore=ignore_patterns('*.svn', 'doc', '*.py', '*.gypi', '*.sh'))

  # Create and copy pkg.
  PKG = join(SDK_tmp, 'pkg')
  os.makedirs(PKG)

  #
  # Create and populate pkg/{args, intl, logging, meta, unittest}
  #

  for library in ['args', 'htmlescape', 'dartdoc', 'intl', 'logging',
                  'meta', 'unittest']:
    copytree(join(HOME, 'pkg', library), join(PKG, library),
             ignore=ignore_patterns('*.svn', 'doc', 'docs',
                                    '*.py', '*.gypi', '*.sh'))

  # TODO(dgrove): Remove this once issue 4788 is addressed.
  copytree(join(HOME, 'lib', 'compiler'), join(PKG, 'compiler'),
             ignore=ignore_patterns('*.svn', 'doc', '*.py', '*.gypi', '*.sh'))

  ReplaceInFiles(
        [join(LIB, '_internal', 'libraries.dart')],
        [('"compiler/', '"../pkg/compiler/')])

  ReplaceInFiles(
        [join(PKG, 'compiler', 'implementation', 'lib', 'io.dart')],
        [('../../runtime/bin', '../../lib/io/runtime')])

  # Fixup dartdoc
  # TODO(dgrove): Remove this once issue 4788 is addressed.
  ReplaceInFiles([
      join(PKG, 'dartdoc', 'lib', 'src', 'mirrors', 'dart2js_mirror.dart'),
      join(PKG, 'dartdoc', 'lib', 'mirrors_util.dart'),
      join(PKG, 'dartdoc', 'lib', 'classify.dart'),
      join(PKG, 'dartdoc', 'lib', 'src', 'client', 'client-live-nav.dart'),
      join(PKG, 'dartdoc', 'lib', 'src', 'client', 'client-static.dart'),
      join(PKG, 'dartdoc', 'lib', 'dartdoc.dart'),
    ], [
      ("../../lib/compiler",
       "../../pkg/compiler"),
    ])

  # Create and copy tools.
  UTIL = join(SDK_tmp, 'util')
  os.makedirs(UTIL)

  if ShouldCopyAnalyzer():
    # Create and copy Analyzer library into 'util'
    ANALYZER_DEST = join(UTIL, 'analyzer')
    os.makedirs(ANALYZER_DEST)

    analyzer_src_jar = join(ANALYZER_HOME, 'util', 'analyzer',
                            'dart_analyzer.jar')
    analyzer_dest_jar = join(ANALYZER_DEST, 'dart_analyzer.jar')
    copyfile(analyzer_src_jar, analyzer_dest_jar)

    jarsToCopy = [ join("args4j", "2.0.12", "args4j-2.0.12.jar"),
                   join("guava", "r09", "guava-r09.jar"),
                   join("json", "r2_20080312", "json.jar") ]
    for jarToCopy in jarsToCopy:
        dest_dir = join (ANALYZER_DEST, os.path.dirname(jarToCopy))
        os.makedirs(dest_dir)
        dest_file = join (ANALYZER_DEST, jarToCopy)
        src_file = join(ANALYZER_HOME, 'util', 'analyzer', jarToCopy)
        copyfile(src_file, dest_file)

  # Create and populate util/pub.
  copytree(join(HOME, 'utils', 'pub'), join(UTIL, 'pub'),
           ignore=ignore_patterns('.svn', 'sdk'))

  # Copy in 7zip for Windows.
  if utils.GuessOS() == 'win32':
    copytree(join(HOME, 'third_party', '7zip'),
             join(join(UTIL, 'pub'), '7zip'),
             ignore=ignore_patterns('.svn'))

    ReplaceInFiles([
        join(UTIL, 'pub', 'io.dart'),
      ], [
        ("var pathTo7zip = '../../third_party/7zip/7za.exe';",
         "var pathTo7zip = '7zip/7za.exe';"),
      ])

  revision = utils.GetSVNRevision()

  # Copy dart2js.
  CopyDart2Js(build_dir, SDK_tmp, revision)

  # Write the 'revision' file
  if revision is not None:
    with open(os.path.join(SDK_tmp, 'revision'), 'w') as f:
      f.write(revision + '\n')
      f.close()

  Copy(join(HOME, 'README.dart-sdk'), join(SDK_tmp, 'README'))

  move(SDK_tmp, SDK)

if __name__ == '__main__':
  sys.exit(Main(sys.argv))
<commit_msg>Strip the dart binary in the SDK on non-Windows platforms.<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
#
# A script which will be invoked from gyp to create an SDK.
#
# Usage: create_sdk.py sdk_directory
#
# The SDK will be used either from the command-line or from the editor.
# Top structure is
#
# ..dart-sdk/
# ....bin/
# ......dart or dart.exe (executable)
# ......dart.lib (import library for VM native extensions on Windows)
# ......dart2js
# ......dart_analyzer
# ......pub
# ....include/
# ......dart_api.h
# ......dart_debugger_api.h
# ....lib/
# ......_internal/
# ......core/
# ......coreimpl/
# ......crypto/
# ......html/
# ......io/
# ......isolate/
# ......json/
# ......math/
# ......mirrors/
# ......uri/
# ......utf/
# ......scalarlist/
# ....pkg/
# ......args/
# ......compiler/
# ......dartdoc/
#.......htmlescape/
# ......intl/
# ......logging/
# ......meta/
# ......unittest/
# ......(more will come here)
# ....util/
# ......analyzer/
# ........dart_analyzer.jar
# ........(third-party libraries for dart_analyzer)
# ......pub/
# ......(more will come here)

import os
import re
import sys
import subprocess
import tempfile
import utils

# TODO(dgrove): Only import modules following Google style guide.
from os.path import basename, dirname, join, realpath, exists, isdir

# TODO(dgrove): Only import modules following Google style guide.
from shutil import copyfile, copymode, copytree, ignore_patterns, rmtree, move

def ReplaceInFiles(paths, subs):
  '''Reads a series of files, applies a series of substitutions to each, and
     saves them back out. subs should by a list of (pattern, replace) tuples.'''
  for path in paths:
    contents = open(path).read()
    for pattern, replace in subs:
      contents = re.sub(pattern, replace, contents)

    dest = open(path, 'w')
    dest.write(contents)
    dest.close()

def Copy(src, dest):
  copyfile(src, dest)
  copymode(src, dest)

# TODO(zundel): this excludes the analyzer from the sdk build until builders
# have all prerequisite software installed.  Also update dart.gyp.
def ShouldCopyAnalyzer():
  os = utils.GuessOS();
  return os == 'linux' or os == 'macos'

def CopyShellScript(src_file, dest_dir):
  '''Copies a shell/batch script to the given destination directory. Handles
     using the appropriate platform-specific file extension.'''
  file_extension = ''
  if utils.GuessOS() == 'win32':
    file_extension = '.bat'

  src = src_file + file_extension
  dest = join(dest_dir, basename(src_file) + file_extension)
  Copy(src, dest)

def CopyDart2Js(build_dir, sdk_root, revision):
  if revision:
    ReplaceInFiles([os.path.join(sdk_root, 'pkg', 'compiler',
                                 'implementation', 'compiler.dart')],
                   [(r"BUILD_ID = 'build number could not be determined'",
                     r"BUILD_ID = '%s'" % revision)])
  if utils.GuessOS() == 'win32':
    dart2js = os.path.join(sdk_root, 'bin', 'dart2js.bat')
    Copy(os.path.join(build_dir, 'dart2js.bat'), dart2js)
    dartdoc = os.path.join(sdk_root, 'bin', 'dartdoc.bat')
    Copy(os.path.join(build_dir, 'dartdoc.bat'), dartdoc)
    # TODO(dgrove) - fix this once issue 4788 is addressed.
    ReplaceInFiles([dart2js],
                   [(r'%SCRIPTPATH%\.\.\\\.\.\\lib', r'%SCRIPTPATH%..\\pkg')]);
    ReplaceInFiles([dartdoc],
                   [(r'%SCRIPTPATH%\.\.\\\.\.\\pkg', r'%SCRIPTPATH%..\\pkg')]);
  else:
    dart2js = os.path.join(sdk_root, 'bin', 'dart2js')
    Copy(os.path.join(build_dir, 'dart2js'), dart2js)
    dartdoc = os.path.join(sdk_root, 'bin', 'dartdoc')
    Copy(os.path.join(build_dir, 'dartdoc'), dartdoc)

    # TODO(dgrove) - fix this once issue 4788 is addressed.
    ReplaceInFiles([dart2js],
                   [(r'\$BIN_DIR/\.\./\.\./lib', r'$BIN_DIR/../pkg')])
    ReplaceInFiles([dartdoc],
                   [(r'\$BIN_DIR/\.\./\.\.', r'$BIN_DIR/..')])

def Main(argv):
  # Pull in all of the gpyi files which will be munged into the sdk.
  io_runtime_sources = \
    (eval(open("runtime/bin/io_sources.gypi").read()))['sources']

  HOME = dirname(dirname(realpath(__file__)))

  SDK_tmp = tempfile.mkdtemp()
  SDK = argv[1]

  # TODO(dgrove) - deal with architectures that are not ia32.

  if exists(SDK):
    rmtree(SDK)

  # Create and populate sdk/bin.
  BIN = join(SDK_tmp, 'bin')
  os.makedirs(BIN)

  # Copy the Dart VM binary and the Windows Dart VM link library
  # into sdk/bin.
  #
  # TODO(dgrove) - deal with architectures that are not ia32.
  build_dir = os.path.dirname(argv[1])
  dart_file_extension = ''
  analyzer_file_extension = ''
  if utils.GuessOS() == 'win32':
    dart_file_extension = '.exe'
    analyzer_file_extension = '.bat'  # TODO(zundel): test on Windows
    dart_import_lib_src = join(HOME, build_dir, 'dart.lib')
    dart_import_lib_dest = join(BIN, 'dart.lib')
    copyfile(dart_import_lib_src, dart_import_lib_dest)
  dart_src_binary = join(HOME, build_dir, 'dart' + dart_file_extension)
  dart_dest_binary = join(BIN, 'dart' + dart_file_extension)
  copyfile(dart_src_binary, dart_dest_binary)
  copymode(dart_src_binary, dart_dest_binary)
  if utils.GuessOS() != 'win32':
    subprocess.call(['strip', dart_dest_binary])

  if ShouldCopyAnalyzer():
    # Copy analyzer into sdk/bin
    ANALYZER_HOME = join(HOME, build_dir, 'analyzer')
    dart_analyzer_src_binary = join(ANALYZER_HOME, 'bin', 'dart_analyzer')
    dart_analyzer_dest_binary = join(BIN,
        'dart_analyzer' + analyzer_file_extension)
    copyfile(dart_analyzer_src_binary, dart_analyzer_dest_binary)
    copymode(dart_analyzer_src_binary, dart_analyzer_dest_binary)

  # Create pub shell script.
  pub_src_script = join(HOME, 'utils', 'pub', 'sdk', 'pub')
  CopyShellScript(pub_src_script, BIN)

  #
  # Create and populate sdk/include.
  #
  INCLUDE = join(SDK_tmp, 'include')
  os.makedirs(INCLUDE)
  copyfile(join(HOME, 'runtime', 'include', 'dart_api.h'),
           join(INCLUDE, 'dart_api.h'))
  copyfile(join(HOME, 'runtime', 'include', 'dart_debugger_api.h'),
           join(INCLUDE, 'dart_debugger_api.h'))

  #
  # Create and populate sdk/lib.
  #

  LIB = join(SDK_tmp, 'lib')
  os.makedirs(LIB)

  #
  # Create and populate lib/io.
  #
  io_dest_dir = join(LIB, 'io')
  os.makedirs(io_dest_dir)
  os.makedirs(join(io_dest_dir, 'runtime'))
  for filename in io_runtime_sources:
    assert filename.endswith('.dart')
    if filename == 'io.dart':
      copyfile(join(HOME, 'runtime', 'bin', filename),
               join(io_dest_dir, 'io_runtime.dart'))
    else:
      copyfile(join(HOME, 'runtime', 'bin', filename),
               join(io_dest_dir, 'runtime', filename))

  # Construct lib/io/io_runtime.dart from whole cloth.
  dest_file = open(join(io_dest_dir, 'io_runtime.dart'), 'a')
  for filename in io_runtime_sources:
    assert filename.endswith('.dart')
    if filename == 'io.dart':
      continue
    dest_file.write('#source("runtime/' + filename + '");\n')
  dest_file.close()

  #
  # Create and populate lib/{core, crypto, isolate, json, uri, utf, ...}.
  #

  os.makedirs(join(LIB, 'html'))
  for library in ['_internal', 'core', 'coreimpl', 'crypto', 'isolate',
                  join('html', 'dart2js'), join('html', 'dartium'), 'json',
                  'math', 'mirrors', 'scalarlist', 'uri', 'utf']:
    copytree(join(HOME, 'lib', library), join(LIB, library),
             ignore=ignore_patterns('*.svn', 'doc', '*.py', '*.gypi', '*.sh'))

  # Create and copy pkg.
  PKG = join(SDK_tmp, 'pkg')
  os.makedirs(PKG)

  #
  # Create and populate pkg/{args, intl, logging, meta, unittest}
  #

  for library in ['args', 'htmlescape', 'dartdoc', 'intl', 'logging',
                  'meta', 'unittest']:
    copytree(join(HOME, 'pkg', library), join(PKG, library),
             ignore=ignore_patterns('*.svn', 'doc', 'docs',
                                    '*.py', '*.gypi', '*.sh'))

  # TODO(dgrove): Remove this once issue 4788 is addressed.
  copytree(join(HOME, 'lib', 'compiler'), join(PKG, 'compiler'),
             ignore=ignore_patterns('*.svn', 'doc', '*.py', '*.gypi', '*.sh'))

  ReplaceInFiles(
        [join(LIB, '_internal', 'libraries.dart')],
        [('"compiler/', '"../pkg/compiler/')])

  ReplaceInFiles(
        [join(PKG, 'compiler', 'implementation', 'lib', 'io.dart')],
        [('../../runtime/bin', '../../lib/io/runtime')])

  # Fixup dartdoc
  # TODO(dgrove): Remove this once issue 4788 is addressed.
  ReplaceInFiles([
      join(PKG, 'dartdoc', 'lib', 'src', 'mirrors', 'dart2js_mirror.dart'),
      join(PKG, 'dartdoc', 'lib', 'mirrors_util.dart'),
      join(PKG, 'dartdoc', 'lib', 'classify.dart'),
      join(PKG, 'dartdoc', 'lib', 'src', 'client', 'client-live-nav.dart'),
      join(PKG, 'dartdoc', 'lib', 'src', 'client', 'client-static.dart'),
      join(PKG, 'dartdoc', 'lib', 'dartdoc.dart'),
    ], [
      ("../../lib/compiler",
       "../../pkg/compiler"),
    ])

  # Create and copy tools.
  UTIL = join(SDK_tmp, 'util')
  os.makedirs(UTIL)

  if ShouldCopyAnalyzer():
    # Create and copy Analyzer library into 'util'
    ANALYZER_DEST = join(UTIL, 'analyzer')
    os.makedirs(ANALYZER_DEST)

    analyzer_src_jar = join(ANALYZER_HOME, 'util', 'analyzer',
                            'dart_analyzer.jar')
    analyzer_dest_jar = join(ANALYZER_DEST, 'dart_analyzer.jar')
    copyfile(analyzer_src_jar, analyzer_dest_jar)

    jarsToCopy = [ join("args4j", "2.0.12", "args4j-2.0.12.jar"),
                   join("guava", "r09", "guava-r09.jar"),
                   join("json", "r2_20080312", "json.jar") ]
    for jarToCopy in jarsToCopy:
        dest_dir = join (ANALYZER_DEST, os.path.dirname(jarToCopy))
        os.makedirs(dest_dir)
        dest_file = join (ANALYZER_DEST, jarToCopy)
        src_file = join(ANALYZER_HOME, 'util', 'analyzer', jarToCopy)
        copyfile(src_file, dest_file)

  # Create and populate util/pub.
  copytree(join(HOME, 'utils', 'pub'), join(UTIL, 'pub'),
           ignore=ignore_patterns('.svn', 'sdk'))

  # Copy in 7zip for Windows.
  if utils.GuessOS() == 'win32':
    copytree(join(HOME, 'third_party', '7zip'),
             join(join(UTIL, 'pub'), '7zip'),
             ignore=ignore_patterns('.svn'))

    ReplaceInFiles([
        join(UTIL, 'pub', 'io.dart'),
      ], [
        ("var pathTo7zip = '../../third_party/7zip/7za.exe';",
         "var pathTo7zip = '7zip/7za.exe';"),
      ])

  revision = utils.GetSVNRevision()

  # Copy dart2js.
  CopyDart2Js(build_dir, SDK_tmp, revision)

  # Write the 'revision' file
  if revision is not None:
    with open(os.path.join(SDK_tmp, 'revision'), 'w') as f:
      f.write(revision + '\n')
      f.close()

  Copy(join(HOME, 'README.dart-sdk'), join(SDK_tmp, 'README'))

  move(SDK_tmp, SDK)

if __name__ == '__main__':
  sys.exit(Main(sys.argv))
<|endoftext|>


<commit_before>import asyncio
import sys
import time
import json
import discord
from discord.ext import commands
from random import choice as rchoice

config = json.load(open('config.json'))
GitError = 'fatal: Not a git repository (or any of the parent directories): .git'
NoCommit = '*No commit'
NoRelease = '*No release*'

class Etc:
    def __init__(self, bot):
        self.bot = bot
        self.support = 'https://discord.gg/PEW4wx9'

    @commands.command()
    async def uptime(self, ctx):
        """Shows uptime of lolbot"""
        # Thanks Luna you make good code lul
        sec = round(time.time() - self.bot.init_time)
        m, s = divmod(sec, 60)
        h, m = divmod(m, 60)
        d, h = divmod(h, 24)
        upEm = discord.Embed(title='Uptime', colour=0x690E8)
        startedOn = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(self.bot.init_time))
        upEm.add_field(name='Started on', value=startedOn + '\n |')
        upEm.add_field(name='Uptime', value=f' {d} days, {h} hours, {m} minutes and {s} seconds')
        await ctx.send(embed=upEm)

    @commands.command()
    async def ping(self, ctx):
        """Well... you know this"""
        one = time.monotonic()
        a = await ctx.send('wew dude')
        two = time.monotonic()
        ms = round((two - one) * 1000, 2)
        gw = round(self.bot.latency * 1000)
        meme = discord.Embed(title='Pong!', colour=0x690E8)
        meme.add_field(name='Normal ping', value=f'**{ms}**ms')
        meme.add_field(name='Gateway ping', value=f'**{gw}**ms')
        await a.delete()
        await ctx.send(embed=meme)

    @commands.command()
    async def stats(self, ctx):
        """A few stats."""
        statInfo = await ctx.bot.application_info()
        statEmbed = discord.Embed(title='Stats', description='This bot is'
        ' powered by [lolbot](https://github.com/tilda/lolbot), a fast and powerful '
        'Python bot.', colour=0x690E8)
        statEmbed.add_field(name='Owner', value=statInfo.owner.mention + '('
        + str(statInfo.owner) + ' - ID: ' + str(statInfo.owner.id) + ')')
        statEmbed.add_field(name='Python', value=sys.version)
        statEmbed.add_field(name='discord.py', value=discord.__version__)
        statEmbed.add_field(name='Servers', value=len(self.bot.guilds))
        statPool = ['What have you done now?', 'Why should I do this again?', 'Oh..',
                'Where did the RAM go?', 'grumble grumble', 'Please hold.', 'No, just, no.',
                'Have you tried rebooting?', 'memework makes the dreamwork!']
        statEmbed.set_footer(text=rchoice(statPool))
        await ctx.send(embed=statEmbed)

    @commands.command()
    async def invite(self, ctx):
        """Gives a invite for the bot (and also the official server)"""
        info = await ctx.bot.application_info()
        invEmb = discord.Embed(colour=0x690E8)
        invEmb.add_field(name='Invite lolbot', value='[Click here]'
                '(' + discord.utils.oauth_url(info.id) + ')')
        invEmb.add_field(name='Official server', value=str(self.support))
        await ctx.send(embed=invEmb)

    @commands.command()
    async def version(self, ctx):
        """Returns current version of lolbot"""
        with ctx.typing():
            tag = await asyncio.create_subprocess_shell('git describe --abbrev=0 --tags',
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.PIPE)
            commit = await asyncio.create_subprocess_shell('git log --oneline --abbrev=2',
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.PIPE)
            co, ce = map(lambda s: s.decode('utf-8'), await commit.communicate())
            to, te = map(lambda st: st.decode('utf-8'), await tag.communicate())
            if GitError in f'{co}{ce}':
                co = NoCommit
                ce = None
            else:
                pass
            if GitError in f'{to}{te}':
                to = NoRelease
                te = None
            else:
                pass
            if ce and te == None:
                runVal = '*no release*'
                rVal0 = '*no tag*'
            else:
                runVal = co
                rVal0 = to
            e = discord.Embed(colour=0x690E8)
            e.add_field(name='Latest tag', value=rVal0)
            e.add_field(name='Currently running', value=runVal)
            e.set_footer(text='powered by git (and stuff)!')
            await ctx.send(embed=e)

def setup(bot):
  bot.add_cog(Etc(bot))
<commit_msg>cogs[utility]: uptime gets some inline rejection<commit_after>import asyncio
import sys
import time
import json
import discord
from discord.ext import commands
from random import choice as rchoice

config = json.load(open('config.json'))
GitError = 'fatal: Not a git repository (or any of the parent directories): .git'
NoCommit = '*No commit'
NoRelease = '*No release*'

class Etc:
    def __init__(self, bot):
        self.bot = bot
        self.support = 'https://discord.gg/PEW4wx9'

    @commands.command()
    async def uptime(self, ctx):
        """Shows uptime of lolbot"""
        # Thanks Luna you make good code lul
        sec = round(time.time() - self.bot.init_time)
        m, s = divmod(sec, 60)
        h, m = divmod(m, 60)
        d, h = divmod(h, 24)
        upEm = discord.Embed(title='Uptime', colour=0x690E8)
        startedOn = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(self.bot.init_time))
        upEm.add_field(name='Started on', value=startedOn, inline=False)
        upEm.add_field(name='Uptime', value=f' {d} days, {h} hours, {m} minutes and {s} seconds', inline=False)
        await ctx.send(embed=upEm)

    @commands.command()
    async def ping(self, ctx):
        """Well... you know this"""
        one = time.monotonic()
        a = await ctx.send('wew dude')
        two = time.monotonic()
        ms = round((two - one) * 1000, 2)
        gw = round(self.bot.latency * 1000)
        meme = discord.Embed(title='Pong!', colour=0x690E8)
        meme.add_field(name='Normal ping', value=f'**{ms}**ms')
        meme.add_field(name='Gateway ping', value=f'**{gw}**ms')
        await a.delete()
        await ctx.send(embed=meme)

    @commands.command()
    async def stats(self, ctx):
        """A few stats."""
        statInfo = await ctx.bot.application_info()
        statEmbed = discord.Embed(title='Stats', description='This bot is'
        ' powered by [lolbot](https://github.com/tilda/lolbot), a fast and powerful '
        'Python bot.', colour=0x690E8)
        statEmbed.add_field(name='Owner', value=statInfo.owner.mention + '('
        + str(statInfo.owner) + ' - ID: ' + str(statInfo.owner.id) + ')')
        statEmbed.add_field(name='Python', value=sys.version)
        statEmbed.add_field(name='discord.py', value=discord.__version__)
        statEmbed.add_field(name='Servers', value=len(self.bot.guilds))
        statPool = ['What have you done now?', 'Why should I do this again?', 'Oh..',
                'Where did the RAM go?', 'grumble grumble', 'Please hold.', 'No, just, no.',
                'Have you tried rebooting?', 'memework makes the dreamwork!']
        statEmbed.set_footer(text=rchoice(statPool))
        await ctx.send(embed=statEmbed)

    @commands.command()
    async def invite(self, ctx):
        """Gives a invite for the bot (and also the official server)"""
        info = await ctx.bot.application_info()
        invEmb = discord.Embed(colour=0x690E8)
        invEmb.add_field(name='Invite lolbot', value='[Click here]'
                '(' + discord.utils.oauth_url(info.id) + ')')
        invEmb.add_field(name='Official server', value=str(self.support))
        await ctx.send(embed=invEmb)

    @commands.command()
    async def version(self, ctx):
        """Returns current version of lolbot"""
        with ctx.typing():
            tag = await asyncio.create_subprocess_shell('git describe --abbrev=0 --tags',
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.PIPE)
            commit = await asyncio.create_subprocess_shell('git log --oneline --abbrev=2',
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.PIPE)
            co, ce = map(lambda s: s.decode('utf-8'), await commit.communicate())
            to, te = map(lambda st: st.decode('utf-8'), await tag.communicate())
            if GitError in f'{co}{ce}':
                co = NoCommit
                ce = None
            else:
                pass
            if GitError in f'{to}{te}':
                to = NoRelease
                te = None
            else:
                pass
            if ce and te == None:
                runVal = '*no release*'
                rVal0 = '*no tag*'
            else:
                runVal = co
                rVal0 = to
            e = discord.Embed(colour=0x690E8)
            e.add_field(name='Latest tag', value=rVal0)
            e.add_field(name='Currently running', value=runVal)
            e.set_footer(text='powered by git (and stuff)!')
            await ctx.send(embed=e)

def setup(bot):
  bot.add_cog(Etc(bot))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re
import sys
import operator

_re_table_op = re.compile('\(([a-z]*), ([a-z]*)\)')

def parse_trails_log(infile):
  trails = []
  for line in infile.readlines():
    trails.append(_re_table_op.findall(line))

  return trails

def output_summary(trails, outfile):
  ops = map(len, trails)
  ops.sort()

  total_trails = len(ops)
  total_ops = reduce(operator.add, ops)
  max_ops = ops[-1]
  median_ops = ops[total_trails / 2]
  average_ops = float(total_ops) / total_trails

  outfile.write('Total number of trails: %10i\n' % total_trails)
  outfile.write('Total number of ops:    %10i\n' % total_ops)
  outfile.write('max ops/trail:          %10i\n' % max_ops)
  outfile.write('median ops/trail:       %10i\n' % median_ops)
  outfile.write('average ops/trail:      %10.2f\n' % average_ops)
  outfile.write('\n')

def output_frequencies(trails, outfile):
  ops = map(len, trails)

  frequencies = {}
  for rank in ops:
    frequencies[rank] = frequencies.get(rank, 0) + 1

  total_trails = len(ops)

  # custom compare function
  def rf_cmp((a, b), (c, d)):
    c = cmp(d, b)
    if not c:
      c = cmp(a, c)
    return c

  ranks = frequencies.items()
  ranks.sort(rf_cmp)

  outfile.write('ops/trail   frequency   percentage\n')
  for (r, f) in ranks:
    p = float(f) * 100 / total_trails
    outfile.write('%4i         %6i       %5.2f\n' % (r, f, p))

if __name__ == '__main__':
  if len(sys.argv) > 2:
    sys.stderr.write('USAGE: %s [LOG-FILE]\n'
      % sys.argv[0])
    sys.exit(1)

  if len(sys.argv) == 1:
    infile = sys.stdin
  else:
    infile = open(sys.argv[1])

  trails = parse_trails_log(infile)

  output_summary(trails, sys.stdout)
  output_frequencies(trails, sys.stdout)
<commit_msg>* tools/dev/trails.py<commit_after>#!/usr/bin/env python

import re
import sys
import operator

_re_table_op = re.compile('\(([a-z]*), ([a-z]*)\)')

def parse_trails_log(infile):
  trails = []
  for line in infile.readlines():
    trail = _re_table_op.findall(line)
    trail.reverse()
    trails.append(trail)

  return trails

def output_summary(trails, outfile):
  ops = map(len, trails)
  ops.sort()

  total_trails = len(ops)
  total_ops = reduce(operator.add, ops)
  max_ops = ops[-1]
  median_ops = ops[total_trails / 2]
  average_ops = float(total_ops) / total_trails

  outfile.write('Total number of trails: %10i\n' % total_trails)
  outfile.write('Total number of ops:    %10i\n' % total_ops)
  outfile.write('max ops/trail:          %10i\n' % max_ops)
  outfile.write('median ops/trail:       %10i\n' % median_ops)
  outfile.write('average ops/trail:      %10.2f\n' % average_ops)
  outfile.write('\n')

# custom compare function
def freqtable_cmp((a, b), (c, d)):
  c = cmp(d, b)
  if not c:
    c = cmp(a, c)
  return c

def output_trail_length_frequencies(trails, outfile):
  ops = map(len, trails)

  frequencies = {}
  for rank in ops:
    frequencies[rank] = frequencies.get(rank, 0) + 1

  total_trails = len(ops)

  ranks = frequencies.items()
  ranks.sort(freqtable_cmp)

  outfile.write('ops/trail   frequency   percentage\n')
  for (r, f) in ranks:
    p = float(f) * 100 / total_trails
    outfile.write('%4i         %6i       %5.2f\n' % (r, f, p))
  outfile.write('\n')

def output_trail_frequencies(trails, outfile):

  total_trails = len(trails)

  # Since lists can't be keys, turn all trails into tuples
  ttrails = map(tuple, trails)

  freqs = {}
  for trail in ttrails:
    freqs[trail] = freqs.get(trail, 0) + 1

  ops = freqs.items()
  ops.sort(freqtable_cmp)

  outfile.write('frequency   percentage   ops/trail   trail\n')
  for (tt, f) in ops:
    p = float(f) * 100 / total_trails
    t = list(tt)
    outfile.write('%6i        %5.2f       %4i       ' % (f, p, len(t)))

    ### Output the trail itself, in it's own column
    line = str(t[0])
    for op in t[1:]:
      op_str = str(op)
      if len(line) + len(op_str) > 50:
        outfile.write('%s,\n' % line)
        outfile.write('                                     ')
        line = op_str
      else:
        line = line + ', ' + op_str
    outfile.write('%s\n' % line)

  outfile.write('\n')

if __name__ == '__main__':
  if len(sys.argv) > 2:
    sys.stderr.write('USAGE: %s [LOG-FILE]\n'
      % sys.argv[0])
    sys.exit(1)

  if len(sys.argv) == 1:
    infile = sys.stdin
  else:
    infile = open(sys.argv[1])

  trails = parse_trails_log(infile)

  output_summary(trails, sys.stdout)
  output_trail_length_frequencies(trails, sys.stdout)
  output_trail_frequencies(trails, sys.stdout)
<|endoftext|>


<commit_before>from discord.ext import commands
import discord
import utils.errors as uerrs
import pyowm

W_CLEAR_SKY =           ':sunny:'
W_FEW_CLOUDS =          ':white_sun_small_cloud:'
W_SCATTERED_CLOUDS =    ':partly_sunny:'
W_BROKEN_CLOUDS =       ':cloud:'
W_SHOWER_RAIN =         ':cloud_rain:'
W_RAIN =                ':cloud_rain:'
W_THUNDERSTORM =        ':thunder_cloud_rain:'
W_SNOW =                ':snowflake:'
W_MIST =                ':foggy:'

OWM_ICONS = {
    '01d': W_CLEAR_SKY,
    '02d': W_FEW_CLOUDS,
    '03d': W_SCATTERED_CLOUDS,
    '04d': W_BROKEN_CLOUDS,
    '09d': W_SHOWER_RAIN,
    '10d': W_RAIN,
    '11d': W_THUNDERSTORM,
    '13d': W_SNOW,
    '50d': W_MIST,
}

class Weather:
    def __init__(self, bot):
        self.bot = bot
        self.loop = self.bot.loop
        self.config = self.bot.config
        if self.config['owm']:
            self.owm = pyowm.OWM(self.config['owm'])
        else:
            self.owm = None

    @commands.command()
    @commands.cooldown(2, 5)
    async def weather(self, ctx, *, loc: str):
        if self.owm:
            try:
                future = self.loop.run_in_executor(None, \
                    self.owm.weather_at_place, loc)
                observation = await future
            except Exception as e:
                raise uerrs.ServiceError(e)
                return
            w = observation.get_weather()
            _wg = lambda t: w.get_temperature(t)['temp']
            _icon = w.get_weather_icon_name()
            icon = OWM_ICONS.get(_icon, '*<no icon>*')
            status = w.get_detailed_status()
            em = discord.Embed(title=f"Weather for '{loc}'")
            o_location = observation.get_location()
            em.add_field(name='Location', value=f'{o_location.get_name()}')
            em.add_field(name='Situation', value=f'{status} {icon}')
            em.add_field(name='Temperature', value=f'`{_wg("celsius")} °C, {_wg("fahrenheit")} °F`')
            await ctx.send(embed=em)
        else:
            return raise uerrs.ServiceError('This instance does not have a OpenWeatherMap API key configured.')

def setup(bot):
    bot.add_cog(Weather(bot))
<commit_msg>cogs[weather]: suck ass python<commit_after>from discord.ext import commands
import discord
import utils.errors as uerrs
import pyowm

W_CLEAR_SKY =           ':sunny:'
W_FEW_CLOUDS =          ':white_sun_small_cloud:'
W_SCATTERED_CLOUDS =    ':partly_sunny:'
W_BROKEN_CLOUDS =       ':cloud:'
W_SHOWER_RAIN =         ':cloud_rain:'
W_RAIN =                ':cloud_rain:'
W_THUNDERSTORM =        ':thunder_cloud_rain:'
W_SNOW =                ':snowflake:'
W_MIST =                ':foggy:'

OWM_ICONS = {
    '01d': W_CLEAR_SKY,
    '02d': W_FEW_CLOUDS,
    '03d': W_SCATTERED_CLOUDS,
    '04d': W_BROKEN_CLOUDS,
    '09d': W_SHOWER_RAIN,
    '10d': W_RAIN,
    '11d': W_THUNDERSTORM,
    '13d': W_SNOW,
    '50d': W_MIST,
}

class Weather:
    def __init__(self, bot):
        self.bot = bot
        self.loop = self.bot.loop
        self.config = self.bot.config
        if self.config['owm']:
            self.owm = pyowm.OWM(self.config['owm'])
        else:
            self.owm = None

    @commands.command()
    @commands.cooldown(2, 5)
    async def weather(self, ctx, *, loc: str):
        if self.owm:
            try:
                future = self.loop.run_in_executor(None, \
                    self.owm.weather_at_place, loc)
                observation = await future
            except Exception as e:
                raise uerrs.ServiceError(e)
                return
            w = observation.get_weather()
            _wg = lambda t: w.get_temperature(t)['temp']
            _icon = w.get_weather_icon_name()
            icon = OWM_ICONS.get(_icon, '*<no icon>*')
            status = w.get_detailed_status()
            em = discord.Embed(title=f"Weather for '{loc}'")
            o_location = observation.get_location()
            em.add_field(name='Location', value=f'{o_location.get_name()}')
            em.add_field(name='Situation', value=f'{status} {icon}')
            em.add_field(name='Temperature', value=f'`{_wg("celsius")} °C, {_wg("fahrenheit")} °F`')
            await ctx.send(embed=em)
        else:
            raise uerrs.ServiceError('This instance does not have a OpenWeatherMap API key configured.')

def setup(bot):
    bot.add_cog(Weather(bot))
<|endoftext|>


<commit_before>"""
colorhandler.py
A flask script that handles http requests to change GPIO PWN duty cycle
with the pi-blaster library.
Author: Stephen Greene
"""
from flask import Flask, redirect, request, render_template, jsonify
from flask_restful import Resource, Api
from pathlib import Path
from flask_limiter import Limiter
from flask_limiter.util import get_remote_address
import time, hashlib, os

app = Flask(__name__)
limiter = Limiter(
    app,
    key_func=get_remote_address)

REDPIN = 22
GREENPIN = 23
BLUEPIN = 24
currColor = "000000"

@app.route("/set", methods=["POST", "GET"])
@limiter.limit("4 per second")
def parseColor():
    color = request.form["color"]
    try:
        setColor(color)
    except ValueError:
        color = "C1007C"
        setColor(color)
        return jsonify({"Input error" : "Default color used"},
                {"Color" : color})

    return jsonify({"Function" : "Set color"},
        {"Color" : color})

@app.route("/setOff")
def turnOutputOff():
    setColor("000000");
    return jsonify({"Function" : "Set lights to OFF"})

@app.route("/")
def index():
    #basic instructions for connection to /
    return jsonify({"/" : "MOLS Help Request"},
        {"/set" : "Color=FFFFFF (6 digit hex format, no # sign)"},
        {"/setOff" : "No params: turns lights off (hex value 000000)"},
        {"Current color" : currColor})

def setColor(color):
    global currColor
    currColor = color
    red = str(REDPIN) + "=" + "%.3f" % (int(color[:2], 16) / 255.0)
    green = str(GREENPIN) + "=" + "%.3f" % (int(color[2:4], 16) / 255.0)
    blue = str(BLUEPIN) + "=" + "%.3f" % (int(color[4:], 16) / 255.0)
    os.system("echo " + red + " >> /dev/pi-blaster")
    os.system("echo " + green + " >> /dev/pi-blaster")
    os.system("echo " + blue + " >> /dev/pi-blaster")

if __name__ == "__main__":
   app.run()
<commit_msg>added rate limit response<commit_after>"""
colorhandler.py
A flask script that handles http requests to change GPIO PWN duty cycle
with the pi-blaster library.
Author: Stephen Greene
"""
from flask import Flask, redirect, request, render_template, jsonify
from flask_restful import Resource, Api
from pathlib import Path
from flask_limiter import Limiter
from flask_limiter.util import get_remote_address
import time, hashlib, os

app = Flask(__name__)
limiter = Limiter(
    app,
    key_func=get_remote_address)

REDPIN = 22
GREENPIN = 23
BLUEPIN = 24
currColor = "000000"

@app.route("/set", methods=["POST", "GET"])
@limiter.limit("4 per second")
def parseColor():
    color = request.form["color"]
    try:
        setColor(color)
    except ValueError:
        color = "C1007C"
        setColor(color)
        return jsonify({"Input error" : "Default color used"},
                {"Color" : color})

    return jsonify({"Function" : "Set color"},
        {"Color" : color})

@app.route("/setOff")
def turnOutputOff():
    setColor("000000");
    return jsonify({"Function" : "Set lights to OFF"})

@app.route("/")
def index():
    #basic instructions for connection to /
    return jsonify({"/" : "MOLS Help Request"},
        {"/set" : "Color=FFFFFF (6 digit hex format, no # sign)"},
        {"/setOff" : "No params: turns lights off (hex value 000000)"},
        {"Current color" : currColor})

@app.errorhandler(429)
def ratelimit_handler(e):
    return jsonify(error="ratelimit exceeded %s" % e.description)

def setColor(color):
    global currColor
    currColor = color
    red = str(REDPIN) + "=" + "%.3f" % (int(color[:2], 16) / 255.0)
    green = str(GREENPIN) + "=" + "%.3f" % (int(color[2:4], 16) / 255.0)
    blue = str(BLUEPIN) + "=" + "%.3f" % (int(color[4:], 16) / 255.0)
    os.system("echo " + red + " >> /dev/pi-blaster")
    os.system("echo " + green + " >> /dev/pi-blaster")
    os.system("echo " + blue + " >> /dev/pi-blaster")

if __name__ == "__main__":
   app.run()
<|endoftext|>


<commit_before>"""
Find color schemes with optimally distinct colors using Delta E (CIE2000).

Steps to generate optimal color schemes:

  1. Compile list of potential colors.
  2. Calculate Delta E between all colors.
  3. Compile list of potential schemes with a certain number of colors each.
  4. Discard schemes if they contain hues that are too similar or if their
     Delta E of any two colors is too low.
  5. Output remaining color schemes.

"""

from datetime import datetime
from itertools import combinations
from itertools import zip_longest
from math import factorial
from multiprocessing import cpu_count
from multiprocessing import Lock
from multiprocessing import Manager
from multiprocessing import Pool
from multiprocessing import Value

from colormath.color_conversions import convert_color
#from colormath.color_diff import delta_e_cie1976 as delta_e
#from colormath.color_diff import delta_e_cie1994 as delta_e
from colormath.color_diff import delta_e_cie2000 as delta_e
#from colormath.color_diff import delta_e_cmc as delta_e
from colormath.color_objects import HSLColor
from colormath.color_objects import LabColor
from colormath.color_objects import sRGBColor

# Number of colors per scheme
n = 6
# Output formats: 'hex', 'hsl', 'rgb' (0–1) or 'rgb_upscaled' (0–255)
color_codes = ['hex']
# Bright or dark background
bright = True
# Spacing of hues in degrees, smaller spacing results in more potential schemes
# Note: In edge cases larger steps can give slightly better results
# (steps of 1° yield 2899305949260 schemes)
# Factors of 360:
# 1, 2, 3, 4, 5, 6, 8, 9, 10, 12, 15, 18, 20, 24, 30, 36, 40, 45, 60
hue_step = 12
# Discard scheme if any hue combination is less than this many degrees apart
min_hue_diff = 12
# Enable to use more RAM for better performance, disable to process larger
# numbers of schemes that will not fit into your RAM
load_schemes_into_ram = False
# Output schemes as soon as they are found
output_schemes_early = False
# Parameters depending on bright or dark background
if bright:
    # Approximate hue of highlight color (e.g. background color for matches
    # when searching in a man page), can be 'None'
    highlight = 60
    # Minimum Delta E between the highlight color and the background
    min_delta_background_highlight = 20
    # Minimum Delta E between all other colors and the background
    min_delta_background_color = 50
    # Luminance is adjusted this much each step until the required delta to the
    # background is achieved
    lum_adjust_highlight = -.001
    lum_adjust_color = -.001
    # Fixed colors, use for example:
    #   sRGBColor.new_from_rgb_hex('#000000')
    #   sRGBColor(128, 0, 255, is_upscaled=True)
    #   sRGBColor(.5, 0, 1)
    #   HSLColor(0, 0, .8)
    # Background
    background = sRGBColor.new_from_rgb_hex('#ffffff')
else:
    # Approximate hue of highlight color (e.g. background color for matches
    # when searching in a man page), can be 'None'
    highlight = 240
    # Minimum Delta E between the highlight color and the background
    min_delta_background_highlight = 20
    # Minimum Delta E between all other colors and the background
    min_delta_background_color = 50
    # Luminance is adjusted this much each step until the required delta to the
    # background is achieved
    lum_adjust_highlight = .001
    lum_adjust_color = .001
    # Fixed colors, use for example:
    #   sRGBColor.new_from_rgb_hex('#000000')
    #   sRGBColor(128, 0, 255, is_upscaled=True)
    #   sRGBColor(.5, 0, 1)
    #   HSLColor(0, 0, .8)
    # Background
    background = sRGBColor.new_from_rgb_hex('#000000')

# Convert colors
background = convert_color(background, LabColor)

def convert_hue(hue, adjust=lum_adjust_color,
                min_delta=min_delta_background_color):
    """Create a color from a hue and adjust its luminance until the Delta E
    between the color and the background is sufficient.
    """
    color_hsl = HSLColor(hue, 1, .5)
    color_lab = convert_color(color_hsl, LabColor)
    while True:
        if delta_e(color_lab, background) < min_delta:
            color_hsl.hsl_l += adjust
            color_lab = convert_color(color_hsl, LabColor)
        else:
            return [hue, color_lab]

def calculate_delta(colors):
    """Calculate Delta E between two colors."""
    # Structure of dictionary: dict([(hue_1, hue_2): delta_e, ...])
    global deltas
    deltas[(colors[0][0], colors[1][0])] = \
        delta_e(colors[0][1], colors[1][1])

def check_scheme(scheme):
    """Check a scheme for similar hues and low Delta E values. Discard if
    insufficient.
    """
    # Due to lazy evaluation with grouper(), scheme can be None
    if not scheme:
        return
    global lock

    # Show progress
    global processed
    with lock:
        processed.value += 1
        processed_schemes = processed.value
    if processed_schemes % 100000 == 0:
        elapsed = datetime.utcnow() - start_time
        progress = processed_schemes / total_schemes
        print('== Progress: {}/{} {}%'.format(
            processed_schemes, total_schemes,
            int(round(progress * 100))))
        print('   Elapsed time: {}'.format(str(elapsed).split('.')[0]))
        print('   Time left: {}'.format(
            str(elapsed * (1 / progress) - elapsed).split('.')[0]))

    # Check hues
    min_hue_diff_scheme = \
        min([scheme[i + 1][0] - scheme[i][0] for i in range(5)]
            + [scheme[0][0] + 360 - scheme[-1][0]])
    if min_hue_diff_scheme < min_hue_diff:
        return

    # Check Delta E values
    # Initialize variable for loop
    min_delta = float('inf')
    global current_min_delta
    for x, y in combinations(scheme, 2):
        delta = deltas[(x[0], y[0])]
        if delta < current_min_delta.value:
            return
        # This is the Delta E of the scheme’s worst color combination
        min_delta = min(min_delta, delta)
    new_best = False
    notify = False
    with lock:
        if current_min_delta.value < min_delta:
            new_best = True
            if int(current_min_delta.value) != int(min_delta):
                notify = True
            current_min_delta.value = min_delta
    if notify:
        print('== New minimum Delta E: {}'.format(min_delta))
    if new_best and output_schemes_early:
        output_scheme(finalize_scheme(
            [min_delta, min_hue_diff_scheme, scheme]))
    return [min_delta, min_hue_diff_scheme, scheme]

def finalize_scheme(scheme):
    """Finalize a scheme by brightening its highlight color until it reaches
    the chosen Delta E to the background and ordering the colors of the scheme.
    """
    scheme[2] = list(scheme[2])
    hues = [hue for hue, _ in scheme[2]]
    # Make sure the first hue is closest to red
    if 360 - max(hues) < min(hues):
        scheme[2].insert(0, scheme[2].pop())
        hues.insert(0, hues.pop())
    if highlight:
        # Find nearest color to chosen highlight color
        highlight_hue = min(hues, key=lambda x: abs(x - highlight))
        highlight_pos = hues.index(highlight_hue)
        # Adjust luminance of highlight color
        color = convert_hue(highlight_hue, lum_adjust_highlight,
                            min_delta_background_highlight)[1]
        # Insert final highlight color into scheme
        scheme[2][highlight_pos] = (highlight_hue, color)
    return scheme

def color_to_str(color):
    """Return a color’s hex, sRGB or HSL representations."""
    s = []
    if 'hex' in color_codes:
        s.append(str(convert_color(color, sRGBColor).get_rgb_hex()))
    if 'rgb_upscaled' in color_codes:
        s.append('rgb{}'.format(
            str(convert_color(color, sRGBColor).get_upscaled_value_tuple())))
    if 'rgb' in color_codes:
        s.append('rgb{}'.format(
            str(convert_color(color, sRGBColor).get_value_tuple())))
    if 'hsl' in color_codes:
        s.append('hsl{}'.format(
            str(convert_color(color, HSLColor).get_value_tuple())))
    return ', '.join(s)

def output_scheme(scheme):
    """Output a color scheme."""
    hues = [color[0] for color in scheme[2]]
    if scheme[1] - min_hue_diff < hue_step:
        print('# Warning: Scheme borders minimal hue difference')
    equation = delta_e.__name__.split('_')[-1]
    print('# Minimum Delta E ({}): {}'.format(equation, scheme[0]))
    print('# Minimum hue difference: {}'.format(scheme[1]))
    print('# Hues: {}'.format(' '.join(map(str, hues))))
    for i in range(n):
        color_str = color_to_str(scheme[2][i][1])
        print(f'Color {i}: {color_str}')

def grouper(iterable, n, fillvalue=None):
    """Collect data into fixed-length chunks or blocks."""
    # Source: https://docs.python.org/3/library/itertools.html#recipes
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
    args = [iter(iterable)] * n
    return zip_longest(*args, fillvalue=fillvalue)

if __name__ == "__main__":
    start_time = datetime.utcnow()
    processes = cpu_count()
    manager = Manager()

    # Compile list of hues, optionally including highlight hue
    if highlight:
        hues = range(highlight, highlight + 360, hue_step)
        hues = sorted([hue if hue < 360 else hue - 360 for hue in hues])
    else:
        hues = range(0, 360, hue_step)

    # Compile list of potential colors as (hue, LabColor) pairs
    colors = manager.list()
    with Pool(processes) as pool:
        colors = pool.map(convert_hue, hues)
        pool.close()
        pool.join()
    colors.sort()
    print('Colors prepared')

    # Calculate Delta E between all colors
    deltas = manager.dict()
    with Pool(processes) as pool:
        pool.map(calculate_delta, combinations(colors, 2))
        pool.close()
        pool.join()
    # Making a dictionary now from multiprocessing.managers.DictProxy will
    # increase subsequent performance a lot
    deltas = dict(deltas)
    print('Color deltas calculated')

    # Compile list of potential schemes with n colors each
    schemes = combinations(colors, n)
    total_schemes = int(
        factorial(len(colors))
        / (factorial(n) * factorial(len(colors) - n)))
    print('Total schemes: {}'.format(total_schemes))

    # Discard schemes if they contain hues that are too similar or if their
    # Delta E of any two colors is too low
    current_min_delta = Value('f', 0)
    processed = Value('i', 0)
    lock = Lock()
    schemes_checked = []
    # Process all other schemes
    with Pool(processes) as pool:
        if load_schemes_into_ram:
            schemes_checked.extend(pool.map(check_scheme, schemes))
        else:
            # Chunks of 500000 have been fastest in my testing, but it probably
            # depends on the number of schemes
            # (Note that there is some overhead for the last chunk because it
            # gets filled with None values to reach the chunk size)
            for chunk in grouper(schemes, 500000):
                schemes_checked.extend(pool.map(check_scheme, chunk))
        pool.close()
        pool.join()

    # Remove None values and sort schemes by their minimum Delta E
    schemes_checked = sorted([scheme for scheme in schemes_checked if scheme],
                             key=lambda x: x[0], reverse=True)

    elapsed = datetime.utcnow() - start_time
    print('Total time: {}'.format(str(elapsed).split('.')[0]))

    # Output optimal color schemes
    print('=====================')
    print('Optimal color schemes')
    print('=====================')
    last_delta = None
    for scheme in schemes_checked:
        if last_delta and scheme[0] < last_delta:
            break
        output_scheme(finalize_scheme(scheme))
        last_delta = scheme[0]
<commit_msg>Remove unnecessary Delta E comparisons.<commit_after>"""
Find color schemes with optimally distinct colors using Delta E (CIE2000).

Steps to generate optimal color schemes:

  1. Compile list of potential colors.
  2. Calculate Delta E between all colors.
  3. Compile list of potential schemes with a certain number of colors each.
  4. Discard schemes if they contain hues that are too similar or if their
     Delta E of any two colors is too low.
  5. Output remaining color schemes.

"""

from datetime import datetime
from itertools import combinations
from itertools import zip_longest
from math import factorial
from multiprocessing import cpu_count
from multiprocessing import Lock
from multiprocessing import Manager
from multiprocessing import Pool
from multiprocessing import Value

from colormath.color_conversions import convert_color
#from colormath.color_diff import delta_e_cie1976 as delta_e
#from colormath.color_diff import delta_e_cie1994 as delta_e
from colormath.color_diff import delta_e_cie2000 as delta_e
#from colormath.color_diff import delta_e_cmc as delta_e
from colormath.color_objects import HSLColor
from colormath.color_objects import LabColor
from colormath.color_objects import sRGBColor

# Number of colors per scheme
n = 6
# Output formats: 'hex', 'hsl', 'rgb' (0–1) or 'rgb_upscaled' (0–255)
color_codes = ['hex']
# Bright or dark background
bright = True
# Spacing of hues in degrees, smaller spacing results in more potential schemes
# Note: In edge cases larger steps can give slightly better results
# (steps of 1° yield 2899305949260 schemes)
# Factors of 360:
# 1, 2, 3, 4, 5, 6, 8, 9, 10, 12, 15, 18, 20, 24, 30, 36, 40, 45, 60
hue_step = 12
# Discard scheme if any hue combination is less than this many degrees apart
min_hue_diff = 12
# Enable to use more RAM for better performance, disable to process larger
# numbers of schemes that will not fit into your RAM
load_schemes_into_ram = False
# Output schemes as soon as they are found
output_schemes_early = False
# Parameters depending on bright or dark background
if bright:
    # Approximate hue of highlight color (e.g. background color for matches
    # when searching in a man page), can be 'None'
    highlight = 60
    # Minimum Delta E between the highlight color and the background
    min_delta_background_highlight = 20
    # Minimum Delta E between all other colors and the background
    min_delta_background_color = 50
    # Luminance is adjusted this much each step until the required delta to the
    # background is achieved
    lum_adjust_highlight = -.001
    lum_adjust_color = -.001
    # Fixed colors, use for example:
    #   sRGBColor.new_from_rgb_hex('#000000')
    #   sRGBColor(128, 0, 255, is_upscaled=True)
    #   sRGBColor(.5, 0, 1)
    #   HSLColor(0, 0, .8)
    # Background
    background = sRGBColor.new_from_rgb_hex('#ffffff')
else:
    # Approximate hue of highlight color (e.g. background color for matches
    # when searching in a man page), can be 'None'
    highlight = 240
    # Minimum Delta E between the highlight color and the background
    min_delta_background_highlight = 20
    # Minimum Delta E between all other colors and the background
    min_delta_background_color = 50
    # Luminance is adjusted this much each step until the required delta to the
    # background is achieved
    lum_adjust_highlight = .001
    lum_adjust_color = .001
    # Fixed colors, use for example:
    #   sRGBColor.new_from_rgb_hex('#000000')
    #   sRGBColor(128, 0, 255, is_upscaled=True)
    #   sRGBColor(.5, 0, 1)
    #   HSLColor(0, 0, .8)
    # Background
    background = sRGBColor.new_from_rgb_hex('#000000')

# Convert colors
background = convert_color(background, LabColor)

def convert_hue(hue, adjust=lum_adjust_color,
                min_delta=min_delta_background_color):
    """Create a color from a hue and adjust its luminance until the Delta E
    between the color and the background is sufficient.
    """
    color_hsl = HSLColor(hue, 1, .5)
    color_lab = convert_color(color_hsl, LabColor)
    while True:
        if delta_e(color_lab, background) < min_delta:
            color_hsl.hsl_l += adjust
            color_lab = convert_color(color_hsl, LabColor)
        else:
            return [hue, color_lab]

def calculate_delta(colors):
    """Calculate Delta E between two colors."""
    # Structure of dictionary: dict([(hue_1, hue_2): delta_e, ...])
    global deltas
    deltas[(colors[0][0], colors[1][0])] = \
        delta_e(colors[0][1], colors[1][1])

def check_scheme(scheme):
    """Check a scheme for similar hues and low Delta E values. Discard if
    insufficient.
    """
    # Due to lazy evaluation with grouper(), scheme can be None
    if not scheme:
        return
    global lock

    # Show progress
    global processed
    with lock:
        processed.value += 1
        processed_schemes = processed.value
    if processed_schemes % 100000 == 0:
        elapsed = datetime.utcnow() - start_time
        progress = processed_schemes / total_schemes
        print('== Progress: {}/{} {}%'.format(
            processed_schemes, total_schemes,
            int(round(progress * 100))))
        print('   Elapsed time: {}'.format(str(elapsed).split('.')[0]))
        print('   Time left: {}'.format(
            str(elapsed * (1 / progress) - elapsed).split('.')[0]))

    # Check hues
    min_hue_diff_scheme = \
        min([scheme[i + 1][0] - scheme[i][0] for i in range(5)]
            + [scheme[0][0] + 360 - scheme[-1][0]])
    if min_hue_diff_scheme < min_hue_diff:
        return

    # Check Delta E values
    # Initialize variable for loop
    min_delta = float('inf')
    global current_min_delta
    # Compare Delta E of adjacent hues
    for c1, c2 in pairs(scheme):
        delta = deltas[(c1[0], c2[0])]
        if delta < current_min_delta.value:
            return
        # This is the Delta E of the scheme’s worst color combination
        min_delta = min(min_delta, delta)
    new_best = False
    notify = False
    with lock:
        if current_min_delta.value < min_delta:
            new_best = True
            if int(current_min_delta.value) != int(min_delta):
                notify = True
            current_min_delta.value = min_delta
    if notify:
        print('== New minimum Delta E: {}'.format(min_delta))
    if new_best and output_schemes_early:
        output_scheme(finalize_scheme(
            [min_delta, min_hue_diff_scheme, scheme]))
    return [min_delta, min_hue_diff_scheme, scheme]

def finalize_scheme(scheme):
    """Finalize a scheme by brightening its highlight color until it reaches
    the chosen Delta E to the background and ordering the colors of the scheme.
    """
    scheme[2] = list(scheme[2])
    hues = [hue for hue, _ in scheme[2]]
    # Make sure the first hue is closest to red
    if 360 - max(hues) < min(hues):
        scheme[2].insert(0, scheme[2].pop())
        hues.insert(0, hues.pop())
    if highlight:
        # Find nearest color to chosen highlight color
        highlight_hue = min(hues, key=lambda x: abs(x - highlight))
        highlight_pos = hues.index(highlight_hue)
        # Adjust luminance of highlight color
        color = convert_hue(highlight_hue, lum_adjust_highlight,
                            min_delta_background_highlight)[1]
        # Insert final highlight color into scheme
        scheme[2][highlight_pos] = (highlight_hue, color)
    return scheme

def color_to_str(color):
    """Return a color’s hex, sRGB or HSL representations."""
    s = []
    if 'hex' in color_codes:
        s.append(str(convert_color(color, sRGBColor).get_rgb_hex()))
    if 'rgb_upscaled' in color_codes:
        s.append('rgb{}'.format(
            str(convert_color(color, sRGBColor).get_upscaled_value_tuple())))
    if 'rgb' in color_codes:
        s.append('rgb{}'.format(
            str(convert_color(color, sRGBColor).get_value_tuple())))
    if 'hsl' in color_codes:
        s.append('hsl{}'.format(
            str(convert_color(color, HSLColor).get_value_tuple())))
    return ', '.join(s)

def output_scheme(scheme):
    """Output a color scheme."""
    hues = [color[0] for color in scheme[2]]
    if scheme[1] - min_hue_diff < hue_step:
        print('# Warning: Scheme borders minimal hue difference')
    equation = delta_e.__name__.split('_')[-1]
    print('# Minimum Delta E ({}): {}'.format(equation, scheme[0]))
    print('# Minimum hue difference: {}'.format(scheme[1]))
    print('# Hues: {}'.format(' '.join(map(str, hues))))
    for i in range(n):
        color_str = color_to_str(scheme[2][i][1])
        print(f'Color {i}: {color_str}')

def grouper(iterable, n, fillvalue=None):
    """Collect data into fixed-length chunks or blocks."""
    # Source: https://docs.python.org/3/library/itertools.html#recipes
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
    args = [iter(iterable)] * n
    return zip_longest(*args, fillvalue=fillvalue)

def pairs(lst):
    """Iterate over pairs in a list and wrap around."""
    # Source: https://stackoverflow.com/a/1257446
    i = iter(lst)
    first = prev = item = next(i)
    for item in i:
        yield prev, item
        prev = item
    # Order matters for dictionary lookup
    yield first, item

if __name__ == "__main__":
    start_time = datetime.utcnow()
    processes = cpu_count()
    manager = Manager()

    # Compile list of hues, optionally including highlight hue
    if highlight:
        hues = range(highlight, highlight + 360, hue_step)
        hues = sorted([hue if hue < 360 else hue - 360 for hue in hues])
    else:
        hues = range(0, 360, hue_step)

    # Compile list of potential colors as (hue, LabColor) pairs
    colors = manager.list()
    with Pool(processes) as pool:
        colors = pool.map(convert_hue, hues)
        pool.close()
        pool.join()
    colors.sort()
    print('Colors prepared')

    # Calculate Delta E between all colors
    deltas = manager.dict()
    with Pool(processes) as pool:
        pool.map(calculate_delta, combinations(colors, 2))
        pool.close()
        pool.join()
    # Making a dictionary now from multiprocessing.managers.DictProxy will
    # increase subsequent performance a lot
    deltas = dict(deltas)
    print('Color deltas calculated')

    # Compile list of potential schemes with n colors each
    schemes = combinations(colors, n)
    total_schemes = int(
        factorial(len(colors))
        / (factorial(n) * factorial(len(colors) - n)))
    print('Total schemes: {}'.format(total_schemes))

    # Discard schemes if they contain hues that are too similar or if their
    # Delta E of any two colors is too low
    current_min_delta = Value('f', 0)
    processed = Value('i', 0)
    lock = Lock()
    schemes_checked = []
    # Process all other schemes
    with Pool(processes) as pool:
        if load_schemes_into_ram:
            schemes_checked.extend(pool.map(check_scheme, schemes))
        else:
            # Chunks of 500000 have been fastest in my testing, but it probably
            # depends on the number of schemes
            # (Note that there is some overhead for the last chunk because it
            # gets filled with None values to reach the chunk size)
            for chunk in grouper(schemes, 500000):
                schemes_checked.extend(pool.map(check_scheme, chunk))
        pool.close()
        pool.join()

    # Remove None values and sort schemes by their minimum Delta E
    schemes_checked = sorted([scheme for scheme in schemes_checked if scheme],
                             key=lambda x: x[0], reverse=True)

    elapsed = datetime.utcnow() - start_time
    print('Total time: {}'.format(str(elapsed).split('.')[0]))

    # Output optimal color schemes
    print('=====================')
    print('Optimal color schemes')
    print('=====================')
    last_delta = None
    for scheme in schemes_checked:
        if last_delta and scheme[0] < last_delta:
            break
        output_scheme(finalize_scheme(scheme))
        last_delta = scheme[0]
<|endoftext|>


<commit_before>import luigi
#import vmd2json
from luigi.parameter import MissingParameterException
import time

#print("")
#print("hi")
#print("")

class CheckFile(luigi.ExternalTask):
    # see what kind of file we are receiving
    in_file = luigi.Parameter()

    def test_parameter(self):
        self.assertRaises(luigi.parameter.ParameterException)
    #try:
    #    len(in_file) > 1
    #except MissingParameterException as e:
    #    print("mising parameter - supply an input file path.  Error: %s", e)
    #    raise e

    #def requires(self):
    #    return []

    def output(self):
        return luigi.LocalTarget(self.in_file)

    #def run(self):
    #    pass

class ConvertFile(luigi.Task):
    # do the conversion
    in_file = luigi.Parameter()

    def requires(self):
        return CheckFile(in_file)

    def output(self):
        return luigi.LocalTarget("converted{}.json".format(time.time()))

    def run(self):
        #with self.input()[0].open() as fin, self.output().open('w') as fout:
        #    for line in fin:
        #        n = int(line.strip())
        #        out = n * n
        #        fout.write("{}:{}\n".format(n, out))

        # for python 3, need to convert xml to binary
        with open(self.input()[0], "rb") as f:
            output_dict = xmltodict.parse(f, xml_attribs=True)
            json.dumps(output_dict, indent=4)
            print('output ', output_dict)

        with open(self.output(), 'w') as file:
            json.dump(output_dict, file)

if __name__ == "__main__":
    luigi.run(['ConvertFile', '--workers', '1'])<commit_msg>Added files via upload<commit_after>import luigi
#import vmd2json
from luigi.parameter import ParameterException
from luigi.contrib.opener import OpenerTarget
import time
import xmltodict
import json

#print("")
#print("hi")
#print("")

class CheckFile(luigi.Task):
    # see what kind of file we are receiving
    in_file = luigi.Parameter()
    #in_file = ('fruits.xml')
    print('in file ', in_file)

    def test_parameter(self):
        self.assertRaises(luigi.parameter.ParameterException)
    #try:
    #    len(in_file) > 1
    #except MissingParameterException as e:
    #    print("mising parameter - supply an input file path.  Error: %s", e)
    #    raise e

    #def requires(self):
    #    return []

    def output(self):
        #return luigi.LocalTarget(self.in_file)
        return self.in_file

    def run(self):
        self.output().open('r').close()

class ConvertFile(luigi.Task):
    # do the conversion
    #in_file = luigi.Parameter()
    in_file = OpenerTarget('file://usr/local/app1/scripts/test/fruits.xml')

    def requires(self):
        print("in requires")
        return CheckFile(in_file=self.in_file)

    def output(self):
        return luigi.LocalTarget(
            path="/usr/local/app1/scripts/test/converted{}.json".format(time.time())
        )

    def run(self):
        #with self.input()[0].open() as fin, self.output().open('w') as fout:
        #    for line in fin:
        #        n = int(line.strip())
        #        out = n * n
        #        fout.write("{}:{}\n".format(n, out))

        # for python 3, need to convert xml to binary - six do this?
        #with self.in_file.open("rb") as f:
        with self.in_file.open("r") as f:
            output_dict = xmltodict.parse(f, xml_attribs=True)
            json.dumps(output_dict, indent=4)
            print('output ', output_dict)

        with self.output().open('w') as file:
            json.dump(output_dict, file)

if __name__ == "__main__":
    luigi.run(['ConvertFile', '--workers', '1'])<|endoftext|>


<commit_before>#
# Created as part of the StratusLab project (http://stratuslab.eu),
# co-funded by the European Commission under the Grant Agreement
# INFSO-RI-261552."
#
# Copyright (c) 2010, SixSq Sarl
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import datetime
import inspect
import os.path
import time
import unittest
import urllib2

from stratuslab.Monitor import Monitor
from stratuslab.Registrar import Registrar
from stratuslab.Runner import Runner
from stratuslab.Uploader import Uploader
from stratuslab.Exceptions import NetworkException
from stratuslab.Exceptions import ConfigurationException
from stratuslab.Exceptions import ExecutionException
from stratuslab.ConfigHolder import ConfigHolder
from stratuslab.Util import execute
from stratuslab.Util import generateSshKeyPair
from stratuslab.Util import ping
from stratuslab.Util import printError
from stratuslab.Util import sshCmd
import Util

class Testor(unittest.TestCase):

    configHolder = None
    testNames = []

    def __init__(self, methodName='dummy'):
        super(Testor, self).__init__(methodName)

        self.vmIds = []
        self.sshKey = '/tmp/id_rsa_smoke_test'
        self.sshKeyPub = self.sshKey + '.pub'
        self.testsToRun = []

        Testor.configHolder.assign(self)
        self._setEnvVars()

    def _setEnvVars(self):
        self._setSingleEnvVar('appRepoUsername', 'STRATUSLAB_REPO_USERNAME')
        self._setSingleEnvVar('appRepoPassword', 'STRATUSLAB_REPO_PASSWORD')
        self._setSingleEnvVar('appRepoUrl', 'STRATUSLAB_REPO_ADDRESS')
        self._setSingleEnvVar('username', 'STRATUSLAB_USERNAME')
        self._setSingleEnvVar('password', 'STRATUSLAB_PASSWORD')
        self._fillEndpointOption()

    def _setSingleEnvVar(self, field, env):
        if env in os.environ:
            setattr(self, field, os.environ[env])

    def _setOption(self, key, value):
        Testor.configHolder.options[key] = value

    def _fillEndpointOption(self):
        if Util.envEndpoint in os.environ:
            return
        if not self.frontendIp:
            raise ConfigurationException('Missing environment variable %s or configuration parameter frontend_ip' % Util.envEndpoint)
        os.environ[Util.envEndpoint] = 'http://%s:2633/RPC2' % self.frontendIp

    def dummy(self):
        pass

    def runTests(self):
        suite = unittest.TestSuite()
        tests = []
        if self.testNames:
            tests = self.testNames
        else:
            tests = self._extractTestMethodNames()

        for test in tests:
            suite.addTest(Testor(test))

        testResult = unittest.TextTestRunner(verbosity=2).run(suite)
        return testResult.wasSuccessful()

    def runMethod(self, method):
        return method()

    def runInstancePublicNetworkTest(self):
        '''Start new instance, ping it via public network and ssh into it, then stop it'''
        self._runInstanceTest()

    def runInstanceLocalNetworkTest_(self):
        '''Start new instance, ping it via local network and ssh into it, then stop it'''
        self._runInstanceTest(True)

    def _runInstanceTest(self, withLocalNetwork=False):
        runner = self._startVm(withLocalNetwork)
        self._repeatCall(self._ping, runner)
        self._repeatCall(self._loginViaSsh, runner)
        self._stopVm(runner)

    def _prepareLog(self, logFile):
        log = open(logFile,'aw')
        log.write('\n'*3 + '=' * 60 + '\n')
        log.write(str(datetime.datetime.now()) + '\n')
        log.write('=' * 60 + '\n'*3)
        return log

    def _startVm(self, withLocalNetwork=False):
        generateSshKeyPair(self.sshKey)

        options = Runner.defaultRunOptions()
        options['username'] = self.oneUsername
        options['password'] = self.onePassword
        options['userKey'] = self.sshKeyPub
        options['verboseLevel'] = self.verboseLevel

        if withLocalNetwork:
            options['isLocalIp'] = True

        configHolder = ConfigHolder(options)
        image = 'http://appliances.stratuslab.org/images/base/ttylinux-9.5-i486-base/1.0/ttylinux-9.5-i486-base-1.0.img.gz'

        runner = Runner(image, configHolder)
        self.vmIds = runner.runInstance()

        for id in self.vmIds:
            vmStarted = runner.waitUntilVmRunningOrTimeout(id)
            if not vmStarted:
                printError('Failed to start VM id: %s' % id)

        return runner

    def _repeatCall(self, method, args=[]):
        numberOfRepetition = 60
        for _ in range(numberOfRepetition):
            failed = False
            try:
                if args:
                    method(args)
                else:
                    method()
            except ExecutionException:
                failed = True
                time.sleep(10)
            else:
                break

        if failed:
            printError('Failed executing method %s %s times, giving-up' % (method, numberOfRepetition), exit=False)
            raise

    def _ping(self, runner):

        for vmId in self.vmIds:
            _, ip = runner.getNetworkDetail(vmId)
            res = ping(ip)
            if not res:
                raise ExecutionException('Failed to ping %s' % ip)

    def _loginViaSsh(self, runner):

        loginCommand = 'ls /tmp'

        for vmId in self.vmIds:
            _, ip = runner.getNetworkDetail(vmId)
            res = sshCmd(loginCommand, ip, self.sshKey)
            if res:
                raise ExecutionException('Failed to SSH into machine for %s with return code %s' % (ip, res))

    def _stopVm(self, runner):
        runner.killInstances(self.vmIds)

    def applianceRepositoryTest(self):
        '''Authenticate, then upload a dummy image to the appliance repository, and remove after'''

        self._checkAttributePresent(['appRepoUsername', 'appRepoPassword'])
        self._testRepoConnection()
        self._uploadAndDeleteDummyImage()

    def _checkAttributePresent(self, attrs):
        for attr in attrs:
            if attr not in self.__dict__:
                raise Exception('Missing attribute %s. Missing an option argument?' % attr)

    def _testRepoConnection(self):
        passwordMgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        passwordMgr.add_password(None,
                                 self.appRepoUrl,
                                 self.appRepoUsername,
                                 self.appRepoPassword)

        handler = urllib2.HTTPBasicAuthHandler(passwordMgr)
        opener = urllib2.build_opener(handler)

        try:
            opener.open(self.appRepoUrl)
        except RuntimeError:
            raise NetworkException('Authentication to appliance repository failed')

    def _uploadAndDeleteDummyImage(self):
        dummyFile = '/tmp/stratus-dummy.img'
        self._generateDummyImage(dummyFile)

        manifest = ''
        options = self.configHolder.options.copy()
        options['repoUsername'] = self.appRepoUsername
        options['repoPassword'] = self.appRepoPassword
        options['repoAddress'] = self.appRepoUrl
        options['uploadOption'] = ''
        uploader = Uploader(manifest, options)
        uploader.uploadFile(dummyFile, os.path.join('base',os.path.basename(dummyFile)))
        uploader.deleteFile(uploader.uploadedFile[-1])

    def _openDevNull(self):
        return open('/dev/null', 'w')

    def _generateDummyImage(self, filename, size=2):
        devNull = open('/dev/null', 'w')
        execute(['dd', 'if=/dev/zero', 'of=%s' % filename, 'bs=1000000', 'count=%s' % size],
        stdout=devNull, stderr=devNull)
        devNull.close()

    def registrarTest(self):
        '''Register a new node with ONE server, check that it is properly registered and remove it'''
        configHolder = self.configHolder.copy()
        configHolder.options['infoDriver'] = 'kvm'
        configHolder.options['virtDriver'] = 'kvm'
        configHolder.options['transfertDriver'] = 'nfs'
        configHolder.options['username'] = self.oneUsername
        configHolder.options['password'] = self.onePassword
        registrar = Registrar(configHolder)
        hostname = 'registrar.ip.test'
        id = registrar.register([hostname])
        monitor = Monitor(configHolder)
        info = monitor.nodeDetail([id])[0]
        self.assertEqual(hostname, info.name)
        registrar.deregister(hostname)
        self.assertRaises(Exception, monitor.nodeDetail,[id])

    def listAvalableTests(self):
        print 'Available tests:'
        for testName, testDoc in self._extractTestDescriptions():
            print '    - %s: %s' % (testName, testDoc)

    def _extractTestDescriptions(self):
        methods = []
        for attrib in self.__class__.__dict__:
            if self._isTestMethod(attrib):
                methods.append((attrib,self.__class__.__dict__[attrib].__doc__))
        return methods

    def _extractTestMethodNames(self):
        methods = []
        for attrib in self.__class__.__dict__:
            if self._isTestMethod(attrib):
                methods.append(attrib)
        return methods

    def _isTestMethod(self, attrib):
        return inspect.ismethod(getattr(self, attrib)) and \
               (attrib.lower().startswith('test') or attrib.lower().endswith('test')) and \
               not attrib.startswith('_')

<commit_msg>Reactivated local IP test<commit_after>#
# Created as part of the StratusLab project (http://stratuslab.eu),
# co-funded by the European Commission under the Grant Agreement
# INFSO-RI-261552."
#
# Copyright (c) 2010, SixSq Sarl
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import datetime
import inspect
import os.path
import time
import unittest
import urllib2

from stratuslab.Monitor import Monitor
from stratuslab.Registrar import Registrar
from stratuslab.Runner import Runner
from stratuslab.Uploader import Uploader
from stratuslab.Exceptions import NetworkException
from stratuslab.Exceptions import ConfigurationException
from stratuslab.Exceptions import ExecutionException
from stratuslab.ConfigHolder import ConfigHolder
from stratuslab.Util import execute
from stratuslab.Util import generateSshKeyPair
from stratuslab.Util import ping
from stratuslab.Util import printError
from stratuslab.Util import sshCmd
import Util

class Testor(unittest.TestCase):

    configHolder = None
    testNames = []

    def __init__(self, methodName='dummy'):
        super(Testor, self).__init__(methodName)

        self.vmIds = []
        self.sshKey = '/tmp/id_rsa_smoke_test'
        self.sshKeyPub = self.sshKey + '.pub'
        self.testsToRun = []

        Testor.configHolder.assign(self)
        self._setEnvVars()

    def _setEnvVars(self):
        self._setSingleEnvVar('appRepoUsername', 'STRATUSLAB_REPO_USERNAME')
        self._setSingleEnvVar('appRepoPassword', 'STRATUSLAB_REPO_PASSWORD')
        self._setSingleEnvVar('appRepoUrl', 'STRATUSLAB_REPO_ADDRESS')
        self._setSingleEnvVar('username', 'STRATUSLAB_USERNAME')
        self._setSingleEnvVar('password', 'STRATUSLAB_PASSWORD')
        self._fillEndpointOption()

    def _setSingleEnvVar(self, field, env):
        if env in os.environ:
            setattr(self, field, os.environ[env])

    def _setOption(self, key, value):
        Testor.configHolder.options[key] = value

    def _fillEndpointOption(self):
        if Util.envEndpoint in os.environ:
            return
        if not self.frontendIp:
            raise ConfigurationException('Missing environment variable %s or configuration parameter frontend_ip' % Util.envEndpoint)
        os.environ[Util.envEndpoint] = 'http://%s:2633/RPC2' % self.frontendIp

    def dummy(self):
        pass

    def runTests(self):
        suite = unittest.TestSuite()
        tests = []
        if self.testNames:
            tests = self.testNames
        else:
            tests = self._extractTestMethodNames()

        for test in tests:
            suite.addTest(Testor(test))

        testResult = unittest.TextTestRunner(verbosity=2).run(suite)
        return testResult.wasSuccessful()

    def runMethod(self, method):
        return method()

    def runInstancePublicNetworkTest(self):
        '''Start new instance, ping it via public network and ssh into it, then stop it'''
        self._runInstanceTest()

    def runInstanceLocalNetworkTest(self):
        '''Start new instance, ping it via local network and ssh into it, then stop it'''
        self._runInstanceTest(True)

    def _runInstanceTest(self, withLocalNetwork=False):
        runner = self._startVm(withLocalNetwork)
        self._repeatCall(self._ping, runner)
        self._repeatCall(self._loginViaSsh, runner)
        self._stopVm(runner)

    def _prepareLog(self, logFile):
        log = open(logFile,'aw')
        log.write('\n'*3 + '=' * 60 + '\n')
        log.write(str(datetime.datetime.now()) + '\n')
        log.write('=' * 60 + '\n'*3)
        return log

    def _startVm(self, withLocalNetwork=False):
        generateSshKeyPair(self.sshKey)

        options = Runner.defaultRunOptions()
        options['username'] = self.oneUsername
        options['password'] = self.onePassword
        options['userKey'] = self.sshKeyPub
        options['verboseLevel'] = self.verboseLevel

        if withLocalNetwork:
            options['isLocalIp'] = True

        configHolder = ConfigHolder(options)
        image = 'http://appliances.stratuslab.org/images/base/ttylinux-9.5-i486-base/1.0/ttylinux-9.5-i486-base-1.0.img.gz'

        runner = Runner(image, configHolder)
        self.vmIds = runner.runInstance()

        for id in self.vmIds:
            vmStarted = runner.waitUntilVmRunningOrTimeout(id)
            if not vmStarted:
                printError('Failed to start VM id: %s' % id)

        return runner

    def _repeatCall(self, method, args=[]):
        numberOfRepetition = 60
        for _ in range(numberOfRepetition):
            failed = False
            try:
                if args:
                    method(args)
                else:
                    method()
            except ExecutionException:
                failed = True
                time.sleep(10)
            else:
                break

        if failed:
            printError('Failed executing method %s %s times, giving-up' % (method, numberOfRepetition), exit=False)
            raise

    def _ping(self, runner):

        for vmId in self.vmIds:
            _, ip = runner.getNetworkDetail(vmId)
            res = ping(ip)
            if not res:
                raise ExecutionException('Failed to ping %s' % ip)

    def _loginViaSsh(self, runner):

        loginCommand = 'ls /tmp'

        for vmId in self.vmIds:
            _, ip = runner.getNetworkDetail(vmId)
            res = sshCmd(loginCommand, ip, self.sshKey)
            if res:
                raise ExecutionException('Failed to SSH into machine for %s with return code %s' % (ip, res))

    def _stopVm(self, runner):
        runner.killInstances(self.vmIds)

    def applianceRepositoryTest(self):
        '''Authenticate, then upload a dummy image to the appliance repository, and remove after'''

        self._checkAttributePresent(['appRepoUsername', 'appRepoPassword'])
        self._testRepoConnection()
        self._uploadAndDeleteDummyImage()

    def _checkAttributePresent(self, attrs):
        for attr in attrs:
            if attr not in self.__dict__:
                raise Exception('Missing attribute %s. Missing an option argument?' % attr)

    def _testRepoConnection(self):
        passwordMgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        passwordMgr.add_password(None,
                                 self.appRepoUrl,
                                 self.appRepoUsername,
                                 self.appRepoPassword)

        handler = urllib2.HTTPBasicAuthHandler(passwordMgr)
        opener = urllib2.build_opener(handler)

        try:
            opener.open(self.appRepoUrl)
        except RuntimeError:
            raise NetworkException('Authentication to appliance repository failed')

    def _uploadAndDeleteDummyImage(self):
        dummyFile = '/tmp/stratus-dummy.img'
        self._generateDummyImage(dummyFile)

        manifest = ''
        options = self.configHolder.options.copy()
        options['repoUsername'] = self.appRepoUsername
        options['repoPassword'] = self.appRepoPassword
        options['repoAddress'] = self.appRepoUrl
        options['uploadOption'] = ''
        uploader = Uploader(manifest, options)
        uploader.uploadFile(dummyFile, os.path.join('base',os.path.basename(dummyFile)))
        uploader.deleteFile(uploader.uploadedFile[-1])

    def _openDevNull(self):
        return open('/dev/null', 'w')

    def _generateDummyImage(self, filename, size=2):
        devNull = open('/dev/null', 'w')
        execute(['dd', 'if=/dev/zero', 'of=%s' % filename, 'bs=1000000', 'count=%s' % size],
        stdout=devNull, stderr=devNull)
        devNull.close()

    def registrarTest(self):
        '''Register a new node with ONE server, check that it is properly registered and remove it'''
        configHolder = self.configHolder.copy()
        configHolder.options['infoDriver'] = 'kvm'
        configHolder.options['virtDriver'] = 'kvm'
        configHolder.options['transfertDriver'] = 'nfs'
        configHolder.options['username'] = self.oneUsername
        configHolder.options['password'] = self.onePassword
        registrar = Registrar(configHolder)
        hostname = 'registrar.ip.test'
        id = registrar.register([hostname])
        monitor = Monitor(configHolder)
        info = monitor.nodeDetail([id])[0]
        self.assertEqual(hostname, info.name)
        registrar.deregister(hostname)
        self.assertRaises(Exception, monitor.nodeDetail,[id])

    def listAvalableTests(self):
        print 'Available tests:'
        for testName, testDoc in self._extractTestDescriptions():
            print '    - %s: %s' % (testName, testDoc)

    def _extractTestDescriptions(self):
        methods = []
        for attrib in self.__class__.__dict__:
            if self._isTestMethod(attrib):
                methods.append((attrib,self.__class__.__dict__[attrib].__doc__))
        return methods

    def _extractTestMethodNames(self):
        methods = []
        for attrib in self.__class__.__dict__:
            if self._isTestMethod(attrib):
                methods.append(attrib)
        return methods

    def _isTestMethod(self, attrib):
        return inspect.ismethod(getattr(self, attrib)) and \
               (attrib.lower().startswith('test') or attrib.lower().endswith('test')) and \
               not attrib.startswith('_')

<|endoftext|>


<commit_before>from django.apps import AppConfig

from casexml.apps.case.signals import case_post_save
from corehq.form_processor.signals import sql_case_post_save

from corehq.apps.data_interfaces.signals import case_changed_receiver

class DataInterfacesAppConfig(AppConfig):
    name = 'corehq.apps.data_interfaces'

    def ready(self):
        from casexml.apps.case.models import CommCareCase
        from corehq.form_processor.models import CommCareCaseSQL

        case_post_save.connect(case_changed_receiver, CommCareCase,
                               dispatch_uid="data_interfaces_case_receiver")
        sql_case_post_save.connect(case_changed_receiver, CommCareCaseSQL,
                                   dispatch_uid="data_interfaces_sql_case_receiver")
<commit_msg>move indirect model imports out of app_config.py<commit_after>from django.apps import AppConfig

from corehq.apps.data_interfaces.signals import case_changed_receiver

class DataInterfacesAppConfig(AppConfig):
    name = 'corehq.apps.data_interfaces'

    def ready(self):
        from casexml.apps.case.models import CommCareCase
        from casexml.apps.case.signals import case_post_save
        from corehq.form_processor.models import CommCareCaseSQL
        from corehq.form_processor.signals import sql_case_post_save

        case_post_save.connect(case_changed_receiver, CommCareCase,
                               dispatch_uid="data_interfaces_case_receiver")
        sql_case_post_save.connect(case_changed_receiver, CommCareCaseSQL,
                                   dispatch_uid="data_interfaces_sql_case_receiver")
<|endoftext|>