<commit_before>import os
import tempfile
import pytest
import numpy as np
from ogusa.parameters import Specifications, revision_warnings_errors

# JSON_REVISION_FILE = """{
#     "revision": {
#         "frisch": 0.3
#     }
# }"""

@pytest.fixture(scope='module')
def revision_file():
    f = tempfile.NamedTemporaryFile(mode="a", delete=False)
    f.write(JSON_REVISION_FILE)
    f.close()
    # Must close and then yield for Windows platform
    yield f
    os.remove(f.name)

def test_create_specs_object():
    specs = Specifications()
    assert specs

def test_create_specs_object_test():
    specs = Specifications(test=True)
    assert specs

def test_compute_default_params():
    specs = Specifications()
    specs.alpha_G = np.ones((10, 1))
    specs.compute_default_params()
    assert specs.alpha_G[10] == 1

def test_get_tax_function_parameters():
    specs = Specifications()
    specs.get_tax_function_parameters(None, run_micro=False)
    assert specs.etr_params.shape == (specs.T, specs.S, 12)
    assert specs.mtrx_params.shape == (specs.T, specs.S, 12)
    assert specs.mtry_params.shape == (specs.T, specs.S, 12)

def test_get_tax_function_parameters_constant_rates():
    specs = Specifications()
    specs.constant_rates = True
    specs.get_tax_function_parameters(None, run_micro=False)
    assert specs.etr_params.shape == (specs.T, specs.S, 12)
    assert specs.mtrx_params.shape == (specs.T, specs.S, 12)
    assert specs.mtry_params.shape == (specs.T, specs.S, 12)
    assert np.allclose(specs.etr_params[:, :, :10],
                       np.zeros((specs.T, specs.S, 10)))
    assert np.allclose(specs.mtrx_params[:, :, :10],
                       np.zeros((specs.T, specs.S, 10)))
    assert np.allclose(specs.mtry_params[:, :, :10],
                       np.zeros((specs.T, specs.S, 10)))

def test_get_tax_function_zero_taxes():
    specs = Specifications()
    specs.zero_taxes = True
    specs.get_tax_function_parameters(None, run_micro=False)
    assert np.allclose(specs.etr_params,
                       np.zeros((specs.T, specs.S, 12)))
    assert np.allclose(specs.mtrx_params,
                       np.zeros((specs.T, specs.S, 12)))
    assert np.allclose(specs.mtry_params,
                       np.zeros((specs.T, specs.S, 12)))

# def test_read_json_revision(revision_file):
#     exp = {"frisch": 0.3}
#     act1 = Specifications.read_json_revision(JSON_REVISION_FILE)
#     assert exp == act1
#     act2 = Specifications.read_json_revision(JSON_REVISION_FILE)
#     assert exp == act2

def test_update_specifications_with_dict():
    spec = Specifications()
    new_spec_dict = {
        'frisch': 0.3,
    }
    spec.update_specifications(new_spec_dict)
    assert spec.frisch == 0.3
    assert len(spec.errors) == 0

def test_update_specification_with_json():
    spec = Specifications()
    new_spec_json = """
        {
            "frisch": 0.3
        }
    """
    spec.update_specifications(new_spec_json)
    assert spec.frisch == 0.3
    assert len(spec.errors) == 0

def test_implement_reform():
    specs = Specifications()
    new_specs = {
        'tG1': 30,
        'T': 80,
        'frisch': 0.3,
        'tax_func_type': 'DEP'
    }

    specs.update_specifications(new_specs)
    assert specs.frisch == 0.3
    assert specs.tG1 == 30
    assert specs.T == 80
    assert specs.tax_func_type == 'DEP'
    assert len(specs.errors) == 0
    # assert len(specs.warnings) == 0

def test_implement_bad_reform1():
    specs = Specifications()
    # tG1 has an upper bound at T / 2
    new_specs = {
        'tG1': 50,
        'T': 80,
    }

    specs.update_specifications(new_specs, raise_errors=False)

    assert len(specs.errors) == 0 # > 0
    # assert specs.errors['tG1'] == 'ERROR: tG1 value 50 > max value 40.0\n'  # to redo when can have param valid values depend on others'
    # assert len(specs.warnings) == 0

def test_implement_bad_reform2():
    specs = Specifications()
    # tG1 has an upper bound at T / 2
    new_specs = {
        'T': 80,
        'tax_func_type': 'not_a_functional_form'
    }

    specs.update_specifications(new_specs, raise_errors=False)

    assert len(specs.errors) > 0
    assert specs.errors['tax_func_type'][0] == 'tax_func_type "not_a_functional_form" must be in list of choices DEP, DEP_totalinc, GS, linear.'
    # assert len(specs.warnings) == 0

def test_revision_warnings_errors():
    user_mods = {'frisch': 0.41}

    ew = revision_warnings_errors(user_mods)
    assert len(ew['errors']) == 0
    assert len(ew['warnings']) == 0

    user_mods = {'frisch': 0.1}

    bad_ew = revision_warnings_errors(user_mods)
    assert len(bad_ew['errors']) > 0
    assert len(bad_ew['warnings']) == 0

## Commenting out because I don't think ParamTools allows this yet
# def test_simple_eval():
#     specs = Specifications()
#     specs.T = 100
#     assert specs.simple_eval('T / 2') == 50
#     assert specs.simple_eval('T * 2') == 200
#     assert specs.simple_eval('T - 2') == 98
#     assert specs.simple_eval('T + 2') == 102
<commit_msg>test read of tax func pickle<commit_after>import os
import tempfile
import pytest
import numpy as np
from ogusa.parameters import Specifications, revision_warnings_errors
from ogusa import utils

# get path to puf if puf.csv in ogusa/ directory
CUR_PATH = os.path.abspath(os.path.dirname(__file__))

JSON_REVISION_FILE = """{
    "revision": {
        "frisch": 0.3
    }
}"""

@pytest.fixture(scope='module')
def revision_file():
    f = tempfile.NamedTemporaryFile(mode="a", delete=False)
    f.write(JSON_REVISION_FILE)
    f.close()
    # Must close and then yield for Windows platform
    yield f
    os.remove(f.name)

def test_create_specs_object():
    specs = Specifications()
    assert specs

def test_create_specs_object_test():
    specs = Specifications(test=True)
    assert specs

def test_compute_default_params():
    specs = Specifications()
    specs.alpha_G = np.ones((10, 1))
    specs.compute_default_params()
    assert specs.alpha_G[10] == 1

def test_get_tax_function_parameters():
    specs = Specifications()
    specs.get_tax_function_parameters(None, run_micro=False)
    assert specs.etr_params.shape == (specs.T, specs.S, 12)
    assert specs.mtrx_params.shape == (specs.T, specs.S, 12)
    assert specs.mtry_params.shape == (specs.T, specs.S, 12)

def test_get_tax_function_parameters_constant_rates():
    specs = Specifications()
    specs.constant_rates = True
    specs.get_tax_function_parameters(None, run_micro=False)
    assert specs.etr_params.shape == (specs.T, specs.S, 12)
    assert specs.mtrx_params.shape == (specs.T, specs.S, 12)
    assert specs.mtry_params.shape == (specs.T, specs.S, 12)
    assert np.allclose(specs.etr_params[:, :, :10],
                       np.zeros((specs.T, specs.S, 10)))
    assert np.allclose(specs.mtrx_params[:, :, :10],
                       np.zeros((specs.T, specs.S, 10)))
    assert np.allclose(specs.mtry_params[:, :, :10],
                       np.zeros((specs.T, specs.S, 10)))

def test_get_tax_function_zero_taxes():
    specs = Specifications()
    specs.zero_taxes = True
    specs.get_tax_function_parameters(None, run_micro=False)
    assert np.allclose(specs.etr_params,
                       np.zeros((specs.T, specs.S, 12)))
    assert np.allclose(specs.mtrx_params,
                       np.zeros((specs.T, specs.S, 12)))
    assert np.allclose(specs.mtry_params,
                       np.zeros((specs.T, specs.S, 12)))

def test_read_tax_func_estimate():
    specs = Specifications()
    pickle_path = os.path.join(CUR_PATH, 'test_io_data')
    pickle_file = 'TxFuncEst_baseline.pkl'
    expected_dict = utils.safe_read_pickle(os.path.join(
        pickle_path, pickle_file))
    expected_dict2 = utils.safe_read_pickle(os.path.join(
        CUR_PATH, pickle_file))
    test_dict = specs.read_tax_func_estimate(os.path.join(
        pickle_path, pickle_file), pickle_file)
    assert np.allclose(expected_dict['tfunc_avg_etr'],
                       test_dict['tfunc_avg_etr'])
    test_dict2 = specs.read_tax_func_estimate('not_a_path', pickle_file)
    assert np.allclose(expected_dict2['tfunc_avg_etr'],
                       test_dict2['tfunc_avg_etr'])

# def test_read_json_revision(revision_file):
#     exp = {"frisch": 0.3}
#     act1 = Specifications.read_json_revision(JSON_REVISION_FILE)
#     assert exp == act1
#     act2 = Specifications.read_json_revision(JSON_REVISION_FILE)
#     assert exp == act2

def test_update_specifications_with_dict():
    spec = Specifications()
    new_spec_dict = {
        'frisch': 0.3,
    }
    spec.update_specifications(new_spec_dict)
    assert spec.frisch == 0.3
    assert len(spec.errors) == 0

def test_update_specification_with_json():
    spec = Specifications()
    new_spec_json = """
        {
            "frisch": 0.3
        }
    """
    spec.update_specifications(new_spec_json)
    assert spec.frisch == 0.3
    assert len(spec.errors) == 0

def test_implement_reform():
    specs = Specifications()
    new_specs = {
        'tG1': 30,
        'T': 80,
        'frisch': 0.3,
        'tax_func_type': 'DEP'
    }

    specs.update_specifications(new_specs)
    assert specs.frisch == 0.3
    assert specs.tG1 == 30
    assert specs.T == 80
    assert specs.tax_func_type == 'DEP'
    assert len(specs.errors) == 0
    # assert len(specs.warnings) == 0

def test_implement_bad_reform1():
    specs = Specifications()
    # tG1 has an upper bound at T / 2
    new_specs = {
        'tG1': 50,
        'T': 80,
    }

    specs.update_specifications(new_specs, raise_errors=False)

    assert len(specs.errors) == 0 # > 0
    # assert specs.errors['tG1'] == 'ERROR: tG1 value 50 > max value 40.0\n'  # to redo when can have param valid values depend on others'
    # assert len(specs.warnings) == 0

def test_implement_bad_reform2():
    specs = Specifications()
    # tG1 has an upper bound at T / 2
    new_specs = {
        'T': 80,
        'tax_func_type': 'not_a_functional_form'
    }

    specs.update_specifications(new_specs, raise_errors=False)

    assert len(specs.errors) > 0
    assert specs.errors['tax_func_type'][0] == 'tax_func_type "not_a_functional_form" must be in list of choices DEP, DEP_totalinc, GS, linear.'
    # assert len(specs.warnings) == 0

def test_revision_warnings_errors():
    user_mods = {'frisch': 0.41}

    ew = revision_warnings_errors(user_mods)
    assert len(ew['errors']) == 0
    assert len(ew['warnings']) == 0

    user_mods = {'frisch': 0.1}

    bad_ew = revision_warnings_errors(user_mods)
    assert len(bad_ew['errors']) > 0
    assert len(bad_ew['warnings']) == 0

## Commenting out because I don't think ParamTools allows this yet
# def test_simple_eval():
#     specs = Specifications()
#     specs.T = 100
#     assert specs.simple_eval('T / 2') == 50
#     assert specs.simple_eval('T * 2') == 200
#     assert specs.simple_eval('T - 2') == 98
#     assert specs.simple_eval('T + 2') == 102
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2014 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# IMPORTANT NOTE: If you make local mods to this file, you must run:
#   %  pnacl/build.sh driver
# in order for them to take effect in the scons build.  This command
# updates the copy in the toolchain/ tree.
#
# Uses llvm tool pnacl-bccompress to add abbreviations into the input file.
# It runs pnacl-bccompress multiple times, using a hill-climbing solution
# to try and find a good local minima for file size.

from driver_env import env
from driver_log import Log
import driver_tools
import pathtools
import os

EXTRA_ENV = {
  'INPUTS'             : '',
  'OUTPUT'             : '',
  'MAX_ATTEMPTS'       : '25',
  'RETRIES'            : '3',
  'SUFFIX'             :'-c',
}

PrepPatterns = [
    ( ('-o','(.*)'),     "env.set('OUTPUT', pathtools.normalize($0))"),
    ( '--max-attempts=(.*)', "env.set('MAX_ATTEMPTS', $0)"),
    ( '--retries=(.*)',  "env.set('RETRIES', $0)"),
    ( ('--suffix','(.*)'), "env.set('SUFFIX', $0)"),
    ( '(-.*)',           driver_tools.UnrecognizedOption),
    ( '(.*)',            "env.append('INPUTS', pathtools.normalize($0))"),
]

def Compress(f_input, f_output):
  """ Hill climb to smallest file.

      This code calls pnacl-compress multiple times to attempt to
      compress the file. That tool works by adding abbreviations that
      have a good likelyhood of shrinking the bitcode file. Unfortunately,
      any time abbreviations are added to PNaCl bitcode files, they can
      get larger because the addition of more abbreviations will require
      more bits to save abbreviation indices, resulting in the file
      actually increasing in size.

      To mitigate this, this driver hill climbs assuming that there
      may be local minima that are the best solution. Hence, anytime
      a local minima is reached, an additional number of attempts
      to see if we can find a smaller bitcode file, which implies
      you are moving closer to another local minima.
  """

  # Number of times we will continue to retry after finding local
  # minimum file size.
  #    max_retry_count: The maximum number of retries.
  #    retry_count: The number of retries left before we give up.
  max_retry_count = int(env.getone('RETRIES'))
  retry_count = max_retry_count
  if max_retry_count < 1:
    Log.Fatal("RETRIES must be >= 1")

  # The suffix to append to the input file, to generate intermediate files.
  #    test_suffix: The prefix of the suffix.
  #    test_index: The index of the current test file (appened to test_suffix).
  test_suffix = env.getone('SUFFIX')
  test_index = 1

  # The maximum number of times we will attempt to compress the file before
  # giving up.
  max_attempts = int(env.getone('MAX_ATTEMPTS'))
  if max_attempts < 1:
    Log.Fatal("MAX_ATTEMPTS must be >= 1")

  # The result of the last attempt to compress a file.
  #    last_file: The name of the file
  #    last_size: The number of bytes in last_file.
  #    last_saved: True if we did not remove last_file.
  last_file = f_input
  last_size = pathtools.getsize(f_input)
  last_saved = True

  # Keeps track of the current best compressed file.
  current_smallest_file = last_file
  current_smallest_size = last_size

  while max_attempts > 0 and retry_count > 0:

    next_file = f_input + test_suffix + str(test_index)
    print "Compressing %s: %s bytes" % (last_file, last_size)

    driver_tools.Run('"${PNACL_COMPRESS}" ' + last_file + ' -o ' + next_file)
    next_size = pathtools.getsize(next_file)
    if not last_saved:
      os.remove(last_file)

    if next_size < current_smallest_size:
      old_file = current_smallest_file
      current_smallest_file = next_file
      current_smallest_size = next_size
      if (f_input != old_file):
        os.remove(old_file)
      retry_count = max_retry_count
      next_saved = True
    else:
      next_saved = False
      retry_count -= 1
    last_file = next_file
    last_size = next_size
    last_saved = next_saved
    max_attempts -= 1
    test_index += 1

  # Install results.
  print "Compressed  %s: %s bytes" % (last_file, last_size)
  print "Best        %s: %s bytes" % (current_smallest_file,
                                      current_smallest_size)
  if not last_saved:
    os.remove(last_file)

  if (f_input == f_output):
    if (f_input == current_smallest_file): return
    f_temp = f_input + test_suffix + "0"
    os.rename(f_input, f_temp)
    os.rename(current_smallest_file, f_input)
    os.remove(f_temp)
  else:
    os.rename(current_smallest_file, f_output)

def main(argv):
  env.update(EXTRA_ENV)
  driver_tools.ParseArgs(argv, PrepPatterns)

  inputs = env.get('INPUTS')
  output = env.getone('OUTPUT')

  if len(inputs) != 1:
    Log.Fatal('Can only have one input')
  f_input = inputs[0]

  # Allow in-place file changes if output isn't specified.
  if output != '':
    f_output = output
  else:
    f_output = f_input

  Compress(f_input, f_output)
  return 0

def get_help(unused_argv):
  script = env.getone('SCRIPT_NAME')
  return """Usage: %s <options> in-file
  This tool compresses a pnacl bitcode (PEXE) file. It does so by
  generating a series of intermediate files. Each file represents
  an attempt to compress the previous file in the series. Uses
  hill-climbing to find the smallest file to use, and sets the
  output file to the best found case.

  The options are:
  -h --help                 Display this output
  -o <file>                 Place the output into <file>. Otherwise, the
                            input file is modified in-place.
  --max-attempts=N          Maximum number of attempts to reduce file size.
  --retries=N               Number of additional attempts to try after
                            a local minimum is found before quiting.
  --suffix XX               Create intermediate compressed files by adding
                            suffix XXN (where N is a number).
""" % script
<commit_msg>Don't delete input file if no compression possible in pnacl-compress<commit_after>#!/usr/bin/python
# Copyright (c) 2014 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# IMPORTANT NOTE: If you make local mods to this file, you must run:
#   %  pnacl/build.sh driver
# in order for them to take effect in the scons build.  This command
# updates the copy in the toolchain/ tree.
#
# Uses llvm tool pnacl-bccompress to add abbreviations into the input file.
# It runs pnacl-bccompress multiple times, using a hill-climbing solution
# to try and find a good local minima for file size.

from driver_env import env
from driver_log import Log
import driver_tools
import pathtools
import os
import shutil

EXTRA_ENV = {
  'INPUTS'             : '',
  'OUTPUT'             : '',
  'MAX_ATTEMPTS'       : '25',
  'RETRIES'            : '3',
  'SUFFIX'             :'-c',
}

PrepPatterns = [
    ( ('-o','(.*)'),     "env.set('OUTPUT', pathtools.normalize($0))"),
    ( '--max-attempts=(.*)', "env.set('MAX_ATTEMPTS', $0)"),
    ( '--retries=(.*)',  "env.set('RETRIES', $0)"),
    ( ('--suffix','(.*)'), "env.set('SUFFIX', $0)"),
    ( '(-.*)',           driver_tools.UnrecognizedOption),
    ( '(.*)',            "env.append('INPUTS', pathtools.normalize($0))"),
]

def Compress(f_input, f_output):
  """ Hill climb to smallest file.

      This code calls pnacl-compress multiple times to attempt to
      compress the file. That tool works by adding abbreviations that
      have a good likelyhood of shrinking the bitcode file. Unfortunately,
      any time abbreviations are added to PNaCl bitcode files, they can
      get larger because the addition of more abbreviations will require
      more bits to save abbreviation indices, resulting in the file
      actually increasing in size.

      To mitigate this, this driver hill climbs assuming that there
      may be local minima that are the best solution. Hence, anytime
      a local minima is reached, an additional number of attempts
      to see if we can find a smaller bitcode file, which implies
      you are moving closer to another local minima.
  """

  # Number of times we will continue to retry after finding local
  # minimum file size.
  #    max_retry_count: The maximum number of retries.
  #    retry_count: The number of retries left before we give up.
  max_retry_count = int(env.getone('RETRIES'))
  retry_count = max_retry_count
  if max_retry_count < 1:
    Log.Fatal("RETRIES must be >= 1")

  # The suffix to append to the input file, to generate intermediate files.
  #    test_suffix: The prefix of the suffix.
  #    test_index: The index of the current test file (appened to test_suffix).
  test_suffix = env.getone('SUFFIX')
  test_index = 1

  # The maximum number of times we will attempt to compress the file before
  # giving up.
  max_attempts = int(env.getone('MAX_ATTEMPTS'))
  if max_attempts < 1:
    Log.Fatal("MAX_ATTEMPTS must be >= 1")

  # The result of the last attempt to compress a file.
  #    last_file: The name of the file
  #    last_size: The number of bytes in last_file.
  #    last_saved: True if we did not remove last_file.
  last_file = f_input
  last_size = pathtools.getsize(f_input)
  last_saved = True

  # Keeps track of the current best compressed file.
  current_smallest_file = last_file
  current_smallest_size = last_size

  while max_attempts > 0 and retry_count > 0:

    next_file = f_input + test_suffix + str(test_index)
    print "Compressing %s: %s bytes" % (last_file, last_size)

    driver_tools.Run('"${PNACL_COMPRESS}" ' + last_file + ' -o ' + next_file)
    next_size = pathtools.getsize(next_file)
    if not last_saved:
      os.remove(last_file)

    if next_size < current_smallest_size:
      old_file = current_smallest_file
      current_smallest_file = next_file
      current_smallest_size = next_size
      if (f_input != old_file):
        os.remove(old_file)
      retry_count = max_retry_count
      next_saved = True
    else:
      next_saved = False
      retry_count -= 1
    last_file = next_file
    last_size = next_size
    last_saved = next_saved
    max_attempts -= 1
    test_index += 1

  # Install results.
  print "Compressed  %s: %s bytes" % (last_file, last_size)
  print "Best        %s: %s bytes" % (current_smallest_file,
                                      current_smallest_size)
  if not last_saved:
    os.remove(last_file)

  if (f_input == f_output):
    if (f_input == current_smallest_file): return
    f_temp = f_input + test_suffix + "0"
    os.rename(f_input, f_temp)
    os.rename(current_smallest_file, f_input)
    os.remove(f_temp)
  elif current_smallest_file == f_input:
    shutil.copyfile(current_smallest_file, f_output)
  else:
    os.rename(current_smallest_file, f_output)

def main(argv):
  env.update(EXTRA_ENV)
  driver_tools.ParseArgs(argv, PrepPatterns)

  inputs = env.get('INPUTS')
  output = env.getone('OUTPUT')

  if len(inputs) != 1:
    Log.Fatal('Can only have one input')
  f_input = inputs[0]

  # Allow in-place file changes if output isn't specified.
  if output != '':
    f_output = output
  else:
    f_output = f_input

  Compress(f_input, f_output)
  return 0

def get_help(unused_argv):
  script = env.getone('SCRIPT_NAME')
  return """Usage: %s <options> in-file
  This tool compresses a pnacl bitcode (PEXE) file. It does so by
  generating a series of intermediate files. Each file represents
  an attempt to compress the previous file in the series. Uses
  hill-climbing to find the smallest file to use, and sets the
  output file to the best found case.

  The options are:
  -h --help                 Display this output
  -o <file>                 Place the output into <file>. Otherwise, the
                            input file is modified in-place.
  --max-attempts=N          Maximum number of attempts to reduce file size.
  --retries=N               Number of additional attempts to try after
                            a local minimum is found before quiting.
  --suffix XX               Create intermediate compressed files by adding
                            suffix XXN (where N is a number).
""" % script
<|endoftext|>


<commit_before>from lxml import etree
from lxml.builder import ElementMaker

NS_PROTECTED = 'http://open511.org/namespaces/internal-field'
NS_GML = 'http://www.opengis.net/gml'
NS_GML_PREFIX = '{' + NS_GML + '}'
XML_LANG = '{http://www.w3.org/XML/1998/namespace}lang'

NSMAP = {
    'gml': NS_GML
}

G = ElementMaker(namespace=NS_GML)

#########
# XML -> JSON
#########

def _maybe_intify(t):
    return int(t) if hasattr(t, 'isdigit') and t.isdigit() else t

def pluralize(s):
    return s[:-1] + 'ies' if s.endswith('phy') else s + 's'

def xml_to_json(root):
    """Convert an Open511 XML document or document fragment to JSON.

    Takes an lxml Element object. Returns a dict ready to be JSON-serialized."""
    j = {}

    if len(root) == 0:  # Tag with no children, return str/int
        return _maybe_intify(root.text)

    if len(root) == 1 and root[0].tag.startswith('{' + NS_GML):  # GML
        return gml_to_geojson(root[0])

    if root.tag == 'open511':
        j['meta'] = {'version': root.get('version')}

    for elem in root:
        name = elem.tag
        if name == 'link' and elem.get('rel'):
            name = elem.get('rel') + '_url'
            if name == 'self_url':
                name = 'url'
            if root.tag == 'open511':
                j['meta'][name] = elem.get('href')
                continue
        elif name.startswith('{' + NS_PROTECTED):
            name = '!' + name[name.index('}') + 1:]
        elif name[0] == '{':
            # Namespace!
            name = '+' + name[name.index('}') + 1:]

        if name in j:
            continue  # duplicate
        elif elem.tag == 'link' and not elem.text:
            j[name] = elem.get('href')
        elif len(elem):
            if name in ('attachments', 'grouped_events'):
                j[name] = [xml_link_to_json(child, to_dict=(name == 'attachments')) for child in elem]
            elif all((name == pluralize(child.tag) for child in elem)):
                # <something><somethings> serializes to a JSON array
                j[name] = [xml_to_json(child) for child in elem]
            else:
                j[name] = xml_to_json(elem)
        else:
            if root.tag == 'open511' and name.endswith('s') and not elem.text:
                # Special case: an empty e.g. <events /> container at the root level
                # should be serialized to [], not null
                j[name] = []
            else:
                j[name] = _maybe_intify(elem.text)

    return j

def xml_link_to_json(link, to_dict=False):
    if to_dict:
        d = {'url': link.get('href')}
        for attr in ('type', 'title', 'length', 'hreflang'):
            if link.get(attr):
                d[attr] = link.get(attr)
        return d
    else:
        return link.get('href')

#########
# JSON -> XML
#########

def json_doc_to_xml(json_obj, lang='en', custom_namespace=None):
    """Converts a Open511 JSON document to XML.

    lang: the appropriate language code

    Takes a dict deserialized from JSON, returns an lxml Element.

    Accepts only the full root-level JSON object from an Open511 response."""
    if 'meta' not in json_obj:
        raise Exception("This function requires a conforming Open511 JSON document with a 'meta' section.")
    json_obj = dict(json_obj)
    meta = json_obj.pop('meta')
    elem = etree.Element("open511", nsmap={
        'gml': NS_GML,
    })
    if lang:
        elem.set(XML_LANG, lang)
    elem.set('version', meta.pop('version'))
    pagination = json_obj.pop('pagination', None)

    json_struct_to_xml(json_obj, elem, custom_namespace=custom_namespace)

    if pagination:
        elem.append(json_struct_to_xml(pagination, 'pagination', custom_namespace=custom_namespace))

    json_struct_to_xml(meta, elem)

    return elem

def json_struct_to_xml(json_obj, root, custom_namespace=None):
    """Converts a Open511 JSON fragment to XML.

    Takes a dict deserialized from JSON, returns an lxml Element.

    This won't provide a confirming document if you pass in a full JSON document;
    it's for translating little fragments, and is mostly used internally."""
    if isinstance(root, basestring):
        if root.startswith('!'):
            root = etree.Element('{%s}%s' % (NS_PROTECTED, root[1:]))
        elif root.startswith('+'):
            if not custom_namespace:
                raise Exception("JSON fields starts with +, but no custom namespace provided")
            root = etree.Element('{%s}%s' % (custom_namespace, root[1:]))
        else:
            root = etree.Element(root)
    if root.tag in ('attachments', 'grouped_events'):
        for link in json_obj:
            root.append(json_link_to_xml(link))
    elif isinstance(json_obj, basestring):
        root.text = json_obj
    elif isinstance (json_obj, (int, float, long)):
        root.text = unicode(json_obj)
    elif isinstance(json_obj, dict):
        if frozenset(json_obj.keys()) == frozenset(('type', 'coordinates')):
            root.append(geojson_to_gml(json_obj))
        else:
            for key, val in json_obj.items():
                if key == 'url':
                    el = json_link_to_xml(val, 'self')
                elif key.endswith('_url'):
                    el = json_link_to_xml(val, key.replace('_url', ''))
                else:
                    el = json_struct_to_xml(val, key, custom_namespace=custom_namespace)
                if el is not None:
                    root.append(el)
    elif isinstance(json_obj, list):
        tag_name = root.tag
        if tag_name.endswith('ies'):
            tag_name = tag_name[:-3] + 'y'
        elif tag_name.endswith('s'):
            tag_name = tag_name[:-1]
        for val in json_obj:
            el = json_struct_to_xml(val, tag_name, custom_namespace=custom_namespace)
            if el is not None:
                root.append(el)
    elif json_obj is None:
        return None
    else:
        raise NotImplementedError
    return root

def json_link_to_xml(val, rel='related'):
    tag = etree.Element('link')
    tag.set('rel', rel)
    if hasattr(val, 'get') and 'url' in val:
        tag.set('href', val['url'])
        for attr in ('type', 'title', 'length', 'hreflang'):
            if val.get(attr):
                tag.set(attr, unicode(val[attr]))
    else:
        tag.set('href', val)
    return tag

##########
# GEOSPATIAL
##########

def geojson_to_gml(gj, set_srs=True):
    """Given a dict deserialized from a GeoJSON object, returns an lxml Element
    of the corresponding GML geometry."""
    tag = G(gj['type'])
    if set_srs:
        tag.set('srsName', 'EPSG:4326')

    if gj['type'] == 'Point':
        tag.append(G.coordinates(','.join(str(c) for c in gj['coordinates'])))
    elif gj['type'] == 'LineString':
        tag.append(G.coordinates(' '.join(
            ','.join(str(c) for c in ll)
            for ll in gj['coordinates']
        )))
    elif gj['type'] == 'Polygon':
        rings = [
            G.LinearRing(
                G.coordinates(' '.join(
                    ','.join(str(c) for c in ll)
                    for ll in ring
                ))
            ) for ring in gj['coordinates']
        ]
        tag.append(G.outerBoundaryIs(rings.pop(0)))
        for ring in rings:
            tag.append(G.innerBoundaryIs(ring))
    elif gj['type'] in ('MultiPoint', 'MultiLineString', 'MultiPolygon'):
        single_type = gj['type'][5:]
        member_tag = single_type[0].lower() + single_type[1:] + 'Member'
        for coord in gj['coordinates']:
            tag.append(
                G(member_tag, geojson_to_gml({'type': single_type, 'coordinates': coord}, set_srs=False))
            )
    else:
        raise NotImplementedError

    return tag

def gml_to_geojson(el):
    """Given an lxml Element of a GML geometry, returns a dict in GeoJSON format."""
    tag = el.tag.replace('{%s}' % NS_GML, '')
    if tag == 'Point':
        coordinates = [float(c) for c in el.findtext('{%s}coordinates' % NS_GML).split(',')]
    elif tag == 'LineString':
        coordinates = [
            [float(x) for x in pair.split(',')]
            for pair in el.findtext('{%s}coordinates' % NS_GML).split(' ')
        ]
    elif tag == 'Polygon':
        coordinates = []
        for ring in el.xpath('gml:outerBoundaryIs/gml:LinearRing/gml:coordinates', namespaces=NSMAP) \
                + el.xpath('gml:innerBoundaryIs/gml:LinearRing/gml:coordinates', namespaces=NSMAP):
            coordinates.append([
                [float(x) for x in pair.split(',')]
                for pair in ring.text.split(' ')
            ])
    elif tag in ('MultiPoint', 'MultiLineString', 'MultiPolygon'):
        single_type = tag[5:]
        member_tag = single_type[0].lower() + single_type[1:] + 'Member'
        coordinates = [
            gml_to_geojson(member)['coordinates']
            for member in el.xpath('gml:%s/gml:%s' % (member_tag, single_type), namespaces=NSMAP)
        ]
    else:
        raise NotImplementedError

    return {
        'type': tag,
        'coordinates': coordinates
    }
<commit_msg>Logic for media_files tag in cameras resource<commit_after>from lxml import etree
from lxml.builder import ElementMaker

NS_PROTECTED = 'http://open511.org/namespaces/internal-field'
NS_GML = 'http://www.opengis.net/gml'
NS_GML_PREFIX = '{' + NS_GML + '}'
XML_LANG = '{http://www.w3.org/XML/1998/namespace}lang'

NSMAP = {
    'gml': NS_GML
}

G = ElementMaker(namespace=NS_GML)

#########
# XML -> JSON
#########

def _maybe_intify(t):
    return int(t) if hasattr(t, 'isdigit') and t.isdigit() else t

def pluralize(s):
    return s[:-1] + 'ies' if s.endswith('phy') else s + 's'

def xml_to_json(root):
    """Convert an Open511 XML document or document fragment to JSON.

    Takes an lxml Element object. Returns a dict ready to be JSON-serialized."""
    j = {}

    if len(root) == 0:  # Tag with no children, return str/int
        return _maybe_intify(root.text)

    if len(root) == 1 and root[0].tag.startswith('{' + NS_GML):  # GML
        return gml_to_geojson(root[0])

    if root.tag == 'open511':
        j['meta'] = {'version': root.get('version')}

    for elem in root:
        name = elem.tag
        if name == 'link' and elem.get('rel'):
            name = elem.get('rel') + '_url'
            if name == 'self_url':
                name = 'url'
            if root.tag == 'open511':
                j['meta'][name] = elem.get('href')
                continue
        elif name.startswith('{' + NS_PROTECTED):
            name = '!' + name[name.index('}') + 1:]
        elif name[0] == '{':
            # Namespace!
            name = '+' + name[name.index('}') + 1:]

        if name in j:
            continue  # duplicate
        elif elem.tag == 'link' and not elem.text:
            j[name] = elem.get('href')
        elif len(elem):
            if name == 'grouped_events':
                # An array of URLs
                j[name] = [xml_link_to_json(child, to_dict=False) for child in elem]
            elif name in ('attachments', 'media_files'):
                # An array of JSON objects
                j[name] = [xml_link_to_json(child, to_dict=True) for child in elem]
            elif all((name == pluralize(child.tag) for child in elem)):
                # <something><somethings> serializes to a JSON array
                j[name] = [xml_to_json(child) for child in elem]
            else:
                j[name] = xml_to_json(elem)
        else:
            if root.tag == 'open511' and name.endswith('s') and not elem.text:
                # Special case: an empty e.g. <events /> container at the root level
                # should be serialized to [], not null
                j[name] = []
            else:
                j[name] = _maybe_intify(elem.text)

    return j

def xml_link_to_json(link, to_dict=False):
    if to_dict:
        d = {'url': link.get('href')}
        for attr in ('type', 'title', 'length', 'hreflang'):
            if link.get(attr):
                d[attr] = link.get(attr)
        return d
    else:
        return link.get('href')

#########
# JSON -> XML
#########

def json_doc_to_xml(json_obj, lang='en', custom_namespace=None):
    """Converts a Open511 JSON document to XML.

    lang: the appropriate language code

    Takes a dict deserialized from JSON, returns an lxml Element.

    Accepts only the full root-level JSON object from an Open511 response."""
    if 'meta' not in json_obj:
        raise Exception("This function requires a conforming Open511 JSON document with a 'meta' section.")
    json_obj = dict(json_obj)
    meta = json_obj.pop('meta')
    elem = etree.Element("open511", nsmap={
        'gml': NS_GML,
    })
    if lang:
        elem.set(XML_LANG, lang)
    elem.set('version', meta.pop('version'))
    pagination = json_obj.pop('pagination', None)

    json_struct_to_xml(json_obj, elem, custom_namespace=custom_namespace)

    if pagination:
        elem.append(json_struct_to_xml(pagination, 'pagination', custom_namespace=custom_namespace))

    json_struct_to_xml(meta, elem)

    return elem

def json_struct_to_xml(json_obj, root, custom_namespace=None):
    """Converts a Open511 JSON fragment to XML.

    Takes a dict deserialized from JSON, returns an lxml Element.

    This won't provide a confirming document if you pass in a full JSON document;
    it's for translating little fragments, and is mostly used internally."""
    if isinstance(root, basestring):
        if root.startswith('!'):
            root = etree.Element('{%s}%s' % (NS_PROTECTED, root[1:]))
        elif root.startswith('+'):
            if not custom_namespace:
                raise Exception("JSON fields starts with +, but no custom namespace provided")
            root = etree.Element('{%s}%s' % (custom_namespace, root[1:]))
        else:
            root = etree.Element(root)
    if root.tag in ('attachments', 'grouped_events', 'media_files'):
        for link in json_obj:
            root.append(json_link_to_xml(link))
    elif isinstance(json_obj, basestring):
        root.text = json_obj
    elif isinstance (json_obj, (int, float, long)):
        root.text = unicode(json_obj)
    elif isinstance(json_obj, dict):
        if frozenset(json_obj.keys()) == frozenset(('type', 'coordinates')):
            root.append(geojson_to_gml(json_obj))
        else:
            for key, val in json_obj.items():
                if key == 'url':
                    el = json_link_to_xml(val, 'self')
                elif key.endswith('_url'):
                    el = json_link_to_xml(val, key.replace('_url', ''))
                else:
                    el = json_struct_to_xml(val, key, custom_namespace=custom_namespace)
                if el is not None:
                    root.append(el)
    elif isinstance(json_obj, list):
        tag_name = root.tag
        if tag_name.endswith('ies'):
            tag_name = tag_name[:-3] + 'y'
        elif tag_name.endswith('s'):
            tag_name = tag_name[:-1]
        for val in json_obj:
            el = json_struct_to_xml(val, tag_name, custom_namespace=custom_namespace)
            if el is not None:
                root.append(el)
    elif json_obj is None:
        return None
    else:
        raise NotImplementedError
    return root

def json_link_to_xml(val, rel='related'):
    tag = etree.Element('link')
    tag.set('rel', rel)
    if hasattr(val, 'get') and 'url' in val:
        tag.set('href', val['url'])
        for attr in ('type', 'title', 'length', 'hreflang'):
            if val.get(attr):
                tag.set(attr, unicode(val[attr]))
    else:
        tag.set('href', val)
    return tag

##########
# GEOSPATIAL
##########

def geojson_to_gml(gj, set_srs=True):
    """Given a dict deserialized from a GeoJSON object, returns an lxml Element
    of the corresponding GML geometry."""
    tag = G(gj['type'])
    if set_srs:
        tag.set('srsName', 'EPSG:4326')

    if gj['type'] == 'Point':
        tag.append(G.coordinates(','.join(str(c) for c in gj['coordinates'])))
    elif gj['type'] == 'LineString':
        tag.append(G.coordinates(' '.join(
            ','.join(str(c) for c in ll)
            for ll in gj['coordinates']
        )))
    elif gj['type'] == 'Polygon':
        rings = [
            G.LinearRing(
                G.coordinates(' '.join(
                    ','.join(str(c) for c in ll)
                    for ll in ring
                ))
            ) for ring in gj['coordinates']
        ]
        tag.append(G.outerBoundaryIs(rings.pop(0)))
        for ring in rings:
            tag.append(G.innerBoundaryIs(ring))
    elif gj['type'] in ('MultiPoint', 'MultiLineString', 'MultiPolygon'):
        single_type = gj['type'][5:]
        member_tag = single_type[0].lower() + single_type[1:] + 'Member'
        for coord in gj['coordinates']:
            tag.append(
                G(member_tag, geojson_to_gml({'type': single_type, 'coordinates': coord}, set_srs=False))
            )
    else:
        raise NotImplementedError

    return tag

def gml_to_geojson(el):
    """Given an lxml Element of a GML geometry, returns a dict in GeoJSON format."""
    tag = el.tag.replace('{%s}' % NS_GML, '')
    if tag == 'Point':
        coordinates = [float(c) for c in el.findtext('{%s}coordinates' % NS_GML).split(',')]
    elif tag == 'LineString':
        coordinates = [
            [float(x) for x in pair.split(',')]
            for pair in el.findtext('{%s}coordinates' % NS_GML).split(' ')
        ]
    elif tag == 'Polygon':
        coordinates = []
        for ring in el.xpath('gml:outerBoundaryIs/gml:LinearRing/gml:coordinates', namespaces=NSMAP) \
                + el.xpath('gml:innerBoundaryIs/gml:LinearRing/gml:coordinates', namespaces=NSMAP):
            coordinates.append([
                [float(x) for x in pair.split(',')]
                for pair in ring.text.split(' ')
            ])
    elif tag in ('MultiPoint', 'MultiLineString', 'MultiPolygon'):
        single_type = tag[5:]
        member_tag = single_type[0].lower() + single_type[1:] + 'Member'
        coordinates = [
            gml_to_geojson(member)['coordinates']
            for member in el.xpath('gml:%s/gml:%s' % (member_tag, single_type), namespaces=NSMAP)
        ]
    else:
        raise NotImplementedError

    return {
        'type': tag,
        'coordinates': coordinates
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2017 Enigma MPC, Inc.
# Copyright 2014 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from catalyst.api import (
    order_target_percent,
    record,
    symbol,
    get_open_orders,
    set_max_leverage,
    schedule_function,
    date_rules,
    attach_pipeline,
    pipeline_output,
)

from catalyst.pipeline import Pipeline
from catalyst.pipeline.data import CryptoPricing
from catalyst.pipeline.factors.crypto import VWAP

def initialize(context):
    context.ASSET_NAME = 'USDT_BTC'
    context.TARGET_INVESTMENT_RATIO = 0.8
    context.SHORT_WINDOW = 30
    context.LONG_WINDOW = 100

    # For all trading pairs in the poloniex bundle, the default denomination
    # currently supported by Catalyst is 1/1000th of a full coin. Use this
    # constant to scale the price of up to that of a full coin if desired.
    context.TICK_SIZE = 1000.0

    context.i = 0
    context.asset = symbol(context.ASSET_NAME)

    set_max_leverage(1.0)

    attach_pipeline(make_pipeline(context), 'vwap_pipeline')

    schedule_function(
        rebalance,
        time_rules=times_rules.every_minute(),
    )

def before_trading_start(context, data):
    context.pipeline_data = pipeline_output('vwap_pipeline')

def make_pipeline(context):
    return Pipeline(
        columns={
            'price': CryptoPricing.open.latest,
            'volume': CryptoPricing.volume.latest,
            'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
            'long_mavg': VWAP(window_length=context.LONG_WINDOW),
        }
    )

def rebalance(context, data):
    context.i += 1

    # skip first LONG_WINDOW bars to fill windows
    if context.i < context.LONG_WINDOW:
        return

    # get pipeline data for asset of interest
    pipeline_data = context.pipeline_data
    pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0]

    # retrieve long and short moving averages from pipeline
    short_mavg = pipeline_data.short_mavg
    long_mavg = pipeline_data.long_mavg
    price = pipeline_data.price
    volume = pipeline_data.volume

    # check that order has not already been placed
    open_orders = get_open_orders()
    if context.asset not in open_orders:
        # check that the asset of interest can currently be traded
        if data.can_trade(context.asset):
            # adjust portfolio based on comparison of long and short vwap
            if short_mavg > long_mavg:
                order_target_percent(
                    context.asset,
                    context.TARGET_INVESTMENT_RATIO,
                )
            elif short_mavg < long_mavg:
                order_target_percent(
                    context.asset,
                    0.0,
                )

    record(
        price=price,
        cash=context.portfolio.cash,
        leverage=context.account.leverage,
        short_mavg=short_mavg,
        long_mavg=long_mavg,
        volume=volume,
    )

def analyze(context=None, results=None):
    import matplotlib.pyplot as plt

    # Plot the portfolio and asset data.
    ax1 = plt.subplot(611)
    results[['portfolio_value']].plot(ax=ax1)
    ax1.set_ylabel('Portfolio value (USD)')

    ax2 = plt.subplot(612, sharex=ax1)
    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
    (context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)

    trans = results.ix[[t != [] for t in results.transactions]]
    amounts = [t[0]['amount'] for t in trans.transactions]

    buys = trans.ix[
        [t[0]['amount'] > 0 for t in trans.transactions]
    ]
    sells = trans.ix[
        [t[0]['amount'] < 0 for t in trans.transactions]
    ]

    ax2.plot(
        buys.index,
        context.TICK_SIZE * results.price[buys.index],
        '^',
        markersize=10,
        color='g',
    )
    ax2.plot(
        sells.index,
        context.TICK_SIZE * results.price[sells.index],
        'v',
        markersize=10,
        color='r',
    )

    ax3 = plt.subplot(613, sharex=ax1)
    results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
    ax3.set_ylabel('Leverage (USD)')

    ax4 = plt.subplot(614, sharex=ax1)
    results[['cash']].plot(ax=ax4)
    ax4.set_ylabel('Cash (USD)')

    results[[
        'treasury',
        'algorithm',
        'benchmark',
    ]] = results[[
        'treasury_period_return',
        'algorithm_period_return',
        'benchmark_period_return',
    ]]

    ax5 = plt.subplot(615, sharex=ax1)
    results[[
        'treasury',
        'algorithm',
        'benchmark',
    ]].plot(ax=ax5)
    ax5.set_ylabel('Percent Change')

    ax6 = plt.subplot(616, sharex=ax1)
    results[['volume']].plot(ax=ax6)
    ax6.set_ylabel('Volume (mBTC/day)')

    plt.legend(loc=3)

    # Show the plot.
    plt.gcf().set_size_inches(18, 8)
    plt.show()
<commit_msg>Update dual_vwap.py<commit_after>''' Catalyst currently does not support the Pipeline implementation
    from Zipline, see Issue #96:
    https://github.com/enigmampc/catalyst/issues/96

    Until the above issue is resolved, this example is non-functional.
    We are keeping this script here for when the issue is resolved
'''

#!/usr/bin/env python
#
# Copyright 2017 Enigma MPC, Inc.
# Copyright 2014 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from catalyst.api import (
    order_target_percent,
    record,
    symbol,
    get_open_orders,
    set_max_leverage,
    schedule_function,
    date_rules,
    attach_pipeline,
    pipeline_output,
)

from catalyst.pipeline import Pipeline
from catalyst.pipeline.data import CryptoPricing
from catalyst.pipeline.factors.crypto import VWAP

def initialize(context):
    context.ASSET_NAME = 'USDT_BTC'
    context.TARGET_INVESTMENT_RATIO = 0.8
    context.SHORT_WINDOW = 30
    context.LONG_WINDOW = 100

    # For all trading pairs in the poloniex bundle, the default denomination
    # currently supported by Catalyst is 1/1000th of a full coin. Use this
    # constant to scale the price of up to that of a full coin if desired.
    context.TICK_SIZE = 1000.0

    context.i = 0
    context.asset = symbol(context.ASSET_NAME)

    set_max_leverage(1.0)

    attach_pipeline(make_pipeline(context), 'vwap_pipeline')

    schedule_function(
        rebalance,
        time_rules=times_rules.every_minute(),
    )

def before_trading_start(context, data):
    context.pipeline_data = pipeline_output('vwap_pipeline')

def make_pipeline(context):
    return Pipeline(
        columns={
            'price': CryptoPricing.open.latest,
            'volume': CryptoPricing.volume.latest,
            'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
            'long_mavg': VWAP(window_length=context.LONG_WINDOW),
        }
    )

def rebalance(context, data):
    context.i += 1

    # skip first LONG_WINDOW bars to fill windows
    if context.i < context.LONG_WINDOW:
        return

    # get pipeline data for asset of interest
    pipeline_data = context.pipeline_data
    pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0]

    # retrieve long and short moving averages from pipeline
    short_mavg = pipeline_data.short_mavg
    long_mavg = pipeline_data.long_mavg
    price = pipeline_data.price
    volume = pipeline_data.volume

    # check that order has not already been placed
    open_orders = get_open_orders()
    if context.asset not in open_orders:
        # check that the asset of interest can currently be traded
        if data.can_trade(context.asset):
            # adjust portfolio based on comparison of long and short vwap
            if short_mavg > long_mavg:
                order_target_percent(
                    context.asset,
                    context.TARGET_INVESTMENT_RATIO,
                )
            elif short_mavg < long_mavg:
                order_target_percent(
                    context.asset,
                    0.0,
                )

    record(
        price=price,
        cash=context.portfolio.cash,
        leverage=context.account.leverage,
        short_mavg=short_mavg,
        long_mavg=long_mavg,
        volume=volume,
    )

def analyze(context=None, results=None):
    import matplotlib.pyplot as plt

    # Plot the portfolio and asset data.
    ax1 = plt.subplot(611)
    results[['portfolio_value']].plot(ax=ax1)
    ax1.set_ylabel('Portfolio value (USD)')

    ax2 = plt.subplot(612, sharex=ax1)
    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
    (context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)

    trans = results.ix[[t != [] for t in results.transactions]]
    amounts = [t[0]['amount'] for t in trans.transactions]

    buys = trans.ix[
        [t[0]['amount'] > 0 for t in trans.transactions]
    ]
    sells = trans.ix[
        [t[0]['amount'] < 0 for t in trans.transactions]
    ]

    ax2.plot(
        buys.index,
        context.TICK_SIZE * results.price[buys.index],
        '^',
        markersize=10,
        color='g',
    )
    ax2.plot(
        sells.index,
        context.TICK_SIZE * results.price[sells.index],
        'v',
        markersize=10,
        color='r',
    )

    ax3 = plt.subplot(613, sharex=ax1)
    results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
    ax3.set_ylabel('Leverage (USD)')

    ax4 = plt.subplot(614, sharex=ax1)
    results[['cash']].plot(ax=ax4)
    ax4.set_ylabel('Cash (USD)')

    results[[
        'treasury',
        'algorithm',
        'benchmark',
    ]] = results[[
        'treasury_period_return',
        'algorithm_period_return',
        'benchmark_period_return',
    ]]

    ax5 = plt.subplot(615, sharex=ax1)
    results[[
        'treasury',
        'algorithm',
        'benchmark',
    ]].plot(ax=ax5)
    ax5.set_ylabel('Percent Change')

    ax6 = plt.subplot(616, sharex=ax1)
    results[['volume']].plot(ax=ax6)
    ax6.set_ylabel('Volume (mBTC/day)')

    plt.legend(loc=3)

    # Show the plot.
    plt.gcf().set_size_inches(18, 8)
    plt.show()
<|endoftext|>


<commit_before>#########################################################################
# Copyright 2011 Cloud Sidekick
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#########################################################################

import catoclient.catocommand
from catoclient.param import Param

class RunTask(catoclient.catocommand.CatoCommand):

    Description = 'Runs a Cato Task.'
    Options = [Param(name='task', short_name='t', long_name='task',
                     optional=False, ptype='string',
                     doc='The ID or Name of the Task to run.'),
               Param(name='version', short_name='v', long_name='version',
                     optional=True, ptype='string',
                     doc='An optional specific Task Version. (Default if omitted.)'),
               Param(name='log_level', short_name='l', long_name='log_level',
                     optional=True, ptype='string',
                     doc='An optional Logging level.  (Normal if omitted.)'),
               Param(name='account', short_name='a', long_name='account',
                     optional=True, ptype='string',
                     doc='The ID or Name of Cloud Account credentials for the Task.'),
               Param(name='service_instance', short_name='i', long_name='service_instance',
                     optional=True, ptype='string',
                     doc='The ID of a Service Instance.'),
               Param(name='parameterfile', short_name='p', long_name='parameterfile',
                     optional=True, ptype='string',
                     doc='The file name of a Parameter XML file.')
               ]

    def main(self):
        try:
            # first, we need to load the parameters xml from a file
            self.parameters = None
            if self.parameterfile:
                import os
                fn = os.path.expanduser(self.parameterfile)
                with open(fn, 'r') as f_in:
                    if not f_in:
                        print("Unable to open file [%s]." % fn)
                    self.parameters = f_in.read()

            results = self.call_api('run_task', ['task', 'version', 'log_level', 'account', 'service_instance', 'parameters'])
            print(results)
        except ValueError:
            # the results could not be parsed as JSON, just return them
            print(results)
        except Exception as ex:
            raise ex
<commit_msg>Fixed arguments for cato-run-task.<commit_after>#########################################################################
# Copyright 2011 Cloud Sidekick
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#########################################################################

import catoclient.catocommand
from catoclient.param import Param

class RunTask(catoclient.catocommand.CatoCommand):

    Description = 'Runs a Cato Task.'
    Options = [Param(name='task', short_name='t', long_name='task',
                     optional=False, ptype='string',
                     doc='The ID or Name of the Task to run.'),
               Param(name='version', short_name='v', long_name='version',
                     optional=True, ptype='string',
                     doc='An optional specific Task Version. (Default if omitted.)'),
               Param(name='log_level', short_name='l', long_name='log_level',
                     optional=True, ptype='string',
                     doc='An optional Logging level.  (Normal if omitted.)'),
               Param(name='account', short_name='a', long_name='account',
                     optional=True, ptype='string',
                     doc='The ID or Name of Cloud Account credentials for the Task.'),
               Param(name='options', short_name='o', long_name='options',
                     optional=True, ptype='string',
                     doc='A JSON object containing additional options for the Task.'),
               Param(name='parameterfile', short_name='p', long_name='parameterfile',
                     optional=True, ptype='string',
                     doc='The file name of a Parameter XML file.')
               ]

    def main(self):
        try:
            # first, we need to load the parameters xml from a file
            self.parameters = None
            if self.parameterfile:
                import os
                fn = os.path.expanduser(self.parameterfile)
                with open(fn, 'r') as f_in:
                    if not f_in:
                        print("Unable to open file [%s]." % fn)
                    self.parameters = f_in.read()

            results = self.call_api('run_task', ['task', 'version', 'log_level', 'account', 'service_instance', 'parameters'])
            print(results)
        except ValueError:
            # the results could not be parsed as JSON, just return them
            print(results)
        except Exception as ex:
            raise ex
<|endoftext|>


<commit_before>"""

We will not use SAM format since it is uncompressed. However,
a few tools may be added. Once could ba to convert SAM to BAM,
which may already be present in other tools such as as pysam.

"""

import pandas as pd
import pylab
import pysam

# pysam uses htlib behing the scene and is very fast
# pysam works great for BAM file but with SAM, it needs to read the file after
# each compete iteration, which is not very useful

class SAM(pysam.AlignmentFile):
    """

    Header of a SAM file has N lines starting with '@' character.
    Using comment='@' and header None does not make the job.Using skiprows=N
    works but requires to know the number of lines in the header.

    .. todo:: read by chunk size for large files ?
    .. todo:: read two files for comparison ?

    FLAGS SAM format::

        1        0x1     template having multiple segments in sequencing
        2        0x2     each segment properly aligned according to the aligner
        4        0x4     segment unmapped
        8        0x8     next segment in the template unmapped
        16      0x10     SEQ being reverse complemented
        32      0x20     SEQ of the next segment in the template being reverse complemented
        64      0x40     the first segment in the template
        128      0x80    the last segment in the template
        256     0x100    secondary alignment
        512     0x200    not passing filters, such as platform/vendor quality controls
        1024     0x400   PCR or optical duplicate
        2048     0x800   supplementary alignme

    """
    def __init__(self, filename, *args):
        # works for py27 but not py35 probably a missing __init__  or __new__ in
        # AlignmentFile class. See e.g., stackoverflow/questions/
        # 26653401/typeerror-object-takes-no-parameters-but-only-in-python-3
        #super(SAM, self).__init__(filename, mode, *args)
        pysam.AlignmentFile.__init__(filename, *args)
        self.skiprows = self._guess_header_length()
        print('deprecated')

    def _guess_header_length(self):
        with open(self.filename, 'r') as fin:
            skiprows = 0
            while True:
                line = fin.readline()
                if line.startswith('@'):
                    skiprows += 1
                else:
                    break
        return skiprows

    def get_read_names(self):
        return self._get_column(0)

    def _get_column(self, col):
        data = []
        with open(self.filename, "r") as fin:
            for header in range(self.skiprows):
                fin.readline()

            for line in fin:
                data.append(line.split()[col])
        return data

    def plot_mapq_distribution(self):
        """Plot distribution of MAPQ scores (fifth column of SAM)

        The maximum MAPQ value that Bowtie 2 generates is 42 (though it doesn't
        say this anywhere in the documentation). In contrast, the maximum MAPQ
        value that BWA will generate is 37 (though once again, you -
        frustratingly - won't find this information in the manual).

        :reference: http://www.acgt.me/blog/2014/12/16/understanding-mapq-scores-in-sam-files-does-37-42
        """
        pylab.clf()
        data = [float(x) for x in self._get_column(3)]
        pylab.hist(data, bins=100, normed=True)
        pylab.grid(True)
        pylab.xlabel('MAPQ score')
        pylab.ylabel('Fraction of reads')

<commit_msg>remove unused and redundant module<commit_after><|endoftext|>


<commit_before>from serpy.fields import Field
import operator
import six

class SerializerBase(Field):
    _field_map = {}

def _compile_field_to_tuple(field, name, serializer_cls):
    getter = field.as_getter(name, serializer_cls)
    if getter is None:
        getter = serializer_cls.default_getter(field.attr or name)

    # Only set a to_value function if it has been overridden for performance.
    to_value = None
    if field._is_to_value_overridden():
        to_value = field.to_value

    return (name, getter, to_value, field.call, field.required, field.label,
            field.getter_takes_serializer)

class SerializerMeta(type):

    @staticmethod
    def _get_fields(direct_fields, serializer_cls):
        field_map = {}
        # Get all the fields from base classes.
        for cls in serializer_cls.__mro__[::-1]:
            if issubclass(cls, SerializerBase):
                field_map.update(cls._field_map)
        field_map.update(direct_fields)

        compiled_fields = [
            _compile_field_to_tuple(field, name, serializer_cls)
            for name, field in field_map.items()
        ]

        return field_map, compiled_fields

    def __new__(cls, name, bases, attrs):
        # Fields declared directly on the class.
        direct_fields = {}

        # Take all the Fields from the attributes.
        for attr_name, field in attrs.items():
            if isinstance(field, Field):
                direct_fields[attr_name] = field
        for k in direct_fields.keys():
            del attrs[k]

        real_cls = super(SerializerMeta, cls).__new__(cls, name, bases, attrs)

        field_map, compiled_fields = cls._get_fields(direct_fields, real_cls)

        real_cls._field_map = field_map
        real_cls._compiled_fields = tuple(compiled_fields)
        return real_cls

class Serializer(six.with_metaclass(SerializerMeta, SerializerBase)):
    """:class:`Serializer` is used as a base for custom serializers.

    The :class:`Serializer` class is also a subclass of :class:`Field`, and can
    be used as a :class:`Field` to create nested schemas. A serializer is
    defined by subclassing :class:`Serializer` and adding each :class:`Field`
    as a class variable:

    Example: ::

        class FooSerializer(Serializer):
            foo = Field()
            bar = Field()

        foo = Foo(foo='hello', bar=5)
        FooSerializer(foo).data
        # {'foo': 'hello', 'bar': 5}

    :param instance: The object or objects to serialize.
    :param bool many: If ``instance`` is a collection of objects, set ``many``
        to ``True`` to serialize to a list.
    """
    #: The default getter used if :meth:`Field.as_getter` returns None.
    default_getter = operator.attrgetter

    def __init__(self, instance=None, many=False, data=None, context=None,
                 **kwargs):
        if data is not None:
            raise RuntimeError(
                'serpy serializers do not support input validation')

        super(Serializer, self).__init__(**kwargs)
        self.instance = instance
        self.many = many
        self._data = None

    def _serialize(self, instance, fields):
        v = {}
        for name, getter, to_value, call, required, label, pass_self in fields:
            if pass_self:
                result = getter(self, instance)
            else:
                result = getter(instance)
                if required or result is not None:
                    if call:
                        result = result()
                    if to_value:
                        result = to_value(result)
            v[label or name] = result

        return v

    def to_value(self, instance):
        fields = self._compiled_fields
        if self.many:
            serialize = self._serialize
            return [serialize(o, fields) for o in instance]
        return self._serialize(instance, fields)

    @property
    def data(self):
        """Get the serialized data from the :class:`Serializer`.

        The data will be cached for future accesses.
        """
        # Cache the data for next time .data is called.
        if self._data is None:
            self._data = self.to_value(self.instance)
        return self._data

class DictSerializer(Serializer):
    """:class:`DictSerializer` serializes python ``dicts`` instead of objects.

    Instead of the serializer's fields fetching data using
    ``operator.attrgetter``, :class:`DictSerializer` uses
    ``operator.itemgetter``.

    Example: ::

        class FooSerializer(DictSerializer):
            foo = IntField()
            bar = FloatField()

        foo = {'foo': '5', 'bar': '2.2'}
        FooSerializer(foo).data
        # {'foo': 5, 'bar': 2.2}
    """
    default_getter = operator.itemgetter
<commit_msg>Adjust name assignment in serialization<commit_after>from serpy.fields import Field
import operator
import six

class SerializerBase(Field):
    _field_map = {}

def _compile_field_to_tuple(field, name, serializer_cls):
    getter = field.as_getter(name, serializer_cls)
    if getter is None:
        getter = serializer_cls.default_getter(field.attr or name)

    # Only set a to_value function if it has been overridden for performance.
    to_value = None
    if field._is_to_value_overridden():
        to_value = field.to_value

    # Set the field name to a supplied label; if none, set it to the attribute name.
    name = field.label or name

    return (name, getter, to_value, field.call, field.required,
            field.getter_takes_serializer)

class SerializerMeta(type):

    @staticmethod
    def _get_fields(direct_fields, serializer_cls):
        field_map = {}
        # Get all the fields from base classes.
        for cls in serializer_cls.__mro__[::-1]:
            if issubclass(cls, SerializerBase):
                field_map.update(cls._field_map)
        field_map.update(direct_fields)

        compiled_fields = [
            _compile_field_to_tuple(field, name, serializer_cls)
            for name, field in field_map.items()
        ]

        return field_map, compiled_fields

    def __new__(cls, name, bases, attrs):
        # Fields declared directly on the class.
        direct_fields = {}

        # Take all the Fields from the attributes.
        for attr_name, field in attrs.items():
            if isinstance(field, Field):
                direct_fields[attr_name] = field
        for k in direct_fields.keys():
            del attrs[k]

        real_cls = super(SerializerMeta, cls).__new__(cls, name, bases, attrs)

        field_map, compiled_fields = cls._get_fields(direct_fields, real_cls)

        real_cls._field_map = field_map
        real_cls._compiled_fields = tuple(compiled_fields)
        return real_cls

class Serializer(six.with_metaclass(SerializerMeta, SerializerBase)):
    """:class:`Serializer` is used as a base for custom serializers.

    The :class:`Serializer` class is also a subclass of :class:`Field`, and can
    be used as a :class:`Field` to create nested schemas. A serializer is
    defined by subclassing :class:`Serializer` and adding each :class:`Field`
    as a class variable:

    Example: ::

        class FooSerializer(Serializer):
            foo = Field()
            bar = Field()

        foo = Foo(foo='hello', bar=5)
        FooSerializer(foo).data
        # {'foo': 'hello', 'bar': 5}

    :param instance: The object or objects to serialize.
    :param bool many: If ``instance`` is a collection of objects, set ``many``
        to ``True`` to serialize to a list.
    """
    #: The default getter used if :meth:`Field.as_getter` returns None.
    default_getter = operator.attrgetter

    def __init__(self, instance=None, many=False, data=None, context=None,
                 **kwargs):
        if data is not None:
            raise RuntimeError(
                'serpy serializers do not support input validation')

        super(Serializer, self).__init__(**kwargs)
        self.instance = instance
        self.many = many
        self._data = None

    def _serialize(self, instance, fields):
        v = {}
        for name, getter, to_value, call, required, pass_self in fields:
            if pass_self:
                result = getter(self, instance)
            else:
                result = getter(instance)
                if required or result is not None:
                    if call:
                        result = result()
                    if to_value:
                        result = to_value(result)
            v[name] = result

        return v

    def to_value(self, instance):
        fields = self._compiled_fields
        if self.many:
            serialize = self._serialize
            return [serialize(o, fields) for o in instance]
        return self._serialize(instance, fields)

    @property
    def data(self):
        """Get the serialized data from the :class:`Serializer`.

        The data will be cached for future accesses.
        """
        # Cache the data for next time .data is called.
        if self._data is None:
            self._data = self.to_value(self.instance)
        return self._data

class DictSerializer(Serializer):
    """:class:`DictSerializer` serializes python ``dicts`` instead of objects.

    Instead of the serializer's fields fetching data using
    ``operator.attrgetter``, :class:`DictSerializer` uses
    ``operator.itemgetter``.

    Example: ::

        class FooSerializer(DictSerializer):
            foo = IntField()
            bar = FloatField()

        foo = {'foo': '5', 'bar': '2.2'}
        FooSerializer(foo).data
        # {'foo': 5, 'bar': 2.2}
    """
    default_getter = operator.itemgetter
<|endoftext|>


<commit_before>"""Tests for certbot.plugins.common."""
import os
import shutil
import tempfile
import unittest

import mock
import OpenSSL

from acme import challenges
from acme import jose

from certbot import achallenges
from certbot import crypto_util

from certbot.tests import acme_util
from certbot.tests import util as test_util

class NamespaceFunctionsTest(unittest.TestCase):
    """Tests for certbot.plugins.common.*_namespace functions."""

    def test_option_namespace(self):
        from certbot.plugins.common import option_namespace
        self.assertEqual("foo-", option_namespace("foo"))

    def test_dest_namespace(self):
        from certbot.plugins.common import dest_namespace
        self.assertEqual("foo_", dest_namespace("foo"))

    def test_dest_namespace_with_dashes(self):
        from certbot.plugins.common import dest_namespace
        self.assertEqual("foo_bar_", dest_namespace("foo-bar"))

class PluginTest(unittest.TestCase):
    """Test for certbot.plugins.common.Plugin."""

    def setUp(self):
        from certbot.plugins.common import Plugin

        class MockPlugin(Plugin):  # pylint: disable=missing-docstring
            @classmethod
            def add_parser_arguments(cls, add):
                add("foo-bar", dest="different_to_foo_bar", x=1, y=None)

        self.plugin_cls = MockPlugin
        self.config = mock.MagicMock()
        self.plugin = MockPlugin(config=self.config, name="mock")

    def test_init(self):
        self.assertEqual("mock", self.plugin.name)
        self.assertEqual(self.config, self.plugin.config)

    def test_option_namespace(self):
        self.assertEqual("mock-", self.plugin.option_namespace)

    def test_option_name(self):
        self.assertEqual("mock-foo_bar", self.plugin.option_name("foo_bar"))

    def test_dest_namespace(self):
        self.assertEqual("mock_", self.plugin.dest_namespace)

    def test_dest(self):
        self.assertEqual("mock_foo_bar", self.plugin.dest("foo-bar"))
        self.assertEqual("mock_foo_bar", self.plugin.dest("foo_bar"))

    def test_conf(self):
        self.assertEqual(self.config.mock_foo_bar, self.plugin.conf("foo-bar"))

    def test_inject_parser_options(self):
        parser = mock.MagicMock()
        self.plugin_cls.inject_parser_options(parser, "mock")
        # note that inject_parser_options doesn't check if dest has
        # correct prefix
        parser.add_argument.assert_called_once_with(
            "--mock-foo-bar", dest="different_to_foo_bar", x=1, y=None)

class AddrTest(unittest.TestCase):
    """Tests for certbot.client.plugins.common.Addr."""

    def setUp(self):
        from certbot.plugins.common import Addr
        self.addr1 = Addr.fromstring("192.168.1.1")
        self.addr2 = Addr.fromstring("192.168.1.1:*")
        self.addr3 = Addr.fromstring("192.168.1.1:80")
        self.addr4 = Addr.fromstring("[fe00::1]")
        self.addr5 = Addr.fromstring("[fe00::1]:*")
        self.addr6 = Addr.fromstring("[fe00::1]:80")
        self.addr7 = Addr.fromstring("[fe00::1]:5")
        self.addr8 = Addr.fromstring("[fe00:1:2:3:4:5:6:7:8:9]:8080")

    def test_fromstring(self):
        self.assertEqual(self.addr1.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr1.get_port(), "")
        self.assertEqual(self.addr2.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr2.get_port(), "*")
        self.assertEqual(self.addr3.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr3.get_port(), "80")
        self.assertEqual(self.addr4.get_addr(), "[fe00::1]")
        self.assertEqual(self.addr4.get_port(), "")
        self.assertEqual(self.addr5.get_addr(), "[fe00::1]")
        self.assertEqual(self.addr5.get_port(), "*")
        self.assertEqual(self.addr6.get_addr(), "[fe00::1]")
        self.assertEqual(self.addr6.get_port(), "80")
        self.assertEqual(self.addr6.get_ipv6_exploded(),
                         "fe00:0:0:0:0:0:0:1")
        self.assertEqual(self.addr1.get_ipv6_exploded(),
                         "")
        self.assertEqual(self.addr7.get_port(), "5")
        self.assertEqual(self.addr8.get_ipv6_exploded(),
                         "fe00:1:2:3:4:5:6:7")

    def test_str(self):
        self.assertEqual(str(self.addr1), "192.168.1.1")
        self.assertEqual(str(self.addr2), "192.168.1.1:*")
        self.assertEqual(str(self.addr3), "192.168.1.1:80")
        self.assertEqual(str(self.addr4), "[fe00::1]")
        self.assertEqual(str(self.addr5), "[fe00::1]:*")
        self.assertEqual(str(self.addr6), "[fe00::1]:80")

    def test_get_addr_obj(self):
        self.assertEqual(str(self.addr1.get_addr_obj("443")), "192.168.1.1:443")
        self.assertEqual(str(self.addr2.get_addr_obj("")), "192.168.1.1")
        self.assertEqual(str(self.addr1.get_addr_obj("*")), "192.168.1.1:*")
        self.assertEqual(str(self.addr4.get_addr_obj("443")), "[fe00::1]:443")
        self.assertEqual(str(self.addr5.get_addr_obj("")), "[fe00::1]")
        self.assertEqual(str(self.addr4.get_addr_obj("*")), "[fe00::1]:*")

    def test_eq(self):
        self.assertEqual(self.addr1, self.addr2.get_addr_obj(""))
        self.assertNotEqual(self.addr1, self.addr2)
        self.assertFalse(self.addr1 == 3333)

        self.assertEqual(self.addr4, self.addr4.get_addr_obj(""))
        self.assertNotEqual(self.addr4, self.addr5)
        self.assertFalse(self.addr4 == 3333)
        from certbot.plugins.common import Addr
        self.assertEqual(self.addr4, Addr.fromstring("[fe00:0:0::1]"))
        self.assertEqual(self.addr4, Addr.fromstring("[fe00:0::0:0:1]"))

    def test_set_inclusion(self):
        from certbot.plugins.common import Addr
        set_a = set([self.addr1, self.addr2])
        addr1b = Addr.fromstring("192.168.1.1")
        addr2b = Addr.fromstring("192.168.1.1:*")
        set_b = set([addr1b, addr2b])

        self.assertEqual(set_a, set_b)

        set_c = set([self.addr4, self.addr5])
        addr4b = Addr.fromstring("[fe00::1]")
        addr5b = Addr.fromstring("[fe00::1]:*")
        set_d = set([addr4b, addr5b])

        self.assertEqual(set_c, set_d)

class TLSSNI01Test(unittest.TestCase):
    """Tests for certbot.plugins.common.TLSSNI01."""

    auth_key = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
    achalls = [
        achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(token=b'token1'), "pending"),
            domain="encryption-example.demo", account_key=auth_key),
        achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(token=b'token2'), "pending"),
            domain="certbot.demo", account_key=auth_key),
    ]

    def setUp(self):
        self.tempdir = tempfile.mkdtemp()
        configurator = mock.MagicMock()
        configurator.config.config_dir = os.path.join(self.tempdir, "config")
        configurator.config.work_dir = os.path.join(self.tempdir, "work")

        from certbot.plugins.common import TLSSNI01
        self.sni = TLSSNI01(configurator=configurator)

    def tearDown(self):
        shutil.rmtree(self.tempdir)

    def test_add_chall(self):
        self.sni.add_chall(self.achalls[0], 0)
        self.assertEqual(1, len(self.sni.achalls))
        self.assertEqual([0], self.sni.indices)

    def test_setup_challenge_cert(self):
        # This is a helper function that can be used for handling
        # open context managers more elegantly. It avoids dealing with
        # __enter__ and __exit__ calls.
        # http://www.voidspace.org.uk/python/mock/helpers.html#mock.mock_open
        mock_open, mock_safe_open = mock.mock_open(), mock.mock_open()

        response = challenges.TLSSNI01Response()
        achall = mock.MagicMock()
        achall.chall.encode.return_value = "token"
        key = test_util.load_pyopenssl_private_key("rsa512_key.pem")
        achall.response_and_validation.return_value = (
            response, (test_util.load_cert("cert.pem"), key))

        with mock.patch("certbot.plugins.common.open",
                        mock_open, create=True):
            with mock.patch("certbot.plugins.common.util.safe_open",
                            mock_safe_open):
                # pylint: disable=protected-access
                self.assertEqual(response, self.sni._setup_challenge_cert(
                    achall, "randomS1"))

        # pylint: disable=no-member
        mock_open.assert_called_once_with(self.sni.get_cert_path(achall), "wb")
        mock_open.return_value.write.assert_called_once_with(
            test_util.load_vector("cert.pem"))
        mock_safe_open.assert_called_once_with(
            self.sni.get_key_path(achall), "wb", chmod=0o400)
        mock_safe_open.return_value.write.assert_called_once_with(
            OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key))

class InstallSslOptionsConfTest(test_util.TempDirTestCase):
    """Tests for certbot.plugins.common.install_ssl_options_conf."""

    def setUp(self):
        super(InstallSslOptionsConfTest, self).setUp()
        self.source_path = os.path.join(self.tempdir, "options-ssl-source.conf")
        self.dest_path = os.path.join(self.tempdir, "options-ssl-dest.conf")
        self.hash_path = os.path.join(self.tempdir, ".options-ssl-conf.txt")
        with open(self.source_path, "w") as f:
            f.write("test directive\n")
        self.hashes = ["someotherhash",
                       "anotheroldhash",
                       crypto_util.sha256sum(self.source_path)]

    def _call(self):
        from certbot.plugins.common import install_ssl_options_conf
        install_ssl_options_conf(self.dest_path,
                                 self.hash_path,
                                 self.source_path,
                                 self.hashes)

    def _current_ssl_options_hash(self):
        return crypto_util.sha256sum(self.source_path)

    def _assert_current_file(self):
        self.assertTrue(os.path.isfile(self.dest_path))
        self.assertEqual(crypto_util.sha256sum(self.dest_path),
            self._current_ssl_options_hash())

    def test_no_file(self):
        self.assertFalse(os.path.isfile(self.dest_path))
        self._call()
        self._assert_current_file()

    def test_current_file(self):
        self._call()
        self._assert_current_file()

    def test_prev_file_updates_to_current(self):
        with mock.patch('certbot.crypto_util.sha256sum') as mock_sha256:
            mock_sha256.return_value = self.hashes[0]
            self._call()
        self._assert_current_file()

    def test_manually_modified_current_file_does_not_update(self):
        self._call()
        with open(self.dest_path, "a") as mod_ssl_conf:
            mod_ssl_conf.write("a new line for the wrong hash\n")
        with mock.patch("certbot.plugins.common.logger") as mock_logger:
            self._call()
            self.assertFalse(mock_logger.warning.called)
        self.assertTrue(os.path.isfile(self.dest_path))
        self.assertEqual(crypto_util.sha256sum(self.source_path),
            self._current_ssl_options_hash())
        self.assertNotEqual(crypto_util.sha256sum(self.dest_path),
            self._current_ssl_options_hash())

    def test_manually_modified_past_file_warns(self):
        with open(self.dest_path, "a") as mod_ssl_conf:
            mod_ssl_conf.write("a new line for the wrong hash\n")
        with open(self.hash_path, "w") as f:
            f.write("hashofanoldversion")
        with mock.patch("certbot.plugins.common.logger") as mock_logger:
            self._call()
            self.assertEqual(mock_logger.warning.call_args[0][0],
                "%s has been manually modified; updated ssl configuration options "
                "saved to %s. We recommend updating %s for security purposes.")
        self.assertEqual(crypto_util.sha256sum(self.source_path),
            self._current_ssl_options_hash())
        # only print warning once
        with mock.patch("certbot.plugins.common.logger") as mock_logger:
            self._call()
            self.assertFalse(mock_logger.warning.called)

if __name__ == "__main__":
    unittest.main()  # pragma: no cover
<commit_msg>Get full test coverage on certbot.plugins.common. (#4765)<commit_after>"""Tests for certbot.plugins.common."""
import os
import shutil
import tempfile
import unittest

import mock
import OpenSSL

from acme import challenges
from acme import jose

from certbot import achallenges
from certbot import crypto_util

from certbot.tests import acme_util
from certbot.tests import util as test_util

class NamespaceFunctionsTest(unittest.TestCase):
    """Tests for certbot.plugins.common.*_namespace functions."""

    def test_option_namespace(self):
        from certbot.plugins.common import option_namespace
        self.assertEqual("foo-", option_namespace("foo"))

    def test_dest_namespace(self):
        from certbot.plugins.common import dest_namespace
        self.assertEqual("foo_", dest_namespace("foo"))

    def test_dest_namespace_with_dashes(self):
        from certbot.plugins.common import dest_namespace
        self.assertEqual("foo_bar_", dest_namespace("foo-bar"))

class PluginTest(unittest.TestCase):
    """Test for certbot.plugins.common.Plugin."""

    def setUp(self):
        from certbot.plugins.common import Plugin

        class MockPlugin(Plugin):  # pylint: disable=missing-docstring
            @classmethod
            def add_parser_arguments(cls, add):
                add("foo-bar", dest="different_to_foo_bar", x=1, y=None)

        self.plugin_cls = MockPlugin
        self.config = mock.MagicMock()
        self.plugin = MockPlugin(config=self.config, name="mock")

    def test_init(self):
        self.assertEqual("mock", self.plugin.name)
        self.assertEqual(self.config, self.plugin.config)

    def test_option_namespace(self):
        self.assertEqual("mock-", self.plugin.option_namespace)

    def test_option_name(self):
        self.assertEqual("mock-foo_bar", self.plugin.option_name("foo_bar"))

    def test_dest_namespace(self):
        self.assertEqual("mock_", self.plugin.dest_namespace)

    def test_dest(self):
        self.assertEqual("mock_foo_bar", self.plugin.dest("foo-bar"))
        self.assertEqual("mock_foo_bar", self.plugin.dest("foo_bar"))

    def test_conf(self):
        self.assertEqual(self.config.mock_foo_bar, self.plugin.conf("foo-bar"))

    def test_inject_parser_options(self):
        parser = mock.MagicMock()
        self.plugin_cls.inject_parser_options(parser, "mock")
        # note that inject_parser_options doesn't check if dest has
        # correct prefix
        parser.add_argument.assert_called_once_with(
            "--mock-foo-bar", dest="different_to_foo_bar", x=1, y=None)

class AddrTest(unittest.TestCase):
    """Tests for certbot.client.plugins.common.Addr."""

    def setUp(self):
        from certbot.plugins.common import Addr
        self.addr1 = Addr.fromstring("192.168.1.1")
        self.addr2 = Addr.fromstring("192.168.1.1:*")
        self.addr3 = Addr.fromstring("192.168.1.1:80")
        self.addr4 = Addr.fromstring("[fe00::1]")
        self.addr5 = Addr.fromstring("[fe00::1]:*")
        self.addr6 = Addr.fromstring("[fe00::1]:80")
        self.addr7 = Addr.fromstring("[fe00::1]:5")
        self.addr8 = Addr.fromstring("[fe00:1:2:3:4:5:6:7:8:9]:8080")

    def test_fromstring(self):
        self.assertEqual(self.addr1.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr1.get_port(), "")
        self.assertEqual(self.addr2.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr2.get_port(), "*")
        self.assertEqual(self.addr3.get_addr(), "192.168.1.1")
        self.assertEqual(self.addr3.get_port(), "80")
        self.assertEqual(self.addr4.get_addr(), "[fe00::1]")
        self.assertEqual(self.addr4.get_port(), "")
        self.assertEqual(self.addr5.get_addr(), "[fe00::1]")
        self.assertEqual(self.addr5.get_port(), "*")
        self.assertEqual(self.addr6.get_addr(), "[fe00::1]")
        self.assertEqual(self.addr6.get_port(), "80")
        self.assertEqual(self.addr6.get_ipv6_exploded(),
                         "fe00:0:0:0:0:0:0:1")
        self.assertEqual(self.addr1.get_ipv6_exploded(),
                         "")
        self.assertEqual(self.addr7.get_port(), "5")
        self.assertEqual(self.addr8.get_ipv6_exploded(),
                         "fe00:1:2:3:4:5:6:7")

    def test_str(self):
        self.assertEqual(str(self.addr1), "192.168.1.1")
        self.assertEqual(str(self.addr2), "192.168.1.1:*")
        self.assertEqual(str(self.addr3), "192.168.1.1:80")
        self.assertEqual(str(self.addr4), "[fe00::1]")
        self.assertEqual(str(self.addr5), "[fe00::1]:*")
        self.assertEqual(str(self.addr6), "[fe00::1]:80")

    def test_get_addr_obj(self):
        self.assertEqual(str(self.addr1.get_addr_obj("443")), "192.168.1.1:443")
        self.assertEqual(str(self.addr2.get_addr_obj("")), "192.168.1.1")
        self.assertEqual(str(self.addr1.get_addr_obj("*")), "192.168.1.1:*")
        self.assertEqual(str(self.addr4.get_addr_obj("443")), "[fe00::1]:443")
        self.assertEqual(str(self.addr5.get_addr_obj("")), "[fe00::1]")
        self.assertEqual(str(self.addr4.get_addr_obj("*")), "[fe00::1]:*")

    def test_eq(self):
        self.assertEqual(self.addr1, self.addr2.get_addr_obj(""))
        self.assertNotEqual(self.addr1, self.addr2)
        self.assertFalse(self.addr1 == 3333)

        self.assertEqual(self.addr4, self.addr4.get_addr_obj(""))
        self.assertNotEqual(self.addr4, self.addr5)
        self.assertFalse(self.addr4 == 3333)
        from certbot.plugins.common import Addr
        self.assertEqual(self.addr4, Addr.fromstring("[fe00:0:0::1]"))
        self.assertEqual(self.addr4, Addr.fromstring("[fe00:0::0:0:1]"))

    def test_set_inclusion(self):
        from certbot.plugins.common import Addr
        set_a = set([self.addr1, self.addr2])
        addr1b = Addr.fromstring("192.168.1.1")
        addr2b = Addr.fromstring("192.168.1.1:*")
        set_b = set([addr1b, addr2b])

        self.assertEqual(set_a, set_b)

        set_c = set([self.addr4, self.addr5])
        addr4b = Addr.fromstring("[fe00::1]")
        addr5b = Addr.fromstring("[fe00::1]:*")
        set_d = set([addr4b, addr5b])

        self.assertEqual(set_c, set_d)

class TLSSNI01Test(unittest.TestCase):
    """Tests for certbot.plugins.common.TLSSNI01."""

    auth_key = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
    achalls = [
        achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(token=b'token1'), "pending"),
            domain="encryption-example.demo", account_key=auth_key),
        achallenges.KeyAuthorizationAnnotatedChallenge(
            challb=acme_util.chall_to_challb(
                challenges.TLSSNI01(token=b'token2'), "pending"),
            domain="certbot.demo", account_key=auth_key),
    ]

    def setUp(self):
        self.tempdir = tempfile.mkdtemp()
        configurator = mock.MagicMock()
        configurator.config.config_dir = os.path.join(self.tempdir, "config")
        configurator.config.work_dir = os.path.join(self.tempdir, "work")

        from certbot.plugins.common import TLSSNI01
        self.sni = TLSSNI01(configurator=configurator)

    def tearDown(self):
        shutil.rmtree(self.tempdir)

    def test_add_chall(self):
        self.sni.add_chall(self.achalls[0], 0)
        self.assertEqual(1, len(self.sni.achalls))
        self.assertEqual([0], self.sni.indices)

    def test_setup_challenge_cert(self):
        # This is a helper function that can be used for handling
        # open context managers more elegantly. It avoids dealing with
        # __enter__ and __exit__ calls.
        # http://www.voidspace.org.uk/python/mock/helpers.html#mock.mock_open
        mock_open, mock_safe_open = mock.mock_open(), mock.mock_open()

        response = challenges.TLSSNI01Response()
        achall = mock.MagicMock()
        achall.chall.encode.return_value = "token"
        key = test_util.load_pyopenssl_private_key("rsa512_key.pem")
        achall.response_and_validation.return_value = (
            response, (test_util.load_cert("cert.pem"), key))

        with mock.patch("certbot.plugins.common.open",
                        mock_open, create=True):
            with mock.patch("certbot.plugins.common.util.safe_open",
                            mock_safe_open):
                # pylint: disable=protected-access
                self.assertEqual(response, self.sni._setup_challenge_cert(
                    achall, "randomS1"))

        # pylint: disable=no-member
        mock_open.assert_called_once_with(self.sni.get_cert_path(achall), "wb")
        mock_open.return_value.write.assert_called_once_with(
            test_util.load_vector("cert.pem"))
        mock_safe_open.assert_called_once_with(
            self.sni.get_key_path(achall), "wb", chmod=0o400)
        mock_safe_open.return_value.write.assert_called_once_with(
            OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key))

class InstallSslOptionsConfTest(test_util.TempDirTestCase):
    """Tests for certbot.plugins.common.install_ssl_options_conf."""

    def setUp(self):
        super(InstallSslOptionsConfTest, self).setUp()
        self.hashes = ["someotherhash"]
        self.dest_path = os.path.join(self.tempdir, "options-ssl-dest.conf")
        self.hash_path = os.path.join(self.tempdir, ".options-ssl-conf.txt")
        self.old_path = os.path.join(self.tempdir, "options-ssl-old.conf")
        self.source_path = os.path.join(self.tempdir, "options-ssl-src.conf")
        for path in (self.source_path, self.old_path,):
            with open(path, "w") as f:
                f.write(path)
            self.hashes.append(crypto_util.sha256sum(path))

    def _call(self):
        from certbot.plugins.common import install_ssl_options_conf
        install_ssl_options_conf(self.dest_path,
                                 self.hash_path,
                                 self.source_path,
                                 self.hashes)

    def _current_ssl_options_hash(self):
        return crypto_util.sha256sum(self.source_path)

    def _assert_current_file(self):
        self.assertTrue(os.path.isfile(self.dest_path))
        self.assertEqual(crypto_util.sha256sum(self.dest_path),
            self._current_ssl_options_hash())

    def test_no_file(self):
        self.assertFalse(os.path.isfile(self.dest_path))
        self._call()
        self._assert_current_file()

    def test_current_file(self):
        # 1st iteration installs the file, the 2nd checks if it needs updating
        for _ in range(2):
            self._call()
            self._assert_current_file()

    def test_prev_file_updates_to_current(self):
        shutil.copyfile(self.old_path, self.dest_path)
        self._call()
        self._assert_current_file()

    def test_manually_modified_current_file_does_not_update(self):
        self._call()
        with open(self.dest_path, "a") as mod_ssl_conf:
            mod_ssl_conf.write("a new line for the wrong hash\n")
        with mock.patch("certbot.plugins.common.logger") as mock_logger:
            self._call()
            self.assertFalse(mock_logger.warning.called)
        self.assertTrue(os.path.isfile(self.dest_path))
        self.assertEqual(crypto_util.sha256sum(self.source_path),
            self._current_ssl_options_hash())
        self.assertNotEqual(crypto_util.sha256sum(self.dest_path),
            self._current_ssl_options_hash())

    def test_manually_modified_past_file_warns(self):
        with open(self.dest_path, "a") as mod_ssl_conf:
            mod_ssl_conf.write("a new line for the wrong hash\n")
        with open(self.hash_path, "w") as f:
            f.write("hashofanoldversion")
        with mock.patch("certbot.plugins.common.logger") as mock_logger:
            self._call()
            self.assertEqual(mock_logger.warning.call_args[0][0],
                "%s has been manually modified; updated ssl configuration options "
                "saved to %s. We recommend updating %s for security purposes.")
        self.assertEqual(crypto_util.sha256sum(self.source_path),
            self._current_ssl_options_hash())
        # only print warning once
        with mock.patch("certbot.plugins.common.logger") as mock_logger:
            self._call()
            self.assertFalse(mock_logger.warning.called)

if __name__ == "__main__":
    unittest.main()  # pragma: no cover
<|endoftext|>


<commit_before>import chainer
from chainer.functions.array import broadcast
from chainer.functions.array import reshape

def bias(x, y, axis=1):
    """Elementwise summation with broadcasting.

    Computes a elementwise summation of two input variables, with the shape of
    the latter variable broadcasted to match the shape of the former. `axis`
    is the first axis of the first variable along which to apply the second
    variable.

    Args:
        x (~chainer.Variable): Input variable to be summed.
        y (~chainer.Variable): Input variable to sum, broadcasted.
        axis (int): The first axis of `x` along which to apply `y`

    Returns:
        ~chainer.Variable: Output variable.

    """
    x_shape = x.data.shape
    y_shape = y.data.shape
    if chainer.is_debug():
        assert x_shape[axis:axis + len(y_shape)] == y_shape
    y1_shape = tuple([1] * axis + list(y_shape) +
                     [1] * (len(x_shape) - axis - len(y_shape)))
    y1 = reshape.reshape(y, y1_shape)
    y2 = broadcast.broadcast_to(y1, x_shape)
    return x + y2
<commit_msg>Fix bias function's documentation.<commit_after>import chainer
from chainer.functions.array import broadcast
from chainer.functions.array import reshape

def bias(x, y, axis=1):
    """Elementwise summation with broadcasting.

    Computes a elementwise summation of two input variables, with the shape of
    the latter variable broadcasted to match the shape of the former. `axis`
    is the first axis of the first variable along which the second variable is
    applied.

    Args:
        x (~chainer.Variable): Input variable to be summed.
        y (~chainer.Variable): Input variable to sum, broadcasted.
        axis (int): The first axis of `x` along which `y` is applied.

    Returns:
        ~chainer.Variable: Output variable.

    """
    x_shape = x.data.shape
    y_shape = y.data.shape
    if chainer.is_debug():
        assert x_shape[axis:axis + len(y_shape)] == y_shape
    y1_shape = tuple([1] * axis + list(y_shape) +
                     [1] * (len(x_shape) - axis - len(y_shape)))
    y1 = reshape.reshape(y, y1_shape)
    y2 = broadcast.broadcast_to(y1, x_shape)
    return x + y2
<|endoftext|>