<commit_before>#!/usr/bin/env python3
'''A utility to update LLVM IR CHECK lines in C/C++ FileCheck test files.

Example RUN lines in .c/.cc test files:

// RUN: %clang -emit-llvm -S %s -o - -O2 | FileCheck %s
// RUN: %clangxx -emit-llvm -S %s -o - -O2 | FileCheck -check-prefix=CHECK-A %s

Usage:

% utils/update_cc_test_checks.py --llvm-bin=release/bin test/a.cc
% utils/update_cc_test_checks.py --c-index-test=release/bin/c-index-test \
  --clang=release/bin/clang /tmp/c/a.cc
'''

import argparse
import collections
import distutils.spawn
import os
import shlex
import string
import subprocess
import sys
import re
import tempfile

from UpdateTestChecks import asm, common

ADVERT = '// NOTE: Assertions have been autogenerated by '

CHECK_RE = re.compile(r'^\s*//\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')
RUN_LINE_RE = re.compile('^//\s*RUN:\s*(.*)$')

SUBST = {
    '%clang': [],
    '%clang_cc1': ['-cc1'],
    '%clangxx': ['--driver-mode=g++'],
}

def get_line2spell_and_mangled(args, clang_args):
  ret = {}
  with tempfile.NamedTemporaryFile() as f:
    # TODO Make c-index-test print mangled names without circumventing through precompiled headers
    status = subprocess.run([args.c_index_test, '-write-pch', f.name, *clang_args],
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if status.returncode:
      sys.stderr.write(status.stdout.decode())
      sys.exit(2)
    output = subprocess.check_output([args.c_index_test,
        '-test-print-mangle', f.name])
    if sys.version_info[0] > 2:
      output = output.decode()
  DeclRE = re.compile(r'^FunctionDecl=(\w+):(\d+):\d+ \(Definition\)')
  MangleRE = re.compile(r'.*\[mangled=([^]]+)\]')
  MatchedDecl = False
  for line in output.splitlines():
    # Get the function source name, line number and mangled name.  Sometimes
    # c-index-test outputs the mangled name on a separate line (this can happen
    # with block comments in front of functions).  Keep scanning until we see
    # the mangled name.
    decl_m = DeclRE.match(line)
    mangle_m = MangleRE.match(line)

    if decl_m:
      MatchedDecl = True
      spell, lineno = decl_m.groups()
    if MatchedDecl and mangle_m:
      mangled = mangle_m.group(1)
      MatchedDecl = False
    else:
      continue

    if mangled == '_' + spell:
      # HACK for MacOS (where the mangled name includes an _ for C but the IR won't):
      mangled = spell
    # Note -test-print-mangle does not print file names so if #include is used,
    # the line number may come from an included file.
    ret[int(lineno)-1] = (spell, mangled)
  if args.verbose:
    for line, func_name in sorted(ret.items()):
      print('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)
  return ret

def config():
  parser = argparse.ArgumentParser(
      description=__doc__,
      formatter_class=argparse.RawTextHelpFormatter)
  parser.add_argument('-v', '--verbose', action='store_true')
  parser.add_argument('--llvm-bin', help='llvm $prefix/bin path')
  parser.add_argument('--clang',
                      help='"clang" executable, defaults to $llvm_bin/clang')
  parser.add_argument('--clang-args',
                      help='Space-separated extra args to clang, e.g. --clang-args=-v')
  parser.add_argument('--c-index-test',
                      help='"c-index-test" executable, defaults to $llvm_bin/c-index-test')
  parser.add_argument(
      '--functions', nargs='+', help='A list of function name regexes. '
      'If specified, update CHECK lines for functions matching at least one regex')
  parser.add_argument(
      '--x86_extra_scrub', action='store_true',
      help='Use more regex for x86 matching to reduce diffs between various subtargets')
  parser.add_argument('-u', '--update-only', action='store_true',
                      help='Only update test if it was already autogened')
  parser.add_argument('tests', nargs='+')
  args = parser.parse_args()
  args.clang_args = shlex.split(args.clang_args or '')

  if args.clang is None:
    if args.llvm_bin is None:
      args.clang = 'clang'
    else:
      args.clang = os.path.join(args.llvm_bin, 'clang')
  if not distutils.spawn.find_executable(args.clang):
    print('Please specify --llvm-bin or --clang', file=sys.stderr)
    sys.exit(1)
  if args.c_index_test is None:
    if args.llvm_bin is None:
      args.c_index_test = 'c-index-test'
    else:
      args.c_index_test = os.path.join(args.llvm_bin, 'c-index-test')
  if not distutils.spawn.find_executable(args.c_index_test):
    print('Please specify --llvm-bin or --c-index-test', file=sys.stderr)
    sys.exit(1)

  return args

def get_function_body(args, filename, clang_args, prefixes, triple_in_cmd, func_dict):
  # TODO Clean up duplication of asm/common build_function_body_dictionary
  # Invoke external tool and extract function bodies.
  raw_tool_output = common.invoke_tool(args.clang, clang_args, filename)
  if '-emit-llvm' in clang_args:
    common.build_function_body_dictionary(
            common.OPT_FUNCTION_RE, common.scrub_body, [],
            raw_tool_output, prefixes, func_dict, args.verbose)
  else:
    print('The clang command line should include -emit-llvm as asm tests '
          'are discouraged in Clang testsuite.', file=sys.stderr)
    sys.exit(1)

def main():
  args = config()
  script_name = os.path.basename(__file__)
  autogenerated_note = (ADVERT + 'utils/' + script_name)

  for filename in args.tests:
    with open(filename) as f:
      input_lines = [l.rstrip() for l in f]

    first_line = input_lines[0] if input_lines else ""
    if 'autogenerated' in first_line and script_name not in first_line:
      common.warn("Skipping test which wasn't autogenerated by " + script_name, filename)
      continue

    if args.update_only:
      if not first_line or 'autogenerated' not in first_line:
        common.warn("Skipping test which isn't autogenerated: " + filename)
        continue

    # Extract RUN lines.
    raw_lines = [m.group(1)
                 for m in [RUN_LINE_RE.match(l) for l in input_lines] if m]
    run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
    for l in raw_lines[1:]:
      if run_lines[-1].endswith("\\"):
        run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
      else:
        run_lines.append(l)

    if args.verbose:
      print('Found {} RUN lines:'.format(len(run_lines)), file=sys.stderr)
      for l in run_lines:
        print('  RUN: ' + l, file=sys.stderr)

    # Build a list of clang command lines and check prefixes from RUN lines.
    run_list = []
    line2spell_and_mangled_list = collections.defaultdict(list)
    for l in run_lines:
      commands = [cmd.strip() for cmd in l.split('|', 1)]

      triple_in_cmd = None
      m = common.TRIPLE_ARG_RE.search(commands[0])
      if m:
        triple_in_cmd = m.groups()[0]

      # Apply %clang substitution rule, replace %s by `filename`, and append args.clang_args
      clang_args = shlex.split(commands[0])
      if clang_args[0] not in SUBST:
        print('WARNING: Skipping non-clang RUN line: ' + l, file=sys.stderr)
        continue
      clang_args[0:1] = SUBST[clang_args[0]]
      clang_args = [filename if i == '%s' else i for i in clang_args] + args.clang_args

      # Extract -check-prefix in FileCheck args
      filecheck_cmd = commands[-1]
      common.verify_filecheck_prefixes(filecheck_cmd)
      if not filecheck_cmd.startswith('FileCheck '):
        print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
        continue
      check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                               for item in m.group(1).split(',')]
      if not check_prefixes:
        check_prefixes = ['CHECK']
      run_list.append((check_prefixes, clang_args, triple_in_cmd))

    # Strip CHECK lines which are in `prefix_set`, update test file.
    prefix_set = set([prefix for p in run_list for prefix in p[0]])
    input_lines = []
    with open(filename, 'r+') as f:
      for line in f:
        m = CHECK_RE.match(line)
        if not (m and m.group(1) in prefix_set) and line != '//\n':
          input_lines.append(line)
      f.seek(0)
      f.writelines(input_lines)
      f.truncate()

    # Execute clang, generate LLVM IR, and extract functions.
    func_dict = {}
    for p in run_list:
      prefixes = p[0]
      for prefix in prefixes:
        func_dict.update({prefix: dict()})
    for prefixes, clang_args, triple_in_cmd in run_list:
      if args.verbose:
        print('Extracted clang cmd: clang {}'.format(clang_args), file=sys.stderr)
        print('Extracted FileCheck prefixes: {}'.format(prefixes), file=sys.stderr)

      get_function_body(args, filename, clang_args, prefixes, triple_in_cmd, func_dict)

      # Invoke c-index-test to get mapping from start lines to mangled names.
      # Forward all clang args for now.
      for k, v in get_line2spell_and_mangled(args, clang_args).items():
        line2spell_and_mangled_list[k].append(v)

    output_lines = [autogenerated_note]
    for idx, line in enumerate(input_lines):
      # Discard any previous script advertising.
      if line.startswith(ADVERT):
        continue
      if idx in line2spell_and_mangled_list:
        added = set()
        for spell, mangled in line2spell_and_mangled_list[idx]:
          # One line may contain multiple function declarations.
          # Skip if the mangled name has been added before.
          # The line number may come from an included file,
          # we simply require the spelling name to appear on the line
          # to exclude functions from other files.
          if mangled in added or spell not in line:
            continue
          if args.functions is None or any(re.search(regex, spell) for regex in args.functions):
            if added:
              output_lines.append('//')
            added.add(mangled)
            common.add_ir_checks(output_lines, '//', run_list, func_dict, mangled)
      output_lines.append(line.rstrip('\n'))

    # Update the test file.
    with open(filename, 'w') as f:
      for line in output_lines:
        f.write(line + '\n')

  return 0

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>[update_cc_test_checks] Support 'clang | opt | FileCheck'<commit_after>#!/usr/bin/env python3
'''A utility to update LLVM IR CHECK lines in C/C++ FileCheck test files.

Example RUN lines in .c/.cc test files:

// RUN: %clang -emit-llvm -S %s -o - -O2 | FileCheck %s
// RUN: %clangxx -emit-llvm -S %s -o - -O2 | FileCheck -check-prefix=CHECK-A %s

Usage:

% utils/update_cc_test_checks.py --llvm-bin=release/bin test/a.cc
% utils/update_cc_test_checks.py --c-index-test=release/bin/c-index-test \
  --clang=release/bin/clang /tmp/c/a.cc
'''

import argparse
import collections
import distutils.spawn
import os
import shlex
import string
import subprocess
import sys
import re
import tempfile

from UpdateTestChecks import asm, common

ADVERT = '// NOTE: Assertions have been autogenerated by '

CHECK_RE = re.compile(r'^\s*//\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')
RUN_LINE_RE = re.compile('^//\s*RUN:\s*(.*)$')

SUBST = {
    '%clang': [],
    '%clang_cc1': ['-cc1'],
    '%clangxx': ['--driver-mode=g++'],
}

def get_line2spell_and_mangled(args, clang_args):
  ret = {}
  with tempfile.NamedTemporaryFile() as f:
    # TODO Make c-index-test print mangled names without circumventing through precompiled headers
    status = subprocess.run([args.c_index_test, '-write-pch', f.name, *clang_args],
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if status.returncode:
      sys.stderr.write(status.stdout.decode())
      sys.exit(2)
    output = subprocess.check_output([args.c_index_test,
        '-test-print-mangle', f.name])
    if sys.version_info[0] > 2:
      output = output.decode()
  DeclRE = re.compile(r'^FunctionDecl=(\w+):(\d+):\d+ \(Definition\)')
  MangleRE = re.compile(r'.*\[mangled=([^]]+)\]')
  MatchedDecl = False
  for line in output.splitlines():
    # Get the function source name, line number and mangled name.  Sometimes
    # c-index-test outputs the mangled name on a separate line (this can happen
    # with block comments in front of functions).  Keep scanning until we see
    # the mangled name.
    decl_m = DeclRE.match(line)
    mangle_m = MangleRE.match(line)

    if decl_m:
      MatchedDecl = True
      spell, lineno = decl_m.groups()
    if MatchedDecl and mangle_m:
      mangled = mangle_m.group(1)
      MatchedDecl = False
    else:
      continue

    if mangled == '_' + spell:
      # HACK for MacOS (where the mangled name includes an _ for C but the IR won't):
      mangled = spell
    # Note -test-print-mangle does not print file names so if #include is used,
    # the line number may come from an included file.
    ret[int(lineno)-1] = (spell, mangled)
  if args.verbose:
    for line, func_name in sorted(ret.items()):
      print('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)
  return ret

def config():
  parser = argparse.ArgumentParser(
      description=__doc__,
      formatter_class=argparse.RawTextHelpFormatter)
  parser.add_argument('-v', '--verbose', action='store_true')
  parser.add_argument('--llvm-bin', help='llvm $prefix/bin path')
  parser.add_argument('--clang',
                      help='"clang" executable, defaults to $llvm_bin/clang')
  parser.add_argument('--clang-args',
                      help='Space-separated extra args to clang, e.g. --clang-args=-v')
  parser.add_argument('--c-index-test',
                      help='"c-index-test" executable, defaults to $llvm_bin/c-index-test')
  parser.add_argument('--opt',
                      help='"opt" executable, defaults to $llvm_bin/opt')
  parser.add_argument(
      '--functions', nargs='+', help='A list of function name regexes. '
      'If specified, update CHECK lines for functions matching at least one regex')
  parser.add_argument(
      '--x86_extra_scrub', action='store_true',
      help='Use more regex for x86 matching to reduce diffs between various subtargets')
  parser.add_argument('-u', '--update-only', action='store_true',
                      help='Only update test if it was already autogened')
  parser.add_argument('tests', nargs='+')
  args = parser.parse_args()
  args.clang_args = shlex.split(args.clang_args or '')

  if args.clang is None:
    if args.llvm_bin is None:
      args.clang = 'clang'
    else:
      args.clang = os.path.join(args.llvm_bin, 'clang')
  if not distutils.spawn.find_executable(args.clang):
    print('Please specify --llvm-bin or --clang', file=sys.stderr)
    sys.exit(1)

  if args.opt is None:
    if args.llvm_bin is None:
      args.opt = 'opt'
    else:
      args.opt = os.path.join(args.llvm_bin, 'opt')
  if not distutils.spawn.find_executable(args.opt):
    # Many uses of this tool will not need an opt binary, because it's only
    # needed for updating a test that runs clang | opt | FileCheck. So we
    # defer this error message until we find that opt is actually needed.
    args.opt = None

  if args.c_index_test is None:
    if args.llvm_bin is None:
      args.c_index_test = 'c-index-test'
    else:
      args.c_index_test = os.path.join(args.llvm_bin, 'c-index-test')
  if not distutils.spawn.find_executable(args.c_index_test):
    print('Please specify --llvm-bin or --c-index-test', file=sys.stderr)
    sys.exit(1)

  return args

def get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict):
  # TODO Clean up duplication of asm/common build_function_body_dictionary
  # Invoke external tool and extract function bodies.
  raw_tool_output = common.invoke_tool(args.clang, clang_args, filename)
  for extra_command in extra_commands:
    extra_args = shlex.split(extra_command)
    with tempfile.NamedTemporaryFile() as f:
      f.write(raw_tool_output.encode())
      f.flush()
      if extra_args[0] == 'opt':
        if args.opt is None:
          print(filename, 'needs to run opt. '
                'Please specify --llvm-bin or --opt', file=sys.stderr)
          sys.exit(1)
        extra_args[0] = args.opt
      raw_tool_output = common.invoke_tool(extra_args[0],
                                           extra_args[1:], f.name)
  if '-emit-llvm' in clang_args:
    common.build_function_body_dictionary(
            common.OPT_FUNCTION_RE, common.scrub_body, [],
            raw_tool_output, prefixes, func_dict, args.verbose)
  else:
    print('The clang command line should include -emit-llvm as asm tests '
          'are discouraged in Clang testsuite.', file=sys.stderr)
    sys.exit(1)

def main():
  args = config()
  script_name = os.path.basename(__file__)
  autogenerated_note = (ADVERT + 'utils/' + script_name)

  for filename in args.tests:
    with open(filename) as f:
      input_lines = [l.rstrip() for l in f]

    first_line = input_lines[0] if input_lines else ""
    if 'autogenerated' in first_line and script_name not in first_line:
      common.warn("Skipping test which wasn't autogenerated by " + script_name, filename)
      continue

    if args.update_only:
      if not first_line or 'autogenerated' not in first_line:
        common.warn("Skipping test which isn't autogenerated: " + filename)
        continue

    # Extract RUN lines.
    raw_lines = [m.group(1)
                 for m in [RUN_LINE_RE.match(l) for l in input_lines] if m]
    run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
    for l in raw_lines[1:]:
      if run_lines[-1].endswith("\\"):
        run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
      else:
        run_lines.append(l)

    if args.verbose:
      print('Found {} RUN lines:'.format(len(run_lines)), file=sys.stderr)
      for l in run_lines:
        print('  RUN: ' + l, file=sys.stderr)

    # Build a list of clang command lines and check prefixes from RUN lines.
    run_list = []
    line2spell_and_mangled_list = collections.defaultdict(list)
    for l in run_lines:
      commands = [cmd.strip() for cmd in l.split('|')]

      triple_in_cmd = None
      m = common.TRIPLE_ARG_RE.search(commands[0])
      if m:
        triple_in_cmd = m.groups()[0]

      # Apply %clang substitution rule, replace %s by `filename`, and append args.clang_args
      clang_args = shlex.split(commands[0])
      if clang_args[0] not in SUBST:
        print('WARNING: Skipping non-clang RUN line: ' + l, file=sys.stderr)
        continue
      clang_args[0:1] = SUBST[clang_args[0]]
      clang_args = [filename if i == '%s' else i for i in clang_args] + args.clang_args

      # Permit piping the output through opt
      if not (len(commands) == 2 or
              (len(commands) == 3 and commands[1].startswith('opt'))):
        print('WARNING: Skipping non-clang RUN line: ' + l, file=sys.stderr)

      # Extract -check-prefix in FileCheck args
      filecheck_cmd = commands[-1]
      common.verify_filecheck_prefixes(filecheck_cmd)
      if not filecheck_cmd.startswith('FileCheck '):
        print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
        continue
      check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                               for item in m.group(1).split(',')]
      if not check_prefixes:
        check_prefixes = ['CHECK']
      run_list.append((check_prefixes, clang_args, commands[1:-1], triple_in_cmd))

    # Strip CHECK lines which are in `prefix_set`, update test file.
    prefix_set = set([prefix for p in run_list for prefix in p[0]])
    input_lines = []
    with open(filename, 'r+') as f:
      for line in f:
        m = CHECK_RE.match(line)
        if not (m and m.group(1) in prefix_set) and line != '//\n':
          input_lines.append(line)
      f.seek(0)
      f.writelines(input_lines)
      f.truncate()

    # Execute clang, generate LLVM IR, and extract functions.
    func_dict = {}
    for p in run_list:
      prefixes = p[0]
      for prefix in prefixes:
        func_dict.update({prefix: dict()})
    for prefixes, clang_args, extra_commands, triple_in_cmd in run_list:
      if args.verbose:
        print('Extracted clang cmd: clang {}'.format(clang_args), file=sys.stderr)
        print('Extracted FileCheck prefixes: {}'.format(prefixes), file=sys.stderr)

      get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict)

      # Invoke c-index-test to get mapping from start lines to mangled names.
      # Forward all clang args for now.
      for k, v in get_line2spell_and_mangled(args, clang_args).items():
        line2spell_and_mangled_list[k].append(v)

    output_lines = [autogenerated_note]
    for idx, line in enumerate(input_lines):
      # Discard any previous script advertising.
      if line.startswith(ADVERT):
        continue
      if idx in line2spell_and_mangled_list:
        added = set()
        for spell, mangled in line2spell_and_mangled_list[idx]:
          # One line may contain multiple function declarations.
          # Skip if the mangled name has been added before.
          # The line number may come from an included file,
          # we simply require the spelling name to appear on the line
          # to exclude functions from other files.
          if mangled in added or spell not in line:
            continue
          if args.functions is None or any(re.search(regex, spell) for regex in args.functions):
            if added:
              output_lines.append('//')
            added.add(mangled)
            common.add_ir_checks(output_lines, '//', run_list, func_dict, mangled, False)
      output_lines.append(line.rstrip('\n'))

    # Update the test file.
    with open(filename, 'w') as f:
      for line in output_lines:
        f.write(line + '\n')

  return 0

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>import pytest

__author__ = 'thauser'
from pnc_cli.swagger_client.apis import ProductmilestonesApi
from pnc_cli.swagger_client.apis import ProductversionsApi
from pnc_cli import utils
from pnc_cli import productmilestones
from pnc_cli import productversions
from test import testutils

versions_api = ProductversionsApi(utils.get_api_client())
milestone_api = ProductmilestonesApi(utils.get_api_client())

@pytest.fixture(scope='function')
def new_version():
    version_number = testutils.gen_random_version()
    while version_number in [x.version for x in versions_api.get_all(page_size=1000000).content]:
        version_number = testutils.gen_random_version()
    version = versions_api.create_new_product_version(body=productversions.create_product_version_object(
        version=version_number,
        product_id=1,
        current_product_milestone_id=1
    )).content
    return version

@pytest.fixture(scope='function')
def new_milestone(new_version):
    milestone = milestone_api.create_new(body=productmilestones.create_milestone_object(
        product_version_id=new_version.id, version=new_version.version + ".1.GA", start_date="2015-01-01",
        planned_release_date="2015-01-02"
    )).content
    return milestone

def test_get_all_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'get_all')

def test_get_all(new_milestone):
    assert milestone_api.get_all(page_index=0, page_size=1000000, sort='', q='').content is not None

def test_create_new_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'create_new')

def test_create_new(new_milestone):
    milestones = [m.id for m in milestone_api.get_all(page_size=1000000).content]
    assert new_milestone.id in milestones

def test_get_all_by_product_version_id_no_version_id():
    testutils.assert_raises_valueerror(milestone_api, 'get_all_by_product_version_id', version_id=None)

def test_get_all_by_product_version_id_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'get_all_by_product_version_id', version_id=1)

def test_get_all_by_product_version_id():
    milestones = milestone_api.get_all_by_product_version_id(version_id=1, page_index=0, page_size=1000000, sort='',
                                                             q='').content
    assert milestones is not None

def test_get_specific_no_id():
    testutils.assert_raises_valueerror(milestone_api, 'get_specific', id=None)

def test_get_specific_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'get_specific', id=1)

def test_get_specific(new_milestone):
    retrieved = milestone_api.get_specific(new_milestone.id).content
    assert new_milestone.to_dict() == retrieved.to_dict()

def test_update_no_id():
    testutils.assert_raises_valueerror(milestone_api, 'update', id=None)

def test_update_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'update', id=1)

def test_update(new_milestone):
    new_milestone.download_url = "updatedUrlHere"
    milestone_api.update(id=new_milestone.id, body=new_milestone)
    updated = milestone_api.get_specific(new_milestone.id).content
    assert updated.to_dict() == new_milestone.to_dict()

<commit_msg>use fixtures to create new products for productmilestones testing<commit_after>import pytest

__author__ = 'thauser'
from pnc_cli.swagger_client.apis import ProductmilestonesApi
from pnc_cli.swagger_client.apis import ProductversionsApi
from pnc_cli.swagger_client.apis import ProductsApi
from pnc_cli import utils
from pnc_cli import productmilestones
from pnc_cli import productversions
from pnc_cli import products
from test import testutils

product_api = ProductsApi(utils.get_api_client())
versions_api = ProductversionsApi(utils.get_api_client())
milestone_api = ProductmilestonesApi(utils.get_api_client())

@pytest.fixture(scope='function')
def new_product():
    product = product_api.create_new(body=products._create_product_object(name=testutils.gen_random_name(),
                                                                          description="PNC CLI: test_productmilestones_api product"
                                                                          )).content
    return product

@pytest.fixture(scope='function')
def new_version(new_product):
    version_number = testutils.gen_random_version()
    existing = product_api.get_product_versions(id=new_product.id, page_size=1000000).content
    while existing is not None and version_number in [x.version for x in existing]:
        version_number = testutils.gen_random_version()
    version = versions_api.create_new_product_version(body=productversions.create_product_version_object(
        version=version_number,
        product_id=1,
        current_product_milestone_id=1
    )).content
    return version

@pytest.fixture(scope='function')
def new_milestone(new_version):
    milestone = milestone_api.create_new(body=productmilestones.create_milestone_object(
        product_version_id=new_version.id, version=new_version.version + ".1.GA", start_date="2015-01-01",
        planned_release_date="2015-01-02"
    )).content
    return milestone

def test_get_all_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'get_all')

def test_get_all(new_milestone):
    assert milestone_api.get_all(page_index=0, page_size=1000000, sort='', q='').content is not None

def test_create_new_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'create_new')

def test_create_new(new_milestone):
    milestones = [m.id for m in milestone_api.get_all(page_size=1000000).content]
    assert new_milestone.id in milestones

def test_get_all_by_product_version_id_no_version_id():
    testutils.assert_raises_valueerror(milestone_api, 'get_all_by_product_version_id', version_id=None)

def test_get_all_by_product_version_id_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'get_all_by_product_version_id', version_id=1)

def test_get_all_by_product_version_id():
    milestones = milestone_api.get_all_by_product_version_id(version_id=1, page_index=0, page_size=1000000, sort='',
                                                             q='').content
    assert milestones is not None

def test_get_specific_no_id():
    testutils.assert_raises_valueerror(milestone_api, 'get_specific', id=None)

def test_get_specific_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'get_specific', id=1)

def test_get_specific(new_milestone):
    retrieved = milestone_api.get_specific(new_milestone.id).content
    assert new_milestone.to_dict() == retrieved.to_dict()

def test_update_no_id():
    testutils.assert_raises_valueerror(milestone_api, 'update', id=None)

def test_update_invalid_param():
    testutils.assert_raises_typeerror(milestone_api, 'update', id=1)

def test_update(new_milestone):
    new_milestone.download_url = "updatedUrlHere"
    milestone_api.update(id=new_milestone.id, body=new_milestone)
    updated = milestone_api.get_specific(new_milestone.id).content
    assert updated.to_dict() == new_milestone.to_dict()

<|endoftext|>


<commit_before># This file is originally from recaptcha-client 1.0.5 (obtained from pypi),
# now modified to use new Recaptcha.

import urllib
import urllib2

import config
import simplejson

API_SSL_SERVER = 'https://www.google.com/recaptcha/api'
# We leave out the URL scheme so that browsers won't complain when accessed
# from a secure website even when use_ssl is not set.
API_SERVER = '//www.google.com/recaptcha/api'
VERIFY_SERVER = 'api-verify.recaptcha.net'

class RecaptchaResponse(object):
    def __init__(self, is_valid, error_code=None):
        self.is_valid = is_valid
        self.error_code = error_code

def get_display_html(public_key, use_ssl=False, error=None,
                     lang='en'):
    """Gets the HTML to display for reCAPTCHA

    public_key -- The public api key
    use_ssl -- Should the request be sent over ssl?
    error -- An error message to display (from RecaptchaResponse.error_code)"""

    error_param = ''
    if error:
        error_param = '&error=%s' % error
    server = API_SERVER
    if use_ssl:
        server = API_SSL_SERVER

    return '''

    <textarea id="g-recaptcha-response" name="g-recaptcha-response"></textarea>

''' % {
    'server': server,
    'lang': lang,
    'public_key': public_key
}

def submit (recaptcha_response):
    """
    Submits a reCAPTCHA request for verification. Returns RecaptchaResponse
    for the request

    recaptcha_response -- The value of recaptcha_response
    """

    if not recaptcha_response:
        return RecaptchaResponse (is_valid = False, error_code = 'incorrect-captcha-sol')

    secret_key = config.get('captcha_secret_key')
    request_url = (
        "https://www.google.com/recaptcha/api/siteverify?secret=?%s&response=%s"
        % (secret_key, recaptcha_response))
    recaptcha_request = urllib2.Request (request_url)
    response = urllib2.urlopen (recaptcha_request)
    result = simplejson.load(response)
    result_code = result['success']
    if result_code:
        return RecaptchaResponse (is_valid=True)
    else:
        return RecaptchaResponse (is_valid=False)
<commit_msg>fixed secret=?%s -> secret=%s<commit_after># This file is originally from recaptcha-client 1.0.5 (obtained from pypi),
# now modified to use new Recaptcha.

import urllib
import urllib2

import config
import simplejson

API_SSL_SERVER = 'https://www.google.com/recaptcha/api'
# We leave out the URL scheme so that browsers won't complain when accessed
# from a secure website even when use_ssl is not set.
API_SERVER = '//www.google.com/recaptcha/api'
VERIFY_SERVER = 'api-verify.recaptcha.net'

class RecaptchaResponse(object):
    def __init__(self, is_valid, error_code=None):
        self.is_valid = is_valid
        self.error_code = error_code

def get_display_html(public_key, use_ssl=False, error=None,
                     lang='en'):
    """Gets the HTML to display for reCAPTCHA

    public_key -- The public api key
    use_ssl -- Should the request be sent over ssl?
    error -- An error message to display (from RecaptchaResponse.error_code)"""

    error_param = ''
    if error:
        error_param = '&error=%s' % error
    server = API_SERVER
    if use_ssl:
        server = API_SSL_SERVER

    return '''

    <textarea id="g-recaptcha-response" name="g-recaptcha-response"></textarea>

''' % {
    'server': server,
    'lang': lang,
    'public_key': public_key
}

def submit (recaptcha_response):
    """
    Submits a reCAPTCHA request for verification. Returns RecaptchaResponse
    for the request

    recaptcha_response -- The value of recaptcha_response
    """

    if not recaptcha_response:
        return RecaptchaResponse (is_valid = False, error_code = 'incorrect-captcha-sol')

    secret_key = config.get('captcha_secret_key')
    request_url = (
        "https://www.google.com/recaptcha/api/siteverify?secret=%s&response=%s"
        % (secret_key, recaptcha_response))
    recaptcha_request = urllib2.Request (request_url)
    response = urllib2.urlopen (recaptcha_request)
    result = simplejson.load(response)
    result_code = result['success']
    if result_code:
        return RecaptchaResponse (is_valid=True)
    else:
        return RecaptchaResponse (is_valid=False)
<|endoftext|>


<commit_before>import functools
import unittest

from cryptography import x509
from cryptography.hazmat.backends import default_backend
from ykman.descriptor import open_device
from ykman.util import (Cve201715361VulnerableError, TRANSPORT)
from ..util import (
    _test_serials, _ykman_cli, fips, neo, piv_attestation, roca)
from .util import (PivTestCase, DEFAULT_PIN, DEFAULT_MANAGEMENT_KEY)

def make_test_case(dev):
    ykman_cli = functools.partial(_ykman_cli, dev.serial)

    class KeyManagement(PivTestCase):
        @classmethod
        def setUpClass(cls):
            ykman_cli('piv', 'reset', '-f')

        @classmethod
        def tearDownClass(cls):
            ykman_cli('piv', 'reset', '-f')

        @roca(False)
        def test_generate_key_default(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-m', DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @roca(True)
        def test_generate_key_default_cve201715361(self):
            with self.assertRaises(Cve201715361VulnerableError):
                ykman_cli(
                    'piv', 'generate-key', '9a',
                    '-m', DEFAULT_MANAGEMENT_KEY, '-')

        @roca(False)
        @fips(False)
        def test_generate_key_rsa1024(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'RSA1024', '-m',
                DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @roca(False)
        def test_generate_key_rsa2048(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'RSA2048',
                '-m', DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @fips(False)
        @roca(True)
        def test_generate_key_rsa1024_cve201715361(self):
            with self.assertRaises(Cve201715361VulnerableError):
                ykman_cli(
                    'piv', 'generate-key', '9a', '-a', 'RSA1024', '-m',
                    DEFAULT_MANAGEMENT_KEY, '-')

        @roca(True)
        def test_generate_key_rsa2048_cve201715361(self):
            with self.assertRaises(Cve201715361VulnerableError):
                ykman_cli(
                    'piv', 'generate-key', '9a', '-a', 'RSA2048',
                    '-m', DEFAULT_MANAGEMENT_KEY, '-')

        def test_generate_key_eccp256(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @neo(False)
        def test_generate_key_eccp384(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP384', '-m',
                DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @neo(False)
        def test_generate_key_pin_policy_always(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '--pin-policy', 'ALWAYS', '-m',
                DEFAULT_MANAGEMENT_KEY, '-a', 'ECCP256', '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @neo(False)
        def test_generate_key_touch_policy_always(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '--touch-policy', 'ALWAYS', '-m',
                DEFAULT_MANAGEMENT_KEY, '-a', 'ECCP256', '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @piv_attestation(True)
        def test_attest_key(self):
            ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256',
                '-m', DEFAULT_MANAGEMENT_KEY, '-')
            output = ykman_cli('piv', 'attest', '9a', '-')
            self.assertIn('BEGIN CERTIFICATE', output)

        def _test_generate_csr(self, algo):
            ykman_cli(
                'piv', 'generate-key', '9a', '-a', algo, '-m',
                DEFAULT_MANAGEMENT_KEY, '/tmp/test-pub-key.pem')
            output = ykman_cli(
                'piv', 'generate-csr', '9a', '/tmp/test-pub-key.pem',
                '-s', 'test-subject', '-P', DEFAULT_PIN, '-')
            csr = x509.load_pem_x509_csr(output.encode(), default_backend())
            self.assertTrue(csr.is_signature_valid)

        @fips(False)
        @roca(False)
        def test_generate_csr_rsa1024(self):
            self._test_generate_csr('RSA1024')

        def test_generate_csr_eccp256(self):
            self._test_generate_csr('ECCP256')

        def test_import_verify_correct_cert_succeeds_with_pin(self):
            # Set up a key in the slot and create a certificate for it
            public_key_pem = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-')

            ykman_cli(
                'piv', 'generate-certificate', '9a', '-',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN, '-s', 'test',
                input=public_key_pem)

            ykman_cli('piv', 'export-certificate', '9a',
                      '/tmp/test-pub-key.pem')

            with self.assertRaises(SystemExit):
                ykman_cli(
                    'piv', 'import-certificate', '--verify',
                    '9a', '/tmp/test-pub-key.pem',
                    '-m', DEFAULT_MANAGEMENT_KEY)

            ykman_cli(
                'piv', 'import-certificate', '--verify',
                '9a', '/tmp/test-pub-key.pem',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN)
            ykman_cli(
                'piv', 'import-certificate', '--verify',
                '9a', '/tmp/test-pub-key.pem',
                '-m', DEFAULT_MANAGEMENT_KEY, input=DEFAULT_PIN)

        def test_import_verify_wrong_cert_fails(self):
            # Set up a key in the slot and create a certificate for it
            public_key_pem = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-')

            ykman_cli(
                'piv', 'generate-certificate', '9a', '-',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN, '-s', 'test',
                input=public_key_pem)

            cert_pem = ykman_cli('piv', 'export-certificate', '9a', '-')

            # Overwrite the key with a new one
            ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-',
                input=public_key_pem)

            with self.assertRaises(SystemExit):
                ykman_cli(
                    'piv', 'import-certificate', '--verify', '9a', '-',
                    '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN,
                    input=cert_pem)

        def test_import_no_verify_wrong_cert_succeeds(self):
            # Set up a key in the slot and create a certificate for it
            public_key_pem = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-')

            ykman_cli(
                'piv', 'generate-certificate', '9a', '-',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN, '-s', 'test',
                input=public_key_pem)

            cert_pem = ykman_cli('piv', 'export-certificate', '9a', '-')

            # Overwrite the key with a new one
            ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-',
                input=public_key_pem)

            with self.assertRaises(SystemExit):
                ykman_cli(
                    'piv', 'import-certificate', '--verify', '9a', '-',
                    '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN,
                    input=cert_pem)

            ykman_cli(
                'piv', 'import-certificate', '9a', '-',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN,
                input=cert_pem)

        @piv_attestation(True)
        def test_export_attestation_certificate(self):
            output = ykman_cli('piv', 'export-certificate', 'f9', '-')
            self.assertIn('BEGIN CERTIFICATE', output)

    fw_version = '.'.join(str(v) for v in dev.version)
    KeyManagement.__qualname__ = f'KeyManagement_{fw_version}_{dev.serial}'

    for attr_name in dir(KeyManagement):
        method = getattr(KeyManagement, attr_name)
        if attr_name.startswith('test') and 'yubikey_conditions' in dir(method):
            conditions = getattr(method, 'yubikey_conditions')
            if not all(cond(dev) for cond in conditions):
                delattr(KeyManagement, attr_name)

    return KeyManagement

def additional_tests():
    suite = unittest.TestSuite()

    for serial in _test_serials:
        with open_device(transports=TRANSPORT.CCID, serial=serial) as dev:
            test_case = make_test_case(dev)
            for attr_name in dir(test_case):
                if attr_name.startswith('test'):
                    suite.addTest(test_case(attr_name))

    return suite
<commit_msg>Generalize test creation a bit<commit_after>import functools
import unittest

from cryptography import x509
from cryptography.hazmat.backends import default_backend
from ykman.descriptor import open_device
from ykman.util import (Cve201715361VulnerableError, TRANSPORT)
from ..util import (
    _test_serials, _ykman_cli, fips, neo, piv_attestation, roca)
from .util import (PivTestCase, DEFAULT_PIN, DEFAULT_MANAGEMENT_KEY)

def make_test_classes(ykman_cli):
    class KeyManagement(PivTestCase):
        @classmethod
        def setUpClass(cls):
            ykman_cli('piv', 'reset', '-f')

        @classmethod
        def tearDownClass(cls):
            ykman_cli('piv', 'reset', '-f')

        @roca(False)
        def test_generate_key_default(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-m', DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @roca(True)
        def test_generate_key_default_cve201715361(self):
            with self.assertRaises(Cve201715361VulnerableError):
                ykman_cli(
                    'piv', 'generate-key', '9a',
                    '-m', DEFAULT_MANAGEMENT_KEY, '-')

        @roca(False)
        @fips(False)
        def test_generate_key_rsa1024(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'RSA1024', '-m',
                DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @roca(False)
        def test_generate_key_rsa2048(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'RSA2048',
                '-m', DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @fips(False)
        @roca(True)
        def test_generate_key_rsa1024_cve201715361(self):
            with self.assertRaises(Cve201715361VulnerableError):
                ykman_cli(
                    'piv', 'generate-key', '9a', '-a', 'RSA1024', '-m',
                    DEFAULT_MANAGEMENT_KEY, '-')

        @roca(True)
        def test_generate_key_rsa2048_cve201715361(self):
            with self.assertRaises(Cve201715361VulnerableError):
                ykman_cli(
                    'piv', 'generate-key', '9a', '-a', 'RSA2048',
                    '-m', DEFAULT_MANAGEMENT_KEY, '-')

        def test_generate_key_eccp256(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @neo(False)
        def test_generate_key_eccp384(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP384', '-m',
                DEFAULT_MANAGEMENT_KEY, '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @neo(False)
        def test_generate_key_pin_policy_always(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '--pin-policy', 'ALWAYS', '-m',
                DEFAULT_MANAGEMENT_KEY, '-a', 'ECCP256', '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @neo(False)
        def test_generate_key_touch_policy_always(self):
            output = ykman_cli(
                'piv', 'generate-key', '9a', '--touch-policy', 'ALWAYS', '-m',
                DEFAULT_MANAGEMENT_KEY, '-a', 'ECCP256', '-')
            self.assertIn('BEGIN PUBLIC KEY', output)

        @piv_attestation(True)
        def test_attest_key(self):
            ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256',
                '-m', DEFAULT_MANAGEMENT_KEY, '-')
            output = ykman_cli('piv', 'attest', '9a', '-')
            self.assertIn('BEGIN CERTIFICATE', output)

        def _test_generate_csr(self, algo):
            ykman_cli(
                'piv', 'generate-key', '9a', '-a', algo, '-m',
                DEFAULT_MANAGEMENT_KEY, '/tmp/test-pub-key.pem')
            output = ykman_cli(
                'piv', 'generate-csr', '9a', '/tmp/test-pub-key.pem',
                '-s', 'test-subject', '-P', DEFAULT_PIN, '-')
            csr = x509.load_pem_x509_csr(output.encode(), default_backend())
            self.assertTrue(csr.is_signature_valid)

        @fips(False)
        @roca(False)
        def test_generate_csr_rsa1024(self):
            self._test_generate_csr('RSA1024')

        def test_generate_csr_eccp256(self):
            self._test_generate_csr('ECCP256')

        def test_import_verify_correct_cert_succeeds_with_pin(self):
            # Set up a key in the slot and create a certificate for it
            public_key_pem = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-')

            ykman_cli(
                'piv', 'generate-certificate', '9a', '-',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN, '-s', 'test',
                input=public_key_pem)

            ykman_cli('piv', 'export-certificate', '9a',
                      '/tmp/test-pub-key.pem')

            with self.assertRaises(SystemExit):
                ykman_cli(
                    'piv', 'import-certificate', '--verify',
                    '9a', '/tmp/test-pub-key.pem',
                    '-m', DEFAULT_MANAGEMENT_KEY)

            ykman_cli(
                'piv', 'import-certificate', '--verify',
                '9a', '/tmp/test-pub-key.pem',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN)
            ykman_cli(
                'piv', 'import-certificate', '--verify',
                '9a', '/tmp/test-pub-key.pem',
                '-m', DEFAULT_MANAGEMENT_KEY, input=DEFAULT_PIN)

        def test_import_verify_wrong_cert_fails(self):
            # Set up a key in the slot and create a certificate for it
            public_key_pem = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-')

            ykman_cli(
                'piv', 'generate-certificate', '9a', '-',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN, '-s', 'test',
                input=public_key_pem)

            cert_pem = ykman_cli('piv', 'export-certificate', '9a', '-')

            # Overwrite the key with a new one
            ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-',
                input=public_key_pem)

            with self.assertRaises(SystemExit):
                ykman_cli(
                    'piv', 'import-certificate', '--verify', '9a', '-',
                    '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN,
                    input=cert_pem)

        def test_import_no_verify_wrong_cert_succeeds(self):
            # Set up a key in the slot and create a certificate for it
            public_key_pem = ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-')

            ykman_cli(
                'piv', 'generate-certificate', '9a', '-',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN, '-s', 'test',
                input=public_key_pem)

            cert_pem = ykman_cli('piv', 'export-certificate', '9a', '-')

            # Overwrite the key with a new one
            ykman_cli(
                'piv', 'generate-key', '9a', '-a', 'ECCP256', '-m',
                DEFAULT_MANAGEMENT_KEY, '--pin-policy', 'ALWAYS', '-',
                input=public_key_pem)

            with self.assertRaises(SystemExit):
                ykman_cli(
                    'piv', 'import-certificate', '--verify', '9a', '-',
                    '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN,
                    input=cert_pem)

            ykman_cli(
                'piv', 'import-certificate', '9a', '-',
                '-m', DEFAULT_MANAGEMENT_KEY, '-P', DEFAULT_PIN,
                input=cert_pem)

        @piv_attestation(True)
        def test_export_attestation_certificate(self):
            output = ykman_cli('piv', 'export-certificate', 'f9', '-')
            self.assertIn('BEGIN CERTIFICATE', output)

    return [KeyManagement]

def make_test_cases(dev):
    ykman_cli = functools.partial(_ykman_cli, dev.serial)

    for test_class in make_test_classes(ykman_cli):

        fw_version = '.'.join(str(v) for v in dev.version)
        test_class.__qualname__ += f'_{fw_version}_{dev.serial}'

        for attr_name in dir(test_class):
            method = getattr(test_class, attr_name)
            if (attr_name.startswith('test')
                    and 'yubikey_conditions' in dir(method)):
                conditions = getattr(method, 'yubikey_conditions')
                if not all(cond(dev) for cond in conditions):
                    delattr(test_class, attr_name)

        yield test_class

def additional_tests():
    suite = unittest.TestSuite()
    for serial in _test_serials:
        with open_device(transports=TRANSPORT.CCID, serial=serial) as dev:
            for test_case in make_test_cases(dev):
                for attr_name in dir(test_case):
                    if attr_name.startswith('test'):
                        suite.addTest(test_case(attr_name))

    return suite
<|endoftext|>


<commit_before>import pgmapcss.db as db
from .compile_eval import compile_eval

def compile_value(prop, stat):
    """returns two values, first a static value which can be put into the 'to_set' dictts, or None if the value is dynamic. The second value is the compiled version."""
    if prop['value_type'] == 'eval':
        return None, compile_eval(prop['value'])

    elif 'unit' in prop and prop['unit'] in ('m', 'u'):
        return None, compile_eval([ 'f:metric', 'v:' + prop['value'] + prop['unit'] ])

    elif prop['value_type'] == 'value' and \
        prop['assignment_type'] == 'P' and \
        'type' in stat['defines'] and \
        prop['key'] in stat['defines']['type']:
        return None, '(current.tags)->' + db.format(prop['value'])

    else:
        return prop['value'], db.format(prop['value'])
<commit_msg>compile_value(): bugfix, check for @type 'tag_value'<commit_after>import pgmapcss.db as db
from .compile_eval import compile_eval

def compile_value(prop, stat):
    """returns two values, first a static value which can be put into the 'to_set' dictts, or None if the value is dynamic. The second value is the compiled version."""
    if prop['value_type'] == 'eval':
        return None, compile_eval(prop['value'])

    elif 'unit' in prop and prop['unit'] in ('m', 'u'):
        return None, compile_eval([ 'f:metric', 'v:' + prop['value'] + prop['unit'] ])

    elif prop['value_type'] == 'value' and \
        prop['assignment_type'] == 'P' and \
        'type' in stat['defines'] and \
        prop['key'] in stat['defines']['type'] and \
        stat['defines']['type'][prop['key']]['value'] == "tag_value":
        return None, '(current.tags)->' + db.format(prop['value'])

    else:
        return prop['value'], db.format(prop['value'])
<|endoftext|>


<commit_before>from pyramid.view import view_config, view_defaults
from pyramid.httpexceptions import HTTPFound
from deform import Form, Button
from deform import ValidationFailure

from phoenix.events import JobStarted
from phoenix.views import MyView
from phoenix.wps import appstruct_to_inputs
from phoenix.wps import WPSSchema
from phoenix.utils import wps_describe_url
from phoenix.security import has_execute_permission

from owslib.wps import WebProcessingService
from owslib.wps import WPSExecution
from owslib.wps import ComplexDataInput

import logging
logger = logging.getLogger(__name__)

@view_defaults(permission='view', layout='default')
class ExecuteProcess(MyView):
    def __init__(self, request):
        self.request = request
        self.execution = None
        if 'job_id' in request.params:
            job = request.db.jobs.find_one(
                {'identifier': request.params['job_id']})
            self.execution = WPSExecution()
            self.execution.checkStatus(url=job['status_location'], sleepSecs=0)
            self.processid = self.execution.process.identifier
            self.service_name = self.request.catalog.get_service_name_by_url(
                url=self.execution.serviceInstance)
        elif 'wps' in request.params:
            self.service_name = request.params.get('wps')
            self.processid = request.params.get('process')
        else:
            self.service_name = None
            self.processid = None
            self.process = None

        if self.service_name:
            # TODO: avoid getcaps
            self.wps = WebProcessingService(
                url=request.route_url('owsproxy',
                                      service_name=self.service_name),
                verify=False)
            # TODO: need to fix owslib to handle special identifiers
            self.process = self.wps.describeprocess(self.processid)
        super(ExecuteProcess, self).__init__(request, name='processes_execute',
                                             title='')

    def breadcrumbs(self):
        breadcrumbs = super(ExecuteProcess, self).breadcrumbs()
        breadcrumbs.append(dict(
            route_path=self.request.route_path('processes'),
            title='Processes'))
        if self.service_name:
            route_path = self.request.route_path(
                'processes_list', _query=[('wps', self.service_name)])
            breadcrumbs.append(dict(
                route_path=route_path,
                title=self.wps.identification.title))
            breadcrumbs.append(dict(
                route_path=self.request.route_path(self.name),
                title=self.process.title))
        return breadcrumbs

    def appstruct(self):
        # TODO: not a nice way to get inputs ... should be cleaned up in owslib
        result = {}
        if self.execution:
            for inp in self.execution.dataInputs:
                if inp.data or inp.reference:
                    if inp.identifier not in result:
                        # init result for param with empty list
                        result[inp.identifier] = []
                    if inp.data:
                        # add literal input, inp.data is a list
                        result[inp.identifier].extend(inp.data)
                    elif inp.reference:
                        # add reference to complex input
                        result[inp.identifier].append(inp.reference)
        for inp in self.process.dataInputs:
            # convert boolean
            if 'boolean' in inp.dataType and inp.identifier in result:
                result[inp.identifier] = [val.lower() == 'true' for val in result[inp.identifier]]
            # TODO: very dirty ... if single value then take the first
            if inp.maxOccurs < 2 and inp.identifier in result:
                result[inp.identifier] = result[inp.identifier][0]
        return result

    def generate_form(self, formid='deform'):
        schema = WPSSchema(request=self.request,
                           process=self.process,
                           user=self.get_user())
        submit_button = Button(name='submit', title='Execute',
                               css_class='btn btn-success btn-lg btn-block',
                               disabled=not has_execute_permission(
                                    self.request, self.service_name))
        return Form(
            schema,
            buttons=(submit_button,),
            formid=formid,
        )

    def process_form(self, form):
        controls = self.request.POST.items()
        try:
            # TODO: uploader puts qqfile in controls
            controls = [control for control in controls if 'qqfile' not in control[0]]
            logger.debug("before validate %s", controls)
            appstruct = form.validate(controls)
            logger.debug("before execute %s", appstruct)
            self.execute(appstruct)
        except ValidationFailure, e:
            logger.exception('validation of exectue view failed.')
            self.session.flash("There are errors on this page.", queue='danger')
            return dict(description=getattr(self.process, 'abstract', ''),
                        url=wps_describe_url(self.wps.url, self.processid),
                        metadata=self.process.metadata,
                        form=e.render())
        if self.request.user is None:
            return HTTPFound(location=self.request.route_url('processes_loading'))
        else:
            return HTTPFound(location=self.request.route_url('monitor'))

    def execute(self, appstruct):
        inputs = appstruct_to_inputs(self.request, appstruct)
        # need to use ComplexDataInput
        complex_inpts = []
        for inpt in self.process.dataInputs:
            if 'ComplexData' in inpt.dataType:
                complex_inpts.append(inpt.identifier)
        new_inputs = []
        for inpt in inputs:
            if inpt[0] in complex_inpts:
                new_inputs.append((inpt[0], ComplexDataInput(inpt[1])))
            else:
                new_inputs.append(inpt)
        inputs = new_inputs
        # prepare outputs
        outputs = []
        for output in self.process.processOutputs:
            outputs.append(
                (output.identifier, output.dataType == 'ComplexData'))

        from phoenix.tasks.execute import execute_process
        result = execute_process.delay(
            userid=self.request.unauthenticated_userid,
            url=self.wps.url,
            identifier=self.process.identifier,
            inputs=inputs, outputs=outputs)
        self.session['task_id'] = result.id
        self.request.registry.notify(JobStarted(self.request, result.id))

    @view_config(renderer='json', route_name='processes_check_queue')
    def check_queue(self):
        status = 'running'
        task_id = self.session.get('task_id')
        collection = self.request.db.jobs
        if collection.find({"task_id": task_id}).count() == 1:
            status = 'ready'
        return dict(status=status)

    @view_config(route_name='processes_loading', renderer='../templates/processes/loading.pt')
    def loading(self):
        task_id = self.session.get('task_id')
        collection = self.request.db.jobs
        if collection.find({"task_id": task_id}).count() == 1:
            job = collection.find_one({"task_id": task_id})
            return HTTPFound(location=self.request.route_path(
                'monitor_details', tab='log', job_id=job.get('identifier')))
        return {}

    @view_config(route_name='processes_execute', renderer='../templates/processes/execute.pt')
    def view(self):
        form = self.generate_form()
        if 'submit' in self.request.POST:
            return self.process_form(form)
        if not has_execute_permission(self.request, self.service_name):
            self.session.flash("You are not allowed to execute processes. Please sign-in.", queue='warning')
        return dict(
            description=getattr(self.process, 'abstract', ''),
            url=wps_describe_url(self.wps.url, self.processid),
            metadata=self.process.metadata,
            form=form.render(self.appstruct()))
<commit_msg>using async_check param<commit_after>from pyramid.view import view_config, view_defaults
from pyramid.httpexceptions import HTTPFound
from deform import Form, Button
from deform import ValidationFailure

from phoenix.events import JobStarted
from phoenix.views import MyView
from phoenix.wps import appstruct_to_inputs
from phoenix.wps import WPSSchema
from phoenix.utils import wps_describe_url
from phoenix.security import has_execute_permission

from owslib.wps import WebProcessingService
from owslib.wps import WPSExecution
from owslib.wps import ComplexDataInput

import logging
logger = logging.getLogger(__name__)

@view_defaults(permission='view', layout='default')
class ExecuteProcess(MyView):
    def __init__(self, request):
        self.request = request
        self.execution = None
        if 'job_id' in request.params:
            job = request.db.jobs.find_one(
                {'identifier': request.params['job_id']})
            self.execution = WPSExecution()
            self.execution.checkStatus(url=job['status_location'], sleepSecs=0)
            self.processid = self.execution.process.identifier
            self.service_name = self.request.catalog.get_service_name_by_url(
                url=self.execution.serviceInstance)
        elif 'wps' in request.params:
            self.service_name = request.params.get('wps')
            self.processid = request.params.get('process')
        else:
            self.service_name = None
            self.processid = None
            self.process = None

        if self.service_name:
            # TODO: avoid getcaps
            self.wps = WebProcessingService(
                url=request.route_url('owsproxy',
                                      service_name=self.service_name),
                verify=False)
            # TODO: need to fix owslib to handle special identifiers
            self.process = self.wps.describeprocess(self.processid)
        super(ExecuteProcess, self).__init__(request, name='processes_execute',
                                             title='')

    def breadcrumbs(self):
        breadcrumbs = super(ExecuteProcess, self).breadcrumbs()
        breadcrumbs.append(dict(
            route_path=self.request.route_path('processes'),
            title='Processes'))
        if self.service_name:
            route_path = self.request.route_path(
                'processes_list', _query=[('wps', self.service_name)])
            breadcrumbs.append(dict(
                route_path=route_path,
                title=self.wps.identification.title))
            breadcrumbs.append(dict(
                route_path=self.request.route_path(self.name),
                title=self.process.title))
        return breadcrumbs

    def appstruct(self):
        # TODO: not a nice way to get inputs ... should be cleaned up in owslib
        result = {}
        if self.execution:
            for inp in self.execution.dataInputs:
                if inp.data or inp.reference:
                    if inp.identifier not in result:
                        # init result for param with empty list
                        result[inp.identifier] = []
                    if inp.data:
                        # add literal input, inp.data is a list
                        result[inp.identifier].extend(inp.data)
                    elif inp.reference:
                        # add reference to complex input
                        result[inp.identifier].append(inp.reference)
        for inp in self.process.dataInputs:
            # convert boolean
            if 'boolean' in inp.dataType and inp.identifier in result:
                result[inp.identifier] = [val.lower() == 'true' for val in result[inp.identifier]]
            # TODO: very dirty ... if single value then take the first
            if inp.maxOccurs < 2 and inp.identifier in result:
                result[inp.identifier] = result[inp.identifier][0]
        return result

    def generate_form(self, formid='deform'):
        schema = WPSSchema(request=self.request,
                           process=self.process,
                           user=self.get_user())
        submit_button = Button(name='submit', title='Execute',
                               css_class='btn btn-success btn-lg btn-block',
                               disabled=not has_execute_permission(
                                    self.request, self.service_name))
        return Form(
            schema,
            buttons=(submit_button,),
            formid=formid,
        )

    def process_form(self, form):
        controls = self.request.POST.items()
        try:
            # TODO: uploader puts qqfile in controls
            controls = [control for control in controls if 'qqfile' not in control[0]]
            logger.debug("before validate %s", controls)
            appstruct = form.validate(controls)
            logger.debug("before execute %s", appstruct)
            self.execute(appstruct)
        except ValidationFailure, e:
            logger.exception('validation of exectue view failed.')
            self.session.flash("There are errors on this page.", queue='danger')
            return dict(description=getattr(self.process, 'abstract', ''),
                        url=wps_describe_url(self.wps.url, self.processid),
                        metadata=self.process.metadata,
                        form=e.render())
        if self.request.user is None:
            return HTTPFound(location=self.request.route_url('processes_loading'))
        else:
            return HTTPFound(location=self.request.route_url('monitor'))

    def execute(self, appstruct):
        inputs = appstruct_to_inputs(self.request, appstruct)
        # need to use ComplexDataInput
        complex_inpts = []
        for inpt in self.process.dataInputs:
            if 'ComplexData' in inpt.dataType:
                complex_inpts.append(inpt.identifier)
        new_inputs = []
        for inpt in inputs:
            if inpt[0] in complex_inpts:
                new_inputs.append((inpt[0], ComplexDataInput(inpt[1])))
            else:
                new_inputs.append(inpt)
        inputs = new_inputs
        # prepare outputs
        outputs = []
        for output in self.process.processOutputs:
            outputs.append(
                (output.identifier, output.dataType == 'ComplexData'))

        from phoenix.tasks.execute import execute_process
        result = execute_process.delay(
            userid=self.request.unauthenticated_userid,
            url=self.wps.url,
            identifier=self.process.identifier,
            inputs=inputs,
            outputs=outputs,
            async=appstruct['_async_check'])
        self.session['task_id'] = result.id
        self.request.registry.notify(JobStarted(self.request, result.id))

    @view_config(renderer='json', route_name='processes_check_queue')
    def check_queue(self):
        status = 'running'
        task_id = self.session.get('task_id')
        collection = self.request.db.jobs
        if collection.find({"task_id": task_id}).count() == 1:
            status = 'ready'
        return dict(status=status)

    @view_config(route_name='processes_loading', renderer='../templates/processes/loading.pt')
    def loading(self):
        task_id = self.session.get('task_id')
        collection = self.request.db.jobs
        if collection.find({"task_id": task_id}).count() == 1:
            job = collection.find_one({"task_id": task_id})
            return HTTPFound(location=self.request.route_path(
                'monitor_details', tab='log', job_id=job.get('identifier')))
        return {}

    @view_config(route_name='processes_execute', renderer='../templates/processes/execute.pt')
    def view(self):
        form = self.generate_form()
        if 'submit' in self.request.POST:
            return self.process_form(form)
        if not has_execute_permission(self.request, self.service_name):
            self.session.flash("You are not allowed to execute processes. Please sign-in.", queue='warning')
        return dict(
            description=getattr(self.process, 'abstract', ''),
            url=wps_describe_url(self.wps.url, self.processid),
            metadata=self.process.metadata,
            form=form.render(self.appstruct()))
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
from astropy.tests.helper import pytest
import numpy as np
from numpy.testing import assert_allclose
import itertools
from ..morphology import (centroid_com, centroid_1dg, centroid_2dg,
                          gaussian1d_moments, data_properties)
from astropy.modeling import models
try:
    import skimage
    HAS_SKIMAGE = True
except ImportError:
    HAS_SKIMAGE = False

XCS = [25.0, 25.7, 26.1]
YCS = [25.0, 26.2, 26.9]
XSTDDEVS = [2.0, 3.2]
YSTDDEVS = [2.0, 5.7]
THETAS = np.array([30., 45.]) * np.pi / 180.

@pytest.mark.parametrize(('xc_ref', 'yc_ref', 'x_stddev', 'y_stddev', 'theta'),
                         list(itertools.product(XCS, YCS, XSTDDEVS, YSTDDEVS,
                                                THETAS)))
@pytest.mark.skipif('not HAS_SKIMAGE')
def test_centroids(xc_ref, yc_ref, x_stddev, y_stddev, theta):
    model = models.Gaussian2D(2.4, xc_ref, yc_ref, x_stddev=x_stddev,
                              y_stddev=y_stddev, theta=theta)
    y, x = np.mgrid[0:50, 0:50]
    data = model(x, y)
    xc, yc = centroid_com(data)
    assert_allclose([xc_ref, yc_ref], [xc, yc], rtol=0, atol=1.e-3)
    xc2, yc2 = centroid_1dg(data)
    assert_allclose([xc_ref, yc_ref], [xc2, yc2], rtol=0, atol=1.e-3)
    xc3, yc3 = centroid_2dg(data)
    assert_allclose([xc_ref, yc_ref], [xc3, yc3], rtol=0, atol=1.e-3)

@pytest.mark.skipif('not HAS_SKIMAGE')
def test_centroid_com_mask():
    """Test centroid_com with and without an image_mask."""
    data = np.ones((2, 2)).astype(np.float)
    mask = [[False, False], [True, True]]
    centroid = centroid_com(data, mask=None)
    centroid_mask = centroid_com(data, mask=mask)
    assert_allclose([0.5, 0.5], centroid, rtol=0, atol=1.e-6)
    assert_allclose([0.5, 0.0], centroid_mask, rtol=0, atol=1.e-6)

@pytest.mark.skipif('not HAS_SKIMAGE')
def test_centroid_com_mask_shape():
    """
    Test if ValueError raises if mask shape doesn't match data
    shape.
    """
    with pytest.raises(ValueError):
        mask = np.zeros((2, 2), dtype=bool)
        centroid_com(np.zeros((4, 4)), mask=mask)

#@pytest.mark.skipif('not HAS_SKIMAGE')
#def test_shape_params_mask():
#    data = np.ones((2, 2)).astype(np.float)
#    mask = [[False, False], [True, True]]
#    params = shape_params(data, mask=None)
#    params_mask = shape_params(data, mask=mask)
#    result = [params['xcen'], params['ycen'], params['eccen']]
#    result_mask = [params_mask['xcen'], params_mask['ycen'],
#                   params_mask['eccen']]
#    assert_allclose([0.5, 0.5, 0.0], result, rtol=0, atol=1.e-6)
#    assert_allclose([0.5, 0.0, 1.0], result_mask, rtol=0, atol=1.e-6)

def test_gaussian1d_moments():
    x = np.arange(100)
    ref = (75, 50, 5)
    g = models.Gaussian1D(*ref)
    data = g(x)
    result = gaussian1d_moments(data)
    assert_allclose(ref, result, rtol=0, atol=1.e-6)
<commit_msg>Add morphology tests<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
from astropy.tests.helper import pytest
import numpy as np
from numpy.testing import assert_allclose
import itertools
from ..morphology import (centroid_com, centroid_1dg, centroid_2dg,
                          gaussian1d_moments, data_properties)
from astropy.modeling import models
try:
    import skimage
    HAS_SKIMAGE = True
except ImportError:
    HAS_SKIMAGE = False

XCS = [25.7]
YCS = [26.2]
XSTDDEVS = [3.2, 4.0]
YSTDDEVS = [5.7, 4.1]
THETAS = np.array([30., 45.]) * np.pi / 180.

@pytest.mark.parametrize(
    ('xc_ref', 'yc_ref', 'x_stddev', 'y_stddev', 'theta'),
    list(itertools.product(XCS, YCS, XSTDDEVS, YSTDDEVS, THETAS)))
@pytest.mark.skipif('not HAS_SKIMAGE')
def test_centroids(xc_ref, yc_ref, x_stddev, y_stddev, theta):
    model = models.Gaussian2D(2.4, xc_ref, yc_ref, x_stddev=x_stddev,
                              y_stddev=y_stddev, theta=theta)
    y, x = np.mgrid[0:50, 0:50]
    data = model(x, y)
    xc, yc = centroid_com(data)
    assert_allclose([xc_ref, yc_ref], [xc, yc], rtol=0, atol=1.e-3)
    xc2, yc2 = centroid_1dg(data)
    assert_allclose([xc_ref, yc_ref], [xc2, yc2], rtol=0, atol=1.e-3)
    xc3, yc3 = centroid_2dg(data)
    assert_allclose([xc_ref, yc_ref], [xc3, yc3], rtol=0, atol=1.e-3)

@pytest.mark.parametrize(
    ('xc_ref', 'yc_ref', 'x_stddev', 'y_stddev', 'theta'),
    list(itertools.product(XCS, YCS, XSTDDEVS, YSTDDEVS, THETAS)))
@pytest.mark.skipif('not HAS_SKIMAGE')
def test_centroids_witherror(xc_ref, yc_ref, x_stddev, y_stddev, theta):
    model = models.Gaussian2D(2.4, xc_ref, yc_ref, x_stddev=x_stddev,
                              y_stddev=y_stddev, theta=theta)
    y, x = np.mgrid[0:50, 0:50]
    data = model(x, y)
    error = np.sqrt(data)
    xc2, yc2 = centroid_1dg(data, error=error)
    assert_allclose([xc_ref, yc_ref], [xc2, yc2], rtol=0, atol=1.e-3)
    xc3, yc3 = centroid_2dg(data, error=error)
    assert_allclose([xc_ref, yc_ref], [xc3, yc3], rtol=0, atol=1.e-3)

@pytest.mark.parametrize(
    ('xc_ref', 'yc_ref', 'x_stddev', 'y_stddev', 'theta'),
    list(itertools.product(XCS, YCS, XSTDDEVS, YSTDDEVS, THETAS)))
@pytest.mark.skipif('not HAS_SKIMAGE')
def test_centroids_withmask(xc_ref, yc_ref, x_stddev, y_stddev, theta):
    model = models.Gaussian2D(2.4, xc_ref, yc_ref, x_stddev=x_stddev,
                              y_stddev=y_stddev, theta=theta)
    y, x = np.mgrid[0:50, 0:50]
    data = model(x, y)
    mask = np.zeros_like(data, dtype=bool)
    data[18, 16] = 50.
    mask[18, 16] = True
    xc, yc = centroid_com(data, mask=mask)
    assert_allclose([xc_ref, yc_ref], [xc, yc], rtol=0, atol=1.e-3)
    xc2, yc2 = centroid_1dg(data, mask=mask)
    assert_allclose([xc_ref, yc_ref], [xc2, yc2], rtol=0, atol=1.e-3)
    xc3, yc3 = centroid_2dg(data, mask=mask)
    assert_allclose([xc_ref, yc_ref], [xc3, yc3], rtol=0, atol=1.e-3)

@pytest.mark.skipif('not HAS_SKIMAGE')
def test_centroids_mask():
    """Test centroid_com with and without an image_mask."""
    data = np.ones((2, 2)).astype(np.float)
    mask = [[False, False], [True, True]]
    centroid = centroid_com(data, mask=None)
    centroid_mask = centroid_com(data, mask=mask)
    assert_allclose([0.5, 0.5], centroid, rtol=0, atol=1.e-6)
    assert_allclose([0.5, 0.0], centroid_mask, rtol=0, atol=1.e-6)

@pytest.mark.skipif('not HAS_SKIMAGE')
def test_centroid_com_mask_shape():
    """
    Test if ValueError raises if mask shape doesn't match data
    shape.
    """
    with pytest.raises(ValueError):
        mask = np.zeros((2, 2), dtype=bool)
        centroid_com(np.zeros((4, 4)), mask=mask)

@pytest.mark.skipif('not HAS_SKIMAGE')
def test_data_properties():
    data = np.ones((2, 2)).astype(np.float)
    mask = np.array([[False, False], [True, True]])
    props = data_properties(data, mask=None)
    props2 = data_properties(data, mask=mask)
    properties = ['xcentroid', 'ycentroid', 'eccentricity']
    result = [props[i].value for i in properties]
    result2 = [props2[i].value for i in properties]
    assert_allclose([0.5, 0.5, 0.0], result, rtol=0, atol=1.e-6)
    assert_allclose([0.5, 0.0, 1.0], result2, rtol=0, atol=1.e-6)

def test_gaussian1d_moments():
    x = np.arange(100)
    ref = (75, 50, 5)
    g = models.Gaussian1D(*ref)
    data = g(x)
    result = gaussian1d_moments(data)
    assert_allclose(ref, result, rtol=0, atol=1.e-6)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import api
import api.group
from api.stats import (check_invalid_instance_submissions, get_all_team_scores,
                       get_group_scores, get_problem_solves,
                       get_registration_count,
                       get_top_teams_score_progressions)

def run():
    """Run the stat caching daemon."""
    with api.create_app().app_context():
        def cache(f, *args, **kwargs):
            result = f(reset_cache=True, *args, **kwargs)
            return result

        print("Caching registration stats...")
        cache(get_registration_count)

        print("Caching the scoreboards...")
        for scoreboard in api.scoreboards.get_all_scoreboards():
            get_all_team_scores(scoreboard_id=scoreboard['sid'])

        print("Caching the score progressions for each scoreboard...")
        for scoreboard in api.scoreboards.get_all_scoreboards():
            cache(get_top_teams_score_progressions,
                  limit=5,
                  scoreboard_id=scoreboard['sid'])

        print("Caching the scores and score progressions for each group...")
        for group in api.group.get_all_groups():
            get_group_scores(gid=group['gid'])
            cache(get_top_teams_score_progressions,
                  limit=5,
                  group_id=group['gid'])

        print("Caching number of solves for each problem...")
        for problem in api.problem.get_all_problems():
            print(problem["name"],
                  cache(get_problem_solves, problem["pid"]))

        print("Caching invalid instance submissions...")
        cache(check_invalid_instance_submissions)

if __name__ == '__main__':
    run()
<commit_msg>Set an active ctf-stats host in a multi-server setup<commit_after>#!/usr/bin/env python3

import api
import api.group
from api.stats import (check_invalid_instance_submissions, get_all_team_scores,
                       get_group_scores, get_problem_solves,
                       get_registration_count,
                       get_top_teams_score_progressions)
import socket

# How long after primary stat host falls off to allow another to take primary
COOLDOWN_TIME = 5 * 60

def run():
    """Run the stat caching daemon."""
    with api.create_app().app_context():
        def cache(f, *args, **kwargs):
            result = f(reset_cache=True, *args, **kwargs)
            return result

        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(("8.8.8.8", 80))
        host = s.getsockname()[0]
        s.close()

        _cache = api.cache.get_cache()
        active_stat_host = _cache.get("active_stat_host")
        if active_stat_host is not None and host != active_stat_host:
            raise SystemExit
        else:
            _cache.set("active_stat_host", host, COOLDOWN_TIME)

        print("Caching registration stats...")
        cache(get_registration_count)

        print("Caching the scoreboards...")
        for scoreboard in api.scoreboards.get_all_scoreboards():
            get_all_team_scores(scoreboard_id=scoreboard['sid'])

        print("Caching the score progressions for each scoreboard...")
        for scoreboard in api.scoreboards.get_all_scoreboards():
            cache(get_top_teams_score_progressions,
                  limit=5,
                  scoreboard_id=scoreboard['sid'])

        print("Caching the scores and score progressions for each group...")
        for group in api.group.get_all_groups():
            get_group_scores(gid=group['gid'])
            cache(get_top_teams_score_progressions,
                  limit=5,
                  group_id=group['gid'])

        print("Caching number of solves for each problem...")
        for problem in api.problem.get_all_problems():
            print(problem["name"],
                  cache(get_problem_solves, problem["pid"]))

        print("Caching invalid instance submissions...")
        cache(check_invalid_instance_submissions)

if __name__ == '__main__':
    run()
<|endoftext|>